From c0834dd4136fd91979f26c02514d723ab102bbcc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:31:42 +0100 Subject: [PATCH 01/40] wip(native): native SQL interpolation and template host (Ports A, paused) --- .../src/text/_helpers/alnum_tables.rs | 9 +- .../src/text/alpha_ranges/unicode_15_0_0.in | 663 +++++++++++++++++ .../src/text/alpha_ranges/unicode_15_1_0.in | 664 +++++++++++++++++ .../src/text/alpha_ranges/unicode_16_0_0.in | 681 ++++++++++++++++++ .../src/text/main/is_python_alnum.rs | 12 +- .../src/text/main/is_python_alpha.rs | 12 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + .../src/text/main/python_text.rs | 7 +- crates/sqlbuild-core/src/text/models.rs | 2 + .../src/text/tests/test_python_compat.rs | 19 + .../src/text/tests/test_types.rs | 1 + crates/sqlbuild-model-config/Cargo.toml | 3 + .../src/templates/_helpers/evaluation.rs | 32 +- .../src/templates/_helpers/tokens.rs | 14 +- .../src/templates/errors.rs | 2 + .../templates/main/expand_template_string.rs | 30 +- .../templates/main/template_error_message.rs | 15 + .../src/templates/models.rs | 8 +- .../tests/test_expand_template_string.rs | 30 +- .../src/templates/tests/test_types.rs | 30 +- .../src/templates/types.rs | 8 +- .../_helpers/model_config/config_templates.rs | 290 ++++++-- .../model_config/model_config_builder.rs | 5 +- .../bindings/_helpers/sqltext/static_sql.rs | 178 ++++- .../sql_interpolation/substitution.rs | 404 +++++++---- .../src/compiler/main/sql_interpolation.rs | 21 +- .../sqlbuild-sqltext/src/compiler/models.rs | 22 + .../src/compiler/tests/helpers.rs | 183 +++-- .../compiler/tests/test_sql_interpolation.rs | 175 ++++- .../src/compiler/tests/test_types.rs | 7 + crates/sqlbuild-sqltext/src/compiler/types.rs | 19 + scripts/generate_python_char_table.py | 4 +- scripts/python_char_table/_helpers/table.py | 27 +- scripts/python_char_table/constants.py | 10 +- scripts/python_char_table/main/generate.py | 17 +- src/sqlbuild/_native.pyi | 18 +- .../compile/_helpers/attachment/core.py | 41 +- .../compile/_helpers/render/sql_vars.py | 419 ++--------- src/sqlbuild/compiler/compile/models.py | 10 + src/sqlbuild/compiler/frontier/types.py | 1 - .../sqlbuild/compiler/attachments/helpers.py | 2 +- .../test_native_attachment_project.py | 2 +- .../compiler/compile/_helpers/_test_types.py | 15 +- .../compiler/compile/_helpers/helpers.py | 106 --- .../compile/_helpers/test_sql_vars.py | 113 +-- .../frontier/test_native_fallback_recorder.py | 10 +- 46 files changed, 3254 insertions(+), 1088 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in create mode 100644 crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in create mode 100644 crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in create mode 100644 crates/sqlbuild-core/src/text/main/is_python_alpha.rs diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs index cf4cef94a7..38b3a6b9a5 100644 --- a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs @@ -1,4 +1,4 @@ -//! The generated `str.isalnum()` tables, one per Unicode version of a supported CPython. +//! The generated `str.isalnum()` and `str.isalpha()` tables, one per supported Unicode version. /// `(unicodedata.unidata_version, ranges)` for every table native text helpers can select. pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ @@ -6,3 +6,10 @@ pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ ("15.1.0", include!("../alnum_ranges/unicode_15_1_0.in")), ("16.0.0", include!("../alnum_ranges/unicode_16_0_0.in")), ]; + +/// `(unicodedata.unidata_version, ranges)` of `str.isalpha()`, in the order of `ALNUM_TABLES`. +pub(crate) const ALPHA_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../alpha_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../alpha_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../alpha_ranges/unicode_16_0_0.in")), +]; diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in new file mode 100644 index 0000000000..303ce0bae7 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in @@ -0,0 +1,663 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 15.0.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in new file mode 100644 index 0000000000..ceaf88df48 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in @@ -0,0 +1,664 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 15.1.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in new file mode 100644 index 0000000000..d2080cb95d --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in @@ -0,0 +1,681 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 16.0.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CD), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7DC), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10D4A, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC4), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18CFF, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5F0), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs index f8575f0ba2..de2b45f741 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs @@ -7,12 +7,14 @@ pub fn is_python_alnum(python: PythonText, character: char) -> bool { if character.is_ascii() { return character.is_ascii_alphanumeric(); } + in_ranges(python.alnum_ranges, character) +} + +/// Whether one character falls inside the inclusive, ordered code point `ranges`. +pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { let code_point: u32 = u32::from(character); - let index: usize = python - .alnum_ranges - .partition_point(|(_, end)| *end < code_point); - python - .alnum_ranges + let index: usize = ranges.partition_point(|(_, end)| *end < code_point); + ranges .get(index) .is_some_and(|(start, _)| *start <= code_point) } diff --git a/crates/sqlbuild-core/src/text/main/is_python_alpha.rs b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs new file mode 100644 index 0000000000..36f3bcf40c --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs @@ -0,0 +1,12 @@ +//! Python's `str.isalpha()` for one character, from a table generated by Python. + +use crate::text::main::is_python_alnum::in_ranges; +use crate::text::models::PythonText; + +/// Whether Python's `str.isalpha()` is true for one character. +pub fn is_python_alpha(python: PythonText, character: char) -> bool { + if character.is_ascii() { + return character.is_ascii_alphabetic(); + } + in_ranges(python.alpha_ranges, character) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index a0a0883793..3944a5830d 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -1,6 +1,7 @@ pub mod close_matches; pub mod decode_python_text; pub mod is_python_alnum; +pub mod is_python_alpha; pub mod is_python_space; pub mod is_python_word; pub mod python_cleandoc; diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs index 23f6a8efee..1208a686b7 100644 --- a/crates/sqlbuild-core/src/text/main/python_text.rs +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -1,6 +1,6 @@ //! Select the Python string semantics of one CPython release. -use crate::text::_helpers::alnum_tables::ALNUM_TABLES; +use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES}; use crate::text::models::{CleandocMargin, PythonText}; /// The semantics of one Python and its `unicodedata.unidata_version`, or `None` if unknown. @@ -14,8 +14,13 @@ pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option LineIndex<'text> { pub struct PythonText { /// Inclusive code point ranges where `str.isalnum()` is true, in order. pub(crate) alnum_ranges: &'static [(u32, u32)], + /// Inclusive code point ranges where `str.isalpha()` is true, in order. + pub(crate) alpha_ranges: &'static [(u32, u32)], pub(crate) cleandoc_margin: CleandocMargin, } diff --git a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs index e78053d265..0043b9ae23 100644 --- a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs +++ b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs @@ -1,5 +1,6 @@ use crate::text::main::close_matches::close_matches; use crate::text::main::is_python_alnum::is_python_alnum; +use crate::text::main::is_python_alpha::is_python_alpha; use crate::text::main::is_python_space::is_python_space; use crate::text::main::is_python_word::is_python_word; use crate::text::main::python_cleandoc::python_cleandoc; @@ -93,6 +94,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "ASCII letter", character: 'a', + expected_alpha: true, expected_alnum: true, expected_word: true, expected_space: false, @@ -100,6 +102,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "underscore is a word character only", character: '_', + expected_alpha: false, expected_alnum: false, expected_word: true, expected_space: false, @@ -107,6 +110,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "Arabic-Indic digit", character: '\u{663}', + expected_alpha: false, expected_alnum: true, expected_word: true, expected_space: false, @@ -114,6 +118,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "letter number", character: '\u{2167}', + expected_alpha: false, expected_alnum: true, expected_word: true, expected_space: false, @@ -121,6 +126,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "combining mark that Rust calls alphabetic", character: '\u{345}', + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: false, @@ -128,6 +134,7 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "circled letter symbol", character: '\u{24b6}', + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: false, @@ -135,13 +142,23 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "information separator is whitespace", character: '\u{1c}', + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: true, }, + CharacterClassTestCase { + description: "Latin letter with an accent is alphabetic", + character: '\u{e9}', + expected_alpha: true, + expected_alnum: true, + expected_word: true, + expected_space: false, + }, CharacterClassTestCase { description: "ideographic space", character: '\u{3000}', + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: true, @@ -151,11 +168,13 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { for test_case in test_cases { assert_eq!( ( + is_python_alpha(python, test_case.character), is_python_alnum(python, test_case.character), is_python_word(python, test_case.character), is_python_space(test_case.character) ), ( + test_case.expected_alpha, test_case.expected_alnum, test_case.expected_word, test_case.expected_space diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index 18e6fbd116..5d109b3375 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -30,6 +30,7 @@ pub(super) struct CloseMatchesTestCase { pub(super) struct CharacterClassTestCase { pub(super) description: &'static str, pub(super) character: char, + pub(super) expected_alpha: bool, pub(super) expected_alnum: bool, pub(super) expected_word: bool, pub(super) expected_space: bool, diff --git a/crates/sqlbuild-model-config/Cargo.toml b/crates/sqlbuild-model-config/Cargo.toml index a4cf9e2917..15f3596bb0 100644 --- a/crates/sqlbuild-model-config/Cargo.toml +++ b/crates/sqlbuild-model-config/Cargo.toml @@ -9,3 +9,6 @@ rust-version.workspace = true [lints] workspace = true + +[dependencies] +sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs index 9564a6139e..a297ddc3cf 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs @@ -1,6 +1,10 @@ //! The Python template evaluator: references, `if`, `eq`, `ne` and `coalesce`. use crate::templates::errors::TemplateError; +use crate::templates::main::template_error_message::{ + is_missing_value_message, template_error_message, +}; +use sqlbuild_core::text::main::python_strip::python_strip; use crate::templates::models::{ ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, }; @@ -35,14 +39,6 @@ pub(crate) fn evaluate( } } -/// Render a value as interpolated text, as `render_project_var_text` does for scalars. -pub(crate) fn render_text( - host: &H, - value: &H::Value, -) -> Result { - comparison_text(host, value) -} - fn evaluate_reference( host: &H, reference: &str, @@ -116,7 +112,12 @@ fn evaluate_function( for argument in arguments { match evaluate(host, argument, options) { Ok(value) if truthiness(host, &value)? => return Ok(value), - Ok(_) | Err(TemplateFailure::Missing(_)) => {} + Ok(_) => {} + Err(TemplateFailure::Missing(error) | TemplateFailure::Invalid(error)) + if is_missing_value_message(&template_error_message( + &error, + host.label(), + )) => {} Err(failure) => return Err(failure), } } @@ -134,7 +135,7 @@ fn argument_count(function: &'static str, expected: &'static str) -> TemplateFai } fn comparison_text(host: &H, value: &H::Value) -> Result { - match host.scalar(value).ok_or(TemplateFailure::Unsupported)? { + match host.scalar(value)? { Scalar::Null => Ok(String::new()), Scalar::Bool(flag) => Ok(if flag { TRUE_LITERAL } else { FALSE_LITERAL }.to_owned()), Scalar::Text(text) => Ok(text), @@ -142,17 +143,12 @@ fn comparison_text(host: &H, value: &H::Value) -> Result(host: &H, value: &H::Value) -> Result { - match host.scalar(value).ok_or(TemplateFailure::Unsupported)? { + match host.scalar(value)? { Scalar::Null => Ok(false), Scalar::Bool(flag) => Ok(flag), - Scalar::Text(text) if text.is_ascii() => { - let normalized = text - .trim_matches( - |character: char| matches!(character, '\t'..='\r' | '\u{1c}'..='\u{1f}' | ' '), - ) - .to_ascii_lowercase(); + Scalar::Text(text) => { + let normalized = python_strip(&text).to_lowercase(); Ok(!FALSE_VALUES.contains(&normalized.as_str())) } - Scalar::Text(_) => Err(TemplateFailure::Unsupported), } } diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs index 83a2f6aa79..f3bef193f1 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs @@ -1,6 +1,7 @@ -//! The Python template tokenizer, deferring wherever Python's `str.isspace` would decide. +//! The Python template tokenizer. use crate::templates::errors::TemplateError; +use sqlbuild_core::text::main::is_python_space::is_python_space; use crate::templates::models::TemplateFailure; #[derive(Clone, Debug, PartialEq, Eq)] @@ -38,9 +39,6 @@ pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> let mut tokens: Vec = Vec::new(); let mut index = 0; while let Some(&character) = characters.get(index) { - if !character.is_ascii() { - return Err(TemplateFailure::Unsupported); - } if is_python_space(character) { index += 1; } else if SYMBOLS.contains(&character) { @@ -59,9 +57,6 @@ pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> } else { let start = index; while let Some(&next) = characters.get(index) { - if !next.is_ascii() { - return Err(TemplateFailure::Unsupported); - } if is_python_space(next) || SYMBOLS.contains(&next) || is_quote(next) { break; } @@ -112,8 +107,3 @@ fn quoted_text(characters: &[char], start: usize) -> Result<(String, usize), Tem fn is_quote(character: char) -> bool { matches!(character, '\'' | '"') } - -/// ASCII characters Python's `str.isspace` accepts, including the information separators. -fn is_python_space(character: char) -> bool { - matches!(character, '\t'..='\r' | '\u{1c}'..='\u{1f}' | ' ') -} diff --git a/crates/sqlbuild-model-config/src/templates/errors.rs b/crates/sqlbuild-model-config/src/templates/errors.rs index a22eb11942..ce9d379beb 100644 --- a/crates/sqlbuild-model-config/src/templates/errors.rs +++ b/crates/sqlbuild-model-config/src/templates/errors.rs @@ -32,4 +32,6 @@ pub enum TemplateError { quote: &'static str, position: usize, }, + /// An error the host worded, such as a value that cannot be interpolated as text. + Message(String), } diff --git a/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs index ebd69d0f51..b88496de79 100644 --- a/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs @@ -1,6 +1,6 @@ //! Expand the `${...}` templates in one string exactly as the Python template resolver does. -use crate::templates::_helpers::evaluation::{evaluate, render_text}; +use crate::templates::_helpers::evaluation::evaluate; use crate::templates::_helpers::expressions::parse_expression; use crate::templates::_helpers::pattern::template_spans; use crate::templates::models::{StringExpansion, TemplateFailure, TemplateOptions}; @@ -23,15 +23,35 @@ pub fn expand_template_string( if spans.is_empty() { return Ok(StringExpansion::Unchanged); } + substituted(host, text, &spans, options).map(StringExpansion::Text) +} + +/// Substitute every template in `text` as text, as `expand_template_string` does in Python. +pub fn expand_template_text( + host: &H, + text: &str, + options: TemplateOptions, +) -> Result { + substituted(host, text, &template_spans(text), options) +} + +fn substituted( + host: &H, + text: &str, + spans: &[(usize, usize)], + options: TemplateOptions, +) -> Result { let mut expanded = String::with_capacity(text.len()); let mut copied = 0; - for (start, end) in spans { + for &(start, end) in spans { expanded.push_str(&text[copied..start]); - let expression = parse_expression(&text[start + 2..end - 1])?; + let body = &text[start + 2..end - 1]; + let expression = parse_expression(body)?; let value = evaluate(host, &expression, options)?; - expanded.push_str(&render_text(host, &value)?); + let label = format!("{} variable '{body}'", host.label()); + expanded.push_str(&host.render(&value, &label)?); copied = end; } expanded.push_str(&text[copied..]); - Ok(StringExpansion::Text(expanded)) + Ok(expanded) } diff --git a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs index 65c24ad112..47c533b600 100644 --- a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs +++ b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs @@ -41,5 +41,20 @@ pub fn template_error_message(error: &TemplateError, label: &str) -> String { TemplateError::UnterminatedString { quote, position } => format!( "template expression has unterminated {quote}-quoted string at position {position}" ), + TemplateError::Message(message) => message.clone(), } } + +/// Whether Python's `coalesce` treats this error message as a missing value and skips it. +pub fn is_missing_value_message(message: &str) -> bool { + MISSING_VALUE_PARTS.iter().any(|part| message.contains(part)) + || (message.contains(UNAVAILABLE_CONTEXT_PART) && message.contains(NO_VALUE_PART)) +} + +const MISSING_VALUE_PARTS: [&str; 3] = [ + "references missing ENV variable", + "references unknown variable", + "references unknown CTX key", +]; +const UNAVAILABLE_CONTEXT_PART: &str = "references CTX key"; +const NO_VALUE_PART: &str = "no value is available"; diff --git a/crates/sqlbuild-model-config/src/templates/models.rs b/crates/sqlbuild-model-config/src/templates/models.rs index ac9385f925..5b77bd385e 100644 --- a/crates/sqlbuild-model-config/src/templates/models.rs +++ b/crates/sqlbuild-model-config/src/templates/models.rs @@ -13,12 +13,12 @@ pub enum Expression { }, } -/// A value whose text Python's `str()` produces without calling user code. +/// A value as the Python resolver compares and tests it: `None`, a `bool` or its `str()`. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Scalar { Null, Bool(bool), - /// An exact `str` or the decimal digits of an exact `int`. + /// The value's `str()`. Text(String), } @@ -38,8 +38,8 @@ pub enum TemplateFailure { Missing(TemplateError), /// Another error Python raises for this template. Invalid(TemplateError), - /// Python expands this template with rules the native expansion does not reproduce. - Unsupported, + /// The host raised an error of its own, which it holds and reports. + Host, } /// How evaluation treats `CTX:` references, mirroring the Python resolver's flags. diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs index 0d66e68a3f..3e6997757e 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs @@ -151,15 +151,37 @@ fn given_template_strings_when_expanding_then_values_and_reads_match_python() { expected_reads: &[], }, TemplateTestCase { - description: "rendering an opaque value is left to Python", + description: "a mapping cannot be interpolated into text", text: "x_${opaque}", - expected: Err(TemplateFailure::Unsupported), + expected: Err(TemplateFailure::Invalid(TemplateError::Message( + "model config variable 'opaque' is an object".to_owned(), + ))), expected_reads: &[], }, TemplateTestCase { - description: "non-ASCII outside quotes is left to Python's whitespace rules", + description: "non-ASCII Python whitespace separates tokens", text: "${coalesce(\u{a0}env)}", - expected: Err(TemplateFailure::Unsupported), + expected: value(Value::Text("prod".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "a mapping is truthy by its text and comparable", + text: "${if(eq(opaque, \"{'a': 1}\"), 'same', 'different')}", + expected: value(Value::Text("same".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "non-ASCII text is truthy unless it strips and lowers to a false value", + text: "${coalesce('\u{3000}FALSE\u{3000}', 'Ünset')}", + expected: value(Value::Text("Ünset".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "coalesce stops at an error that is not a missing value", + text: "${coalesce(other:thing, env)}", + expected: Err(TemplateFailure::Invalid(TemplateError::UnsupportedNamespace( + "other".to_owned(), + ))), expected_reads: &[], }, TemplateTestCase { diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_types.rs b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs index 793c262d2a..18cdac2a02 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; +use crate::templates::errors::TemplateError; use crate::templates::models::{ContextValue, Scalar, StringExpansion, TemplateFailure}; use crate::templates::types::TemplateHost; @@ -9,7 +10,7 @@ pub(super) enum Value { Null, Bool(bool), Text(String), - /// A value only Python can render, such as a float or a mapping. + /// A mapping, which compares by its `str()` but cannot be interpolated. Opaque, } @@ -62,14 +63,29 @@ impl TemplateHost for TestHost { Value::Null } - fn scalar(&self, value: &Value) -> Option { - match value { - Value::Null => Some(Scalar::Null), - Value::Bool(flag) => Some(Scalar::Bool(*flag)), - Value::Text(text) => Some(Scalar::Text(text.clone())), - Value::Opaque => None, + fn scalar(&self, value: &Value) -> Result { + Ok(match value { + Value::Null => Scalar::Null, + Value::Bool(flag) => Scalar::Bool(*flag), + Value::Text(text) => Scalar::Text(text.clone()), + Value::Opaque => Scalar::Text("{'a': 1}".to_owned()), + }) + } + + fn render(&self, value: &Value, label: &str) -> Result { + match self.scalar(value)? { + Scalar::Null => Ok(String::new()), + Scalar::Bool(flag) => Ok(flag.to_string()), + Scalar::Text(_) if *value == Value::Opaque => Err(TemplateFailure::Invalid( + TemplateError::Message(format!("{label} is an object")), + )), + Scalar::Text(text) => Ok(text), } } + + fn label(&self) -> &str { + "model config" + } } /// One template string, its expansion and the reads it records. diff --git a/crates/sqlbuild-model-config/src/templates/types.rs b/crates/sqlbuild-model-config/src/templates/types.rs index 1c3adb5926..66f326eb7d 100644 --- a/crates/sqlbuild-model-config/src/templates/types.rs +++ b/crates/sqlbuild-model-config/src/templates/types.rs @@ -19,6 +19,10 @@ pub trait TemplateHost { fn boolean(&self, flag: bool) -> Result; /// Return the null value. fn null(&self) -> Self::Value; - /// Return the scalar `value` is, or `None` when only Python can render or compare it. - fn scalar(&self, value: &Self::Value) -> Option; + /// Return `value` as the resolver compares it: `None`, a `bool` or its `str()`. + fn scalar(&self, value: &Self::Value) -> Result; + /// Render `value` inside a larger string as `render_project_var_text` does for `label`. + fn render(&self, value: &Self::Value, label: &str) -> Result; + /// The context label error messages start with. + fn label(&self) -> &str; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index f5993fc70a..0f0a440a08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -1,15 +1,19 @@ -//! `${...}` templates in authored Python config values, expanded natively. +//! `${...}` templates in authored Python config values and effective vars, expanded natively. use std::cell::RefCell; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ - PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::errors::TemplateError; +use sqlbuild_model_config::templates::main::expand_template_string::{ + expand_template_string, expand_template_text, +}; use sqlbuild_model_config::templates::main::template_error_message::template_error_message; use sqlbuild_model_config::templates::models::{ ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, @@ -21,22 +25,36 @@ use crate::bindings::_helpers::model_config::config_errors::native_config_error; const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -const UNSUPPORTED_OUTCOME: &str = "unsupported"; +const EFFECTIVE_VARS_LABEL: &str = "effective vars"; +const PROJECT_VAR_VALUES_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; +const RENDER_PROJECT_VAR_TEXT: &str = "render_project_var_text"; /// Variables, the process environment and context values, all read through Python. pub(crate) struct PythonHost<'py> { py: Python<'py>, sources: TemplateSources<'py>, + label: String, + effective: Option>, reads: RefCell>, + raised: RefCell>, +} + +/// Effective vars resolved on demand, as Python's `_EffectiveVarsResolver` resolves them. +struct EffectiveVars<'py> { + resolved: Bound<'py, PyDict>, + resolving: RefCell>, } impl<'py> PythonHost<'py> { /// Read variables, the environment and context values from these Python objects. - pub(crate) fn new(py: Python<'py>, sources: TemplateSources<'py>) -> Self { + pub(crate) fn new(py: Python<'py>, sources: TemplateSources<'py>, label: &str) -> Self { Self { py, sources, + label: label.to_owned(), + effective: None, reads: RefCell::new(Vec::new()), + raised: RefCell::new(None), } } @@ -44,6 +62,77 @@ impl<'py> PythonHost<'py> { pub(crate) fn into_reads(self) -> Vec<(&'static str, String)> { self.reads.into_inner() } + + /// Hold a Python error until the expansion returns, and stop evaluation. + fn hold(&self, error: PyErr) -> TemplateFailure { + self.raised.borrow_mut().get_or_insert(error); + TemplateFailure::Host + } + + /// The stop for `failure`, taking the held Python error of a host failure. + pub(crate) fn stop(&self, failure: TemplateFailure) -> Stop { + match failure { + TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { + Stop::Failure(error) + } + TemplateFailure::Host => Stop::Python(self.raised.borrow_mut().take().unwrap_or_else( + || PyRuntimeError::new_err("template expansion stopped without an error"), + )), + } + } + + fn text_of(&self, value: &Bound<'py, PyAny>) -> Result { + value + .downcast::() + .map_err(|error| self.hold(error.into()))? + .to_str() + .map(str::to_owned) + .map_err(|error| self.hold(error)) + } + + /// Resolve effective var `name`, keeping Python's resolving stack on errors. + fn resolve( + &self, + effective: &EffectiveVars<'py>, + name: &str, + ) -> Result>, TemplateFailure> { + let Some(raw) = self.sources.0.get_item(name).map_err(|error| self.hold(error))? else { + return Ok(None); + }; + if let Some(value) = effective + .resolved + .get_item(name) + .map_err(|error| self.hold(error))? + { + return Ok(Some(value)); + } + if effective.resolving.borrow().iter().any(|key| key == name) { + let mut cycle = effective.resolving.borrow().clone(); + cycle.push(name.to_owned()); + return Err(TemplateFailure::Invalid(TemplateError::Message(format!( + "{EFFECTIVE_VARS_LABEL} contain a cyclic reference: {}", + cycle.join(" -> ") + )))); + } + effective.resolving.borrow_mut().push(name.to_owned()); + let value = if raw.is_instance_of::() { + let text = self.text_of(&raw)?; + let options = TemplateOptions { + allow_context: false, + preserve_context_tokens: false, + preserve_unknown_context: false, + }; + PyString::new(self.py, &expand_template_text(self, &text, options)?).into_any() + } else { + raw + }; + effective.resolving.borrow_mut().pop(); + effective + .resolved + .set_item(name, &value) + .map_err(|error| self.hold(error))?; + Ok(Some(value)) + } } /// The variables, the `os.environ` mapping and the context values one expansion reads. @@ -58,37 +147,28 @@ pub(crate) struct TemplateSources<'py>( #[derive(FromPyObject)] struct TemplateFlags(bool, bool, bool, String); -/// A template failure, or a Python error raised while building the expanded containers. +/// A template error, or a Python error raised while reading values or building containers. pub(crate) enum Stop { - Failure(TemplateFailure), + Failure(TemplateError), Python(PyErr), } -impl From for Stop { - fn from(failure: TemplateFailure) -> Self { - Self::Failure(failure) - } -} - -/// The `CompileInputError` Python raises for a failed template in the context `label`. -pub(crate) fn template_error(failure: &TemplateFailure, label: &str) -> Option { - match failure { - TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { - Some(ConfigError::compile(template_error_message(error, label))) - } - TemplateFailure::Unsupported => None, - } -} - -fn unsupported(_: PyErr) -> TemplateFailure { - TemplateFailure::Unsupported +/// The `CompileInputError` Python raises for a template error in the context `label`. +pub(crate) fn template_error(error: &TemplateError, label: &str) -> ConfigError { + ConfigError::compile(template_error_message(error, label)) } impl<'py> TemplateHost for PythonHost<'py> { type Value = Bound<'py, PyAny>; fn variable(&self, name: &str) -> Result, TemplateFailure> { - self.sources.0.get_item(name).map_err(unsupported) + if let Some(effective) = &self.effective { + return self.resolve(effective, name); + } + self.sources + .0 + .get_item(name) + .map_err(|error| self.hold(error)) } fn environment(&self, name: &str) -> Result, TemplateFailure> { @@ -96,8 +176,11 @@ impl<'py> TemplateHost for PythonHost<'py> { .borrow_mut() .push((ENVIRONMENT_READ, name.to_owned())); let environment: &Bound<'py, PyAny> = &self.sources.1; - if environment.contains(name).map_err(unsupported)? { - environment.get_item(name).map(Some).map_err(unsupported) + if environment.contains(name).map_err(|error| self.hold(error))? { + environment + .get_item(name) + .map(Some) + .map_err(|error| self.hold(error)) } else { Ok(None) } @@ -107,7 +190,12 @@ impl<'py> TemplateHost for PythonHost<'py> { self.reads .borrow_mut() .push((CONTEXT_READ, name.to_owned())); - match self.sources.2.get_item(name).map_err(unsupported)? { + match self + .sources + .2 + .get_item(name) + .map_err(|error| self.hold(error))? + { None => Ok(ContextValue::Unknown), Some(value) if value.is_none() => Ok(ContextValue::Unavailable), Some(value) => Ok(ContextValue::Value(value)), @@ -126,26 +214,46 @@ impl<'py> TemplateHost for PythonHost<'py> { self.py.None().into_bound(self.py) } - fn scalar(&self, value: &Self::Value) -> Option { + fn scalar(&self, value: &Self::Value) -> Result { if value.is_none() { - Some(Scalar::Null) - } else if let Ok(flag) = value.downcast_exact::() { - Some(Scalar::Bool(flag.is_true())) - } else if value.is_exact_instance_of::() - && let Ok(number) = value.extract::() - { - Some(Scalar::Text(number.to_string())) - } else if let Ok(text) = value.downcast_exact::() - && let Ok(text) = text.to_str() - { - Some(Scalar::Text(text.to_owned())) - } else { - None + return Ok(Scalar::Null); + } + if let Ok(flag) = value.downcast_exact::() { + return Ok(Scalar::Bool(flag.is_true())); + } + let text = value.str().map_err(|error| self.hold(error))?; + self.text_of(text.as_any()).map(Scalar::Text) + } + + fn render(&self, value: &Self::Value, label: &str) -> Result { + if value.is_exact_instance_of::() { + return self.text_of(value); + } + let rendered = self + .py + .import(PROJECT_VAR_VALUES_MODULE) + .and_then(|module| module.getattr(RENDER_PROJECT_VAR_TEXT)) + .and_then(|render| { + let arguments = PyDict::new(self.py); + arguments.set_item("value", value)?; + arguments.set_item("label", label)?; + render.call((), Some(&arguments)) + }); + match rendered { + Ok(text) => self.text_of(&text), + Err(error) if error.is_instance_of::(self.py) => Err( + TemplateFailure::Invalid(TemplateError::Message(error.value(self.py).to_string())), + ), + Err(error) => Err(self.hold(error)), } } + + fn label(&self) -> &str { + &self.label + } } -/// Return `(value, reads)` as `expand_template_data` would, `(error, reads)` or `unsupported`. +/// Return `(value, reads)` as `expand_template_data` would, or `(error, reads)`. #[pyfunction] fn expand_config_templates<'py>( py: Python<'py>, @@ -160,41 +268,79 @@ fn expand_config_templates<'py>( preserve_context_tokens, preserve_unknown_context, }; - let host = PythonHost::new(py, sources); - match expanded(&host, &value, options) { - Ok(result) => Ok((result, host.into_reads()) - .into_pyobject(py)? - .into_any() - .unbind()), - Err(Stop::Failure(failure)) => match template_error(&failure, &label) { - Some(error) => Ok((native_config_error(py, error)?, host.into_reads()) - .into_pyobject(py)? - .into_any() - .unbind()), - None => Ok(PyString::new(py, UNSUPPORTED_OUTCOME).into_any().unbind()), - }, - Err(Stop::Python(error)) => Err(error), + let host = PythonHost::new(py, sources, &label); + let result = expanded(&host, &value, options); + outcome(py, host, result, &label) +} + +/// Return `(resolved vars, reads)` as `expand_effective_vars` would, or `(error, reads)`. +#[pyfunction] +fn expand_effective_vars<'py>( + py: Python<'py>, + raw_values: Bound<'py, PyDict>, + environment: Bound<'py, PyAny>, +) -> PyResult> { + let mut host = PythonHost::new( + py, + TemplateSources(raw_values.clone(), environment, PyDict::new(py)), + EFFECTIVE_VARS_LABEL, + ); + let resolved = PyDict::new(py); + host.effective = Some(EffectiveVars { + resolved: resolved.clone(), + resolving: RefCell::new(Vec::new()), + }); + let mut result: Result, Stop> = Ok(resolved.clone().into_any()); + for key in raw_values.keys() { + let name = key.str()?.to_str()?.to_owned(); + if let Err(failure) = host.variable(&name) { + result = Err(host.stop(failure)); + break; + } } + outcome(py, host, result, EFFECTIVE_VARS_LABEL) +} + +fn outcome<'py>( + py: Python<'py>, + host: PythonHost<'py>, + result: Result, Stop>, + label: &str, +) -> PyResult> { + let value = match result { + Ok(value) => value, + Err(Stop::Failure(error)) => native_config_error(py, template_error(&error, label))? + .into_pyobject(py)? + .into_any(), + Err(Stop::Python(error)) => return Err(error), + }; + Ok((value, host.into_reads()) + .into_pyobject(py)? + .into_any() + .unbind()) } +/// Expand every template in `value` and the containers inside it, as Python's +/// `expand_template_data` does. pub(crate) fn expanded<'py>( host: &PythonHost<'py>, value: &Bound<'py, PyAny>, options: TemplateOptions, ) -> Result, Stop> { let py = host.py; - if let Ok(text) = value.downcast_exact::() { - let text = text.to_str().map_err(|_| TemplateFailure::Unsupported)?; + if let Ok(text) = value.downcast::() { + let text = text.to_str().map_err(Stop::Python)?; if !text.contains(TEMPLATE_OPEN_TOKEN) { return Ok(value.clone()); } - return Ok(match expand_template_string(host, text, options)? { - StringExpansion::Unchanged => value.clone(), - StringExpansion::Value(result) => result, - StringExpansion::Text(result) => PyString::new(py, &result).into_any(), - }); + return match expand_template_string(host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), + Ok(StringExpansion::Value(result)) => Ok(result), + Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), + Err(failure) => Err(host.stop(failure)), + }; } - if let Ok(mapping) = value.downcast_exact::() { + if let Ok(mapping) = value.downcast::() { let result = PyDict::new(py); for (key, item) in mapping.iter() { result @@ -203,7 +349,7 @@ pub(crate) fn expanded<'py>( } return Ok(result.into_any()); } - if let Ok(items) = value.downcast_exact::() { + if let Ok(items) = value.downcast::() { let result = items .iter() .map(|item| expanded(host, &item, options)) @@ -212,7 +358,7 @@ pub(crate) fn expanded<'py>( .map(Bound::into_any) .map_err(Stop::Python); } - if let Ok(items) = value.downcast_exact::() { + if let Ok(items) = value.downcast::() { let result = items .iter() .map(|item| expanded(host, &item, options)) @@ -221,17 +367,11 @@ pub(crate) fn expanded<'py>( .map(Bound::into_any) .map_err(Stop::Python); } - if value.is_instance_of::() - || value.is_instance_of::() - || value.is_instance_of::() - || value.is_instance_of::() - { - return Err(Stop::Failure(TemplateFailure::Unsupported)); - } Ok(value.clone()) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(expand_config_templates, module)?)?; + module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 4370cec3df..a1101192f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -364,6 +364,7 @@ impl<'py> Expansion<'_, 'py> { let host = PythonHost::new( self.py, TemplateSources(variables.clone(), environment, context), + label, ); let options = TemplateOptions { allow_context: true, @@ -373,9 +374,7 @@ impl<'py> Expansion<'_, 'py> { let result = expanded(&host, value, options); self.reads.extend(host.into_reads()); result.map_err(|stop| match stop { - Stop::Failure(failure) => { - template_error(&failure, label).map_or(Halt::Defer, Halt::Config) - } + Stop::Failure(error) => Halt::Config(template_error(&error, label)), Stop::Python(error) => Halt::Error(error), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index caefa71b05..acde1b8eab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -1,22 +1,172 @@ -//! Substitute static project variables and extract static SQL references. +//! Interpolate `@@` tokens in authored SQL and extract static SQL references. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use std::cell::RefCell; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; +use pyo3::{FromPyObject, PyErr, pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; +use sqlbuild_sqltext::compiler::models::InterpolationRead; +use sqlbuild_sqltext::compiler::types::InterpolationHost; use crate::bindings::_helpers::boundary::panics::compiler_guard; +const ENVIRONMENT_READ: &str = "env"; +const CONTEXT_READ: &str = "ctx"; + +/// `(sql or None when unchanged, spans, reads, error message or None)` for one SQL text. +type InterpolationRow = ( + Option, + Vec<(usize, usize, usize, usize)>, + Vec<(&'static str, String)>, + Option, +); + +/// The project variables, `os.environ`, the context values (or `None`) and the Python that +/// renders a non-string variable as text, raising `ValueError` when it cannot be text. +#[derive(FromPyObject)] +struct InterpolationSources<'py>( + Bound<'py, PyDict>, + Bound<'py, PyAny>, + Option>, + Bound<'py, PyAny>, +); + +struct PythonInterpolationHost<'py> { + sources: InterpolationSources<'py>, + /// A Python exception other than a rendering `ValueError`, raised after the scan. + raised: RefCell>, +} + +impl PythonInterpolationHost<'_> { + fn hold(&self, error: PyErr) -> String { + let message = error.to_string(); + self.raised.borrow_mut().get_or_insert(error); + message + } + + fn text(&self, value: &Bound<'_, PyAny>) -> Result { + value + .downcast::() + .map_err(|error| self.hold(error.into()))? + .to_str() + .map(str::to_owned) + .map_err(|error| self.hold(error)) + } +} + +impl InterpolationHost for PythonInterpolationHost<'_> { + fn variable(&self, name: &str) -> Option> { + let value = match self.sources.0.get_item(name) { + Ok(value) => value?, + Err(error) => return Some(Err(self.hold(error))), + }; + if value.is_exact_instance_of::() { + return Some(self.text(&value)); + } + let label = format!("SQL variable '@@{name}'"); + let py = value.py(); + Some(match self.sources.3.call1((value, label)) { + Ok(rendered) => self.text(&rendered), + Err(error) if error.is_instance_of::(py) => { + Err(error.value(py).to_string()) + } + Err(error) => Err(self.hold(error)), + }) + } + + fn variable_names(&self) -> Vec { + let mut names: Vec = self + .sources + .0 + .keys() + .iter() + .map(|key| key.str().map(|text| text.to_string()).unwrap_or_default()) + .collect(); + names.sort(); + names + } + + fn environment(&self, name: &str) -> Result, String> { + let environment = &self.sources.1; + if !environment + .contains(name) + .map_err(|error| self.hold(error))? + { + return Ok(None); + } + let value = environment + .get_item(name) + .map_err(|error| self.hold(error))?; + self.text(&value).map(Some) + } + + fn context_allowed(&self) -> bool { + self.sources.2.is_some() + } + + fn context(&self, name: &str) -> Option> { + let value = self.sources.2.as_ref()?.get_item(name).ok()??; + if value.is_none() { + return Some(None); + } + Some(self.text(&value).ok()) + } + + fn context_names(&self) -> Vec { + self.sources + .2 + .iter() + .flat_map(|context| context.keys()) + .map(|key| key.str().map(|text| text.to_string()).unwrap_or_default()) + .collect() + } +} + +/// Interpolate each `(sql, file path)`; an error is returned in its row, not raised. #[pyfunction] -fn substitute_static_project_vars( - sqls: Vec, - variables: Vec<(String, String)>, -) -> PyResult)>> { - compiler_guard(|| { - Ok( - sqlbuild_sqltext::compiler::main::sql_interpolation::substitute_batch( - &sqls, &variables, +fn interpolate_sql_batch<'py>( + sqls: Vec<(String, String)>, + sources: InterpolationSources<'py>, + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult> { + let python = python_text(python_version, unicode_version).ok_or_else(|| { + PyRuntimeError::new_err("SQL interpolation needs a supported Python (see D017)") + })?; + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let mut rows = Vec::with_capacity(sqls.len()); + for (sql, file_path) in &sqls { + let row = match interpolate_sql(python, &host, sql, file_path) { + Ok(result) => (result.sql, result.spans, read_rows(result.reads), None), + Err(failure) => ( + None, + Vec::new(), + read_rows(failure.reads), + Some(failure.message), ), - ) - }) + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + rows.push(row); + } + Ok(rows) +} + +fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { + reads + .into_iter() + .map(|read| match read { + InterpolationRead::Environment(name) => (ENVIRONMENT_READ, name), + InterpolationRead::Context(name) => (CONTEXT_READ, name), + }) + .collect() } #[pyfunction] @@ -31,7 +181,7 @@ fn extract_static_sql_references( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?; + module.add_function(wrap_pyfunction!(interpolate_sql_batch, module)?)?; module.add_function(wrap_pyfunction!(extract_static_sql_references, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index fadcd2af08..8abf85c405 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -1,99 +1,245 @@ -//! Conservative native fast path for compile-wide scalar SQL variables. +//! `@@name`, `@@ENV:NAME` and `@@CTX:name` interpolation of authored SQL. +use crate::compiler::models::{InterpolatedSql, InterpolationFailure, InterpolationRead}; +use crate::compiler::types::{CharSpan, InterpolationHost}; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; -use std::collections::HashMap; -use std::ops::Range; - -pub(crate) const UNCHANGED: u8 = 0; -pub(crate) const SUBSTITUTED: u8 = 1; -pub(crate) const FALLBACK: u8 = 2; -/// Python raises for an unknown variable, named in the result, unless only Python holds it. -pub(crate) const UNKNOWN_VARIABLE: u8 = 3; -pub(crate) const UNCLOSED_QUOTE: u8 = 4; -pub(crate) const UNCLOSED_BLOCK_COMMENT: u8 = 5; - -/// Why substitution stops before the end of the SQL. -enum Stop { - Fallback, - UnknownVariable(String), +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_alpha::is_python_alpha; +use sqlbuild_core::text::models::PythonText; + +const CONTEXT: &str = "SQL interpolation"; +const TOKEN: &str = "@@"; +const ENVIRONMENT_PREFIX: &str = "ENV:"; +const CONTEXT_PREFIX: &str = "CTX:"; + +/// One substitution in byte offsets: `(source start, source end, output start, output end)`. +type ByteSpan = (usize, usize, usize, usize); + +struct Interpolation<'run, H: InterpolationHost> { + python: PythonText, + host: &'run H, + file_path: &'run str, + output: String, + spans: Vec, + reads: Vec, } -impl Stop { - fn result(self) -> (u8, Option) { - match self { - Self::Fallback => (FALLBACK, None), - Self::UnknownVariable(name) => (UNKNOWN_VARIABLE, Some(name)), +/// Interpolate `sql` as Python's `substitute_sql_vars_with_spans` did, or return its error. +pub(crate) fn interpolate( + python: PythonText, + host: &H, + sql: &str, + file_path: &str, +) -> Result { + let mut run = Interpolation { + python, + host, + file_path, + output: String::new(), + spans: Vec::new(), + reads: Vec::new(), + }; + if !sql.contains(TOKEN) { + return Ok(InterpolatedSql { + sql: None, + spans: Vec::new(), + reads: Vec::new(), + }); + } + match run.scan(sql) { + Ok(()) => { + let spans = character_spans(sql, &run.output, &run.spans); + let changed = run.output != sql; + Ok(InterpolatedSql { + sql: changed.then_some(run.output), + spans, + reads: run.reads, + }) } + Err(message) => Err(InterpolationFailure { + message, + reads: run.reads, + }), } } -struct SubstitutionState { - output: Option, - copy_start: usize, -} +impl Interpolation<'_, H> { + fn scan(&mut self, sql: &str) -> Result<(), String> { + let bytes = sql.as_bytes(); + let mut index = 0; + while index < bytes.len() { + let text_end = match python_non_code_end(bytes, index) { + Ok(Some(end)) => end, + Ok(None) => index, + Err(Unclosed::BlockComment) => { + return Err(format!("{CONTEXT} contains an unclosed block comment")); + } + Err(Unclosed::Quote | Unclosed::Parenthesis) => { + return Err(format!("{CONTEXT} contains an unclosed quoted string")); + } + }; + if text_end > index { + let comment = + bytes[index..].starts_with(b"--") || bytes[index..].starts_with(b"/*"); + if comment { + self.output.push_str(&sql[index..text_end]); + } else { + self.quoted_text(sql, index, text_end)?; + } + index = text_end; + continue; + } + if bytes[index..].starts_with(TOKEN.as_bytes()) { + index = self.token(sql, index, bytes.len())?; + continue; + } + let character = sql[index..].chars().next().unwrap_or_default(); + self.output.push(character); + index += character.len_utf8(); + } + Ok(()) + } -pub(crate) fn substitute_batch( - sqls: &[String], - variables: &[(String, String)], -) -> Vec<(u8, Option)> { - let variables: HashMap<&str, &str> = variables - .iter() - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect(); - sqls.iter() - .map(|sql| substitute_one(sql, &variables)) - .collect() -} + /// Python's `_interpolate_sql_segment`: every token inside one quoted literal. + fn quoted_text(&mut self, sql: &str, start: usize, end: usize) -> Result<(), String> { + let mut index = start; + while let Some(offset) = sql[index..end].find(TOKEN) { + self.output.push_str(&sql[index..index + offset]); + index = self.token(sql, index + offset, end)?; + } + self.output.push_str(&sql[index..end]); + Ok(()) + } -fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option) { - if !sql.contains("@@") { - return (UNCHANGED, None); + /// Render the token at `start`, which may not read past `limit`; return where it ends. + fn token(&mut self, sql: &str, start: usize, limit: usize) -> Result { + let text = &sql[..limit]; + let output_start = self.output.len(); + let end = self.rendered_token(text, start)?; + self.spans + .push((start, end, output_start, self.output.len())); + Ok(end) } - let bytes = sql.as_bytes(); - let mut state = SubstitutionState { - output: None, - copy_start: 0, - }; - let mut index = 0; - while index < bytes.len() { - let text_end = match python_non_code_end(bytes, index) { - Ok(Some(end)) => end, - Ok(None) => index, - Err(Unclosed::BlockComment) => return (UNCLOSED_BLOCK_COMMENT, None), - Err(Unclosed::Quote) => return (UNCLOSED_QUOTE, None), - Err(Unclosed::Parenthesis) => return (FALLBACK, None), - }; - if text_end > index { - let comment = bytes[index..].starts_with(b"--") || bytes[index..].starts_with(b"/*"); - if !comment { - match substitute_quoted_text(sql, index..text_end, variables, state) { - Ok(next_state) => state = next_state, - Err(stop) => return stop.result(), - } + fn rendered_token(&mut self, sql: &str, start: usize) -> Result { + let file_path = self.file_path; + if sql[start..].starts_with("@@@") { + let name_start = start + 3; + let end = if self.starts_identifier(sql, name_start) { + self.identifier_end(sql, name_start) + } else { + name_start + }; + self.output.push_str(&sql[start..end]); + return Ok(end); + } + let token_start = start + TOKEN.len(); + if sql[token_start..].starts_with(ENVIRONMENT_PREFIX) { + let name_start = token_start + ENVIRONMENT_PREFIX.len(); + let name_end = self.name_end(sql, name_start, &['_']); + if name_end == name_start { + return Err(format!( + "invalid environment interpolation token in '{file_path}'" + )); } - index = text_end; - continue; + let name = &sql[name_start..name_end]; + self.reads + .push(InterpolationRead::Environment(name.to_owned())); + let value = self.host.environment(name)?.ok_or_else(|| { + format!("unknown environment variable '@@ENV:{name}' in '{file_path}'") + })?; + self.output.push_str(&value); + return Ok(name_end); } - if bytes[index..].starts_with(b"@@") { - match substitute_token(sql, index, variables, state) { - Ok((end, next_state)) => { - state = next_state; - index = end; - } - Err(stop) => return stop.result(), + if sql[token_start..].starts_with(CONTEXT_PREFIX) { + let name_start = token_start + CONTEXT_PREFIX.len(); + let greedy_end = self.name_end(sql, name_start, &['_', '.']); + if greedy_end == name_start { + return Err(format!("invalid CTX interpolation token in '{file_path}'")); + } + if !self.host.context_allowed() { + return Err(format!( + "SQL text in '{file_path}' does not allow @@CTX templates" + )); } - continue; + let name_end = self.known_context_end(sql, name_start, greedy_end); + let name = &sql[name_start..name_end]; + self.reads.push(InterpolationRead::Context(name.to_owned())); + let value = self.host.context(name).ok_or_else(|| { + format!("SQL text in '{file_path}' references unknown CTX key '{name}'") + })?; + let value = value.ok_or_else(|| { + format!( + "SQL text in '{file_path}' references CTX key '{name}' but no value is \ + available" + ) + })?; + self.output.push_str(&value); + return Ok(name_end); + } + if self.starts_identifier(sql, token_start) { + let name_end = self.identifier_end(sql, token_start); + let name = &sql[token_start..name_end]; + let value = self.host.variable(name).ok_or_else(|| { + let names = self.host.variable_names(); + let available = if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + }; + format!( + "unknown project variable '@@{name}' in '{file_path}'. Available vars: \ + {available}" + ) + })??; + self.output.push_str(&value); + return Ok(name_end); } - index += 1; + self.output.push_str(TOKEN); + Ok(token_start) } - match state.output { - Some(mut rendered) => { - rendered.push_str(&sql[state.copy_start..]); - (SUBSTITUTED, Some(rendered)) + + fn starts_identifier(&self, sql: &str, at: usize) -> bool { + sql[at..] + .chars() + .next() + .is_some_and(|character| character == '_' || is_python_alpha(self.python, character)) + } + + /// The end of the identifier starting at `start`, whose first character already matched. + fn identifier_end(&self, sql: &str, start: usize) -> usize { + let first = sql[start..].chars().next().map_or(0, char::len_utf8); + self.name_end(sql, start + first, &['_']) + } + + /// The end of the run of Python `str.isalnum()` characters and `extra` from `start`. + fn name_end(&self, sql: &str, start: usize, extra: &[char]) -> usize { + sql[start..] + .char_indices() + .find(|(_, character)| { + !(extra.contains(character) || is_python_alnum(self.python, *character)) + }) + .map_or(sql.len(), |(offset, _)| start + offset) + } + + /// Python's `_known_context_name_end`: the longest known name before a `.` in the greedy one. + fn known_context_end(&self, sql: &str, start: usize, greedy_end: usize) -> usize { + let greedy = &sql[start..greedy_end]; + if self.host.context(greedy).is_some() { + return greedy_end; } - None => (UNCHANGED, None), + self.host + .context_names() + .into_iter() + .filter(|name| { + greedy + .strip_prefix(name.as_str()) + .is_some_and(|rest| rest.starts_with('.')) + }) + .map(|name| name.len()) + .max() + .map_or(greedy_end, |length| start + length) } } @@ -109,85 +255,33 @@ fn python_non_code_end(bytes: &[u8], index: usize) -> Result, Uncl .ok_or(Unclosed::Quote) } -/// Python's `_interpolate_sql_segment`: substitute every token inside one quoted literal. -fn substitute_quoted_text( - sql: &str, - text: Range, - variables: &HashMap<&str, &str>, - mut state: SubstitutionState, -) -> Result { - let mut index = text.start; - while let Some(offset) = sql[index..text.end].find("@@") { - let (token_end, next_state) = substitute_token(sql, index + offset, variables, state)?; - if token_end > text.end { - return Err(Stop::Fallback); - } - state = next_state; - index = token_end; - } - Ok(state) -} - -fn substitute_token( - sql: &str, - start: usize, - variables: &HashMap<&str, &str>, - mut state: SubstitutionState, -) -> Result<(usize, SubstitutionState), Stop> { - let bytes = sql.as_bytes(); - if bytes[start..].starts_with(b"@@@") { - let name_start = start + 3; - let end = if bytes - .get(name_start) - .copied() - .is_some_and(is_identifier_start) - { - consume_ascii_identifier(bytes, name_start).ok_or(Stop::Fallback)? - } else { - name_start - }; - return Ok((end, state)); - } - let name_start = start + 2; - if bytes[name_start..].starts_with(b"ENV:") || bytes[name_start..].starts_with(b"CTX:") { - return Err(Stop::Fallback); - } - if !bytes - .get(name_start) - .copied() - .is_some_and(is_identifier_start) - { - return Err(Stop::Fallback); - } - let name_end = consume_ascii_identifier(bytes, name_start).ok_or(Stop::Fallback)?; - let name = &sql[name_start..name_end]; - let value = variables - .get(name) - .ok_or_else(|| Stop::UnknownVariable(name.to_owned()))?; - let rendered = state - .output - .get_or_insert_with(|| String::with_capacity(sql.len())); - rendered.push_str(&sql[state.copy_start..start]); - rendered.push_str(value); - state.copy_start = name_end; - Ok((name_end, state)) +/// Convert byte spans to the code-point spans Python's `ExpansionSpan` holds. +fn character_spans(source: &str, output: &str, spans: &[ByteSpan]) -> Vec { + let source_chars = char_offsets(source); + let output_chars = char_offsets(output); + spans + .iter() + .map(|(source_start, source_end, output_start, output_end)| { + ( + source_chars[*source_start], + source_chars[*source_end], + output_chars[*output_start], + output_chars[*output_end], + ) + }) + .collect() } -fn consume_ascii_identifier(bytes: &[u8], start: usize) -> Option { - let mut end = start + 1; - while bytes.get(end).copied().is_some_and(is_identifier_continue) { - end += 1; - } - if bytes.get(end).is_some_and(|value| !value.is_ascii()) { - return None; +/// The code-point offset of every byte boundary of `text`, indexed by byte offset. +fn char_offsets(text: &str) -> Vec { + let mut offsets = vec![0; text.len() + 1]; + let mut count = 0; + for (byte, character) in text.char_indices() { + for slot in &mut offsets[byte..byte + character.len_utf8()] { + *slot = count; + } + count += 1; } - Some(end) -} - -fn is_identifier_start(value: u8) -> bool { - value.is_ascii_alphabetic() || value == b'_' -} - -fn is_identifier_continue(value: u8) -> bool { - value.is_ascii_alphanumeric() || value == b'_' + offsets[text.len()] = count; + offsets } diff --git a/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs index 6ce7e114ff..c27d2040a4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs @@ -1,8 +1,17 @@ -//! Public compiler entry point for conservative scalar SQL interpolation. +//! Public compiler entry point for `@@` SQL interpolation. -pub fn substitute_batch( - sqls: &[String], - variables: &[(String, String)], -) -> Vec<(u8, Option)> { - crate::compiler::_helpers::sql_interpolation::substitution::substitute_batch(sqls, variables) +use crate::compiler::models::{InterpolatedSql, InterpolationFailure}; +use crate::compiler::types::InterpolationHost; +use sqlbuild_core::text::models::PythonText; + +/// Interpolate `@@` tokens in `sql`, reading variables, environment and context from `host`. +pub fn interpolate_sql( + python: PythonText, + host: &H, + sql: &str, + file_path: &str, +) -> Result { + crate::compiler::_helpers::sql_interpolation::substitution::interpolate( + python, host, sql, file_path, + ) } diff --git a/crates/sqlbuild-sqltext/src/compiler/models.rs b/crates/sqlbuild-sqltext/src/compiler/models.rs index 3780be29c5..0ebf1118fd 100644 --- a/crates/sqlbuild-sqltext/src/compiler/models.rs +++ b/crates/sqlbuild-sqltext/src/compiler/models.rs @@ -69,3 +69,25 @@ pub struct NestingFailure { pub message: String, pub help: String, } + +/// One environment or context read made by SQL interpolation, in order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InterpolationRead { + Environment(String), + Context(String), +} + +/// Interpolated SQL (`None` when unchanged), its code-point substitution spans and its reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InterpolatedSql { + pub sql: Option, + pub spans: Vec, + pub reads: Vec, +} + +/// The error interpolation stopped at, with the reads made before it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InterpolationFailure { + pub message: String, + pub reads: Vec, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index 9a5b9624f4..c833ee1d5e 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -1,95 +1,20 @@ use crate::compiler::_helpers::model_headers::tokenization::{ MAX_TOKENIZER_WORKERS, TOKENIZER_WORKER_STACK_BYTES, build_tokenizer_pool, parse_batch, }; -use crate::compiler::_helpers::sql_interpolation::substitution::{ - FALLBACK, SUBSTITUTED, UNCHANGED, UNCLOSED_BLOCK_COMMENT, UNCLOSED_QUOTE, UNKNOWN_VARIABLE, - substitute_batch, -}; use crate::compiler::_helpers::sql_references::extraction::extract; use crate::compiler::main::declaration_references::scan_declaration_references; use crate::compiler::main::model_header_single_parsing::parse_one; +use crate::compiler::main::sql_interpolation::interpolate_sql; use crate::compiler::models::{ AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, + InterpolationRead, }; use crate::compiler::tests::test_types::HeaderNestingTestCase; +use crate::compiler::types::{CharSpan, InterpolationHost}; +use sqlbuild_core::text::main::python_text::python_text; +use std::collections::BTreeMap; use std::thread; -pub(crate) fn scalar_variables_preserve_lexical_boundaries() -> bool { - let sqls = vec![ - "SELECT @@revision, '@@status', @@@window_start".to_owned(), - "-- @@revision\nSELECT /* @@status */ 1".to_owned(), - "SELECT '@@revision''s'".to_owned(), - ]; - substitute_batch( - &sqls, - &[ - ("revision".to_owned(), "7".to_owned()), - ("status".to_owned(), "ready".to_owned()), - ], - ) == vec![ - ( - SUBSTITUTED, - Some("SELECT 7, 'ready', @@@window_start".to_owned()), - ), - (UNCHANGED, None), - (SUBSTITUTED, Some("SELECT '7''s'".to_owned())), - ] -} - -pub(crate) fn dynamic_or_malformed_sql_requests_fallback() -> bool { - let sqls = vec![ - "SELECT @@ENV:USER, @@missing".to_owned(), - "SELECT @@missing, @@ENV:USER".to_owned(), - "SELECT '@@missing'".to_owned(), - "SELECT @@revision, 'unterminated".to_owned(), - "SELECT @@revision /* unterminated".to_owned(), - "SELECT @@révision".to_owned(), - ]; - substitute_batch(&sqls, &[("revision".to_owned(), "7".to_owned())]) - == vec![ - (FALLBACK, None), - (UNKNOWN_VARIABLE, Some("missing".to_owned())), - (UNKNOWN_VARIABLE, Some("missing".to_owned())), - (UNCLOSED_QUOTE, None), - (UNCLOSED_BLOCK_COMMENT, None), - (FALLBACK, None), - ] -} - -pub(crate) fn dollar_quoted_text_is_quoted_for_substitution() -> bool { - let sqls = vec![ - "SELECT $$--@@region$$ AS x".to_owned(), - "SELECT $$ /* $$ AS a, '@@region' AS b -- */".to_owned(), - "SELECT $tag$ it's $$ -- @@region $tag$, @@region".to_owned(), - "SELECT price$1$ -- @@region\n, $1 /* @@region */".to_owned(), - "SELECT a$$b$$ -- @@region".to_owned(), - ]; - substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())]) - == vec![ - (SUBSTITUTED, Some("SELECT $$--north$$ AS x".to_owned())), - ( - SUBSTITUTED, - Some("SELECT $$ /* $$ AS a, 'north' AS b -- */".to_owned()), - ), - ( - SUBSTITUTED, - Some("SELECT $tag$ it's $$ -- north $tag$, north".to_owned()), - ), - (UNCHANGED, None), - (UNCHANGED, None), - ] -} - -pub(crate) fn unclosed_dollar_quote_stops_as_unclosed_quote() -> bool { - let sqls = vec![ - "SELECT $tag$ @@region".to_owned(), - "SELECT @@region, $$ open".to_owned(), - "-- @@region\nSELECT $t$ closes with another tag $u$".to_owned(), - ]; - substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())]) - == vec![(UNCLOSED_QUOTE, None); sqls.len()] -} - pub(crate) fn simple_references_preserve_authored_order() -> bool { extract( "SELECT * FROM __source(\"orders\") UNION ALL SELECT * FROM __dbt_ref(\"shop\" , \"customers\")", @@ -337,19 +262,87 @@ pub(crate) fn nesting_error_at(position: usize) -> String { format!("values nest deeper than 256 levels at position {position}") } -pub(crate) fn doubled_backticks_close_one_segment_and_open_the_next() -> bool { - let sqls = vec![ - "SELECT `@@zz``".to_owned(), - "SELECT `@@region``@@region`, '@@region''s'".to_owned(), - "SELECT `a``b` -- @@region".to_owned(), - ]; - substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())]) - == vec![ - (UNKNOWN_VARIABLE, Some("zz".to_owned())), - ( - SUBSTITUTED, - Some("SELECT `north``north`, 'north''s'".to_owned()), - ), - (UNCHANGED, None), - ] +/// Variables, environment and context for interpolation tests. +pub(crate) struct MapHost { + pub(crate) variables: BTreeMap<&'static str, Result<&'static str, &'static str>>, + pub(crate) environment: BTreeMap<&'static str, &'static str>, + pub(crate) context: Option>>, +} + +impl InterpolationHost for MapHost { + fn variable(&self, name: &str) -> Option> { + self.variables + .get(name) + .map(|value| value.map(str::to_owned).map_err(str::to_owned)) + } + + fn variable_names(&self) -> Vec { + self.variables + .keys() + .map(|name| (*name).to_owned()) + .collect() + } + + fn environment(&self, name: &str) -> Result, String> { + Ok(self.environment.get(name).map(|value| (*value).to_owned())) + } + + fn context_allowed(&self) -> bool { + self.context.is_some() + } + + fn context(&self, name: &str) -> Option> { + self.context + .as_ref()? + .get(name) + .map(|value| value.map(str::to_owned)) + } + + fn context_names(&self) -> Vec { + self.context + .iter() + .flat_map(BTreeMap::keys) + .map(|name| (*name).to_owned()) + .collect() + } +} + +/// A host with `region = north`, `revision = 7`, `REGION=eu` and context `this`, `this.schema`. +pub(crate) fn interpolation_host(context: bool) -> MapHost { + MapHost { + variables: BTreeMap::from([ + ("region", Ok("north")), + ("revision", Ok("7")), + ("größe", Ok("large")), + ("layout", Err("SQL variable '@@layout' is an object")), + ]), + environment: BTreeMap::from([("REGION", "eu"), ("ÜBER", "yes")]), + context: context.then(|| { + BTreeMap::from([ + ("this", Some("orders")), + ("this.schema", Some("sales")), + ("run_id", None), + ]) + }), + } +} + +/// Interpolate `sql` as `models/orders.sql`: the rendered SQL or the error message. +pub(crate) fn interpolated(host: &MapHost, sql: &str) -> Result { + let python = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); + interpolate_sql(python, host, sql, "models/orders.sql") + .map(|result| result.sql.unwrap_or_else(|| sql.to_owned())) + .map_err(|failure| failure.message) +} + +/// The spans and reads of interpolating `sql`. +pub(crate) fn interpolation_facts( + host: &MapHost, + sql: &str, +) -> (Vec, Vec) { + let python = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); + match interpolate_sql(python, host, sql, "models/orders.sql") { + Ok(result) => (result.spans, result.reads), + Err(failure) => (Vec::new(), failure.reads), + } } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index 362e82c422..2461676343 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -1,47 +1,152 @@ -use crate::compiler::tests::helpers::{ - dollar_quoted_text_is_quoted_for_substitution, - doubled_backticks_close_one_segment_and_open_the_next, - dynamic_or_malformed_sql_requests_fallback, scalar_variables_preserve_lexical_boundaries, - unclosed_dollar_quote_stops_as_unclosed_quote, -}; -use crate::compiler::tests::test_types::StaticSqlOperationTestCase; +use crate::compiler::models::InterpolationRead; +use crate::compiler::tests::helpers::{interpolated, interpolation_facts, interpolation_host}; +use crate::compiler::tests::test_types::InterpolationTestCase; #[test] -fn given_sql_interpolation_cases_when_substituting_then_expected_behavior_holds() { +fn given_sql_when_interpolating_then_text_and_errors_match_python() { + let unknown_variable = "unknown project variable '@@missing' in 'models/orders.sql'. \ + Available vars: größe, layout, region, revision"; let test_cases = [ - StaticSqlOperationTestCase { - description: "doubled backticks close one quoted segment and open the next", - run: doubled_backticks_close_one_segment_and_open_the_next, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "scalar variables preserve lexical boundaries", - run: scalar_variables_preserve_lexical_boundaries, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "dynamic SQL falls back; unknown names and unclosed text stop", - run: dynamic_or_malformed_sql_requests_fallback, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "dollar-quoted text is quoted text, not comments or code", - run: dollar_quoted_text_is_quoted_for_substitution, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "unclosed dollar quotes stop as unclosed quoted text", - run: unclosed_dollar_quote_stops_as_unclosed_quote, - expected_success: true, + InterpolationTestCase { + description: "variables in code and quotes; comments and @@@ stay", + sql: "SELECT @@revision, '@@region' -- @@region\n/* @@region */, @@@window_start, @@1", + context: false, + expected: Ok("SELECT 7, 'north' -- @@region\n/* @@region */, @@@window_start, @@1"), + }, + InterpolationTestCase { + description: "doubled quotes and backticks", + sql: "SELECT '@@revision''s', `@@region``@@region`", + context: false, + expected: Ok("SELECT '7''s', `north``north`"), + }, + InterpolationTestCase { + description: "dollar-quoted text is quoted text", + sql: "SELECT $tag$ it's $$ -- @@region $tag$, @@region, price$1$ -- @@region", + context: false, + expected: Ok("SELECT $tag$ it's $$ -- north $tag$, north, price$1$ -- @@region"), + }, + InterpolationTestCase { + description: "environment variables, including non-ASCII names", + sql: "SELECT '@@ENV:REGION', @@ENV:ÜBER", + context: false, + expected: Ok("SELECT 'eu', yes"), + }, + InterpolationTestCase { + description: "non-ASCII variable names", + sql: "SELECT @@größe", + context: false, + expected: Ok("SELECT large"), + }, + InterpolationTestCase { + description: "context keys take the longest known dotted prefix", + sql: "GRANT SELECT ON @@CTX:this.schema.@@CTX:this.extra TO reporting", + context: true, + expected: Ok("GRANT SELECT ON sales.orders.extra TO reporting"), + }, + InterpolationTestCase { + description: "an unknown variable lists the available ones", + sql: "SELECT '@@missing'", + context: false, + expected: Err(unknown_variable), + }, + InterpolationTestCase { + description: "a structured variable reports its rendering error", + sql: "SELECT @@layout", + context: false, + expected: Err("SQL variable '@@layout' is an object"), + }, + InterpolationTestCase { + description: "a missing environment variable", + sql: "SELECT @@ENV:SALES_REGION", + context: false, + expected: Err( + "unknown environment variable '@@ENV:SALES_REGION' in 'models/orders.sql'", + ), + }, + InterpolationTestCase { + description: "an empty environment name", + sql: "SELECT @@ENV:-1", + context: false, + expected: Err("invalid environment interpolation token in 'models/orders.sql'"), + }, + InterpolationTestCase { + description: "context outside hooks", + sql: "SELECT @@CTX:this", + context: false, + expected: Err("SQL text in 'models/orders.sql' does not allow @@CTX templates"), + }, + InterpolationTestCase { + description: "an empty context name", + sql: "SELECT @@CTX:", + context: true, + expected: Err("invalid CTX interpolation token in 'models/orders.sql'"), + }, + InterpolationTestCase { + description: "an unknown context key", + sql: "SELECT @@CTX:that", + context: true, + expected: Err("SQL text in 'models/orders.sql' references unknown CTX key 'that'"), + }, + InterpolationTestCase { + description: "a context key without a value", + sql: "SELECT @@CTX:run_id", + context: true, + expected: Err( + "SQL text in 'models/orders.sql' references CTX key 'run_id' but no value is \ + available", + ), + }, + InterpolationTestCase { + description: "an unclosed quote", + sql: "SELECT @@revision, 'open", + context: false, + expected: Err("SQL interpolation contains an unclosed quoted string"), + }, + InterpolationTestCase { + description: "an unclosed dollar quote", + sql: "SELECT @@region, $$ open", + context: false, + expected: Err("SQL interpolation contains an unclosed quoted string"), + }, + InterpolationTestCase { + description: "an unclosed block comment", + sql: "SELECT @@revision /* open", + context: false, + expected: Err("SQL interpolation contains an unclosed block comment"), }, ]; - for test_case in test_cases { - let actual_success = (test_case.run)(); + let host = interpolation_host(test_case.context); assert_eq!( - actual_success, test_case.expected_success, + interpolated(&host, test_case.sql), + test_case.expected.map(str::to_owned).map_err(str::to_owned), "{}", test_case.description ); } } + +#[test] +fn given_tokens_when_interpolating_then_spans_count_code_points_and_reads_keep_order() { + let host = interpolation_host(true); + + let (spans, reads) = + interpolation_facts(&host, "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@"); + + assert_eq!( + spans, + vec![ + (12, 19, 12, 17), + (21, 33, 19, 21), + (35, 45, 23, 29), + (46, 48, 30, 32) + ] + ); + assert_eq!( + reads, + vec![ + InterpolationRead::Environment("REGION".to_owned()), + InterpolationRead::Context("this".to_owned()), + ] + ); +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index ecff676596..5e0b67cfa4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -44,3 +44,10 @@ pub(crate) struct HeaderNestingLocationTestCase { pub(crate) header_line: usize, pub(crate) expected_failure: Option, } + +pub(crate) struct InterpolationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) context: bool, + pub(crate) expected: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index f5f935751a..545ba2ca48 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1 +1,20 @@ pub type StaticReference = (String, String, Option, Option); + +/// One substitution in code points: `(source start, source end, output start, output end)`. +pub type CharSpan = (usize, usize, usize, usize); + +/// The project variables, process environment and context values one interpolation reads. +pub trait InterpolationHost { + /// The text of project variable `name`, `Err(message)` when it cannot be text, or `None`. + fn variable(&self, name: &str) -> Option>; + /// Every project variable name, sorted. + fn variable_names(&self) -> Vec; + /// The value of environment variable `name`, or `Err(message)` when it cannot be read. + fn environment(&self, name: &str) -> Result, String>; + /// Whether this SQL may use `@@CTX:` tokens. + fn context_allowed(&self) -> bool; + /// `Some(value)` for a known context key, whose value may be unavailable (`None`). + fn context(&self, name: &str) -> Option>; + /// Every known context key. + fn context_names(&self) -> Vec; +} diff --git a/scripts/generate_python_char_table.py b/scripts/generate_python_char_table.py index 1aa907d86c..0118088f34 100644 --- a/scripts/generate_python_char_table.py +++ b/scripts/generate_python_char_table.py @@ -1,4 +1,4 @@ -"""Direct wrapper for generating the native `str.isalnum()` table of the running Python.""" +"""Direct wrapper for generating the native `str` character tables of the running Python.""" from __future__ import annotations @@ -6,7 +6,7 @@ def main() -> int: - """Regenerate the native Python character table.""" + """Regenerate the native Python character tables.""" return generate_python_char_table() diff --git a/scripts/python_char_table/_helpers/table.py b/scripts/python_char_table/_helpers/table.py index 89d9822654..43802eda6b 100644 --- a/scripts/python_char_table/_helpers/table.py +++ b/scripts/python_char_table/_helpers/table.py @@ -1,4 +1,4 @@ -"""Range computation and Rust rendering of Python's `str.isalnum()` table.""" +"""Range computation and Rust rendering of Python's `str.isalnum()` and `str.isalpha()` tables.""" from __future__ import annotations @@ -7,29 +7,30 @@ from scripts.python_char_table.constants import ( MAX_CODE_POINT, - TABLE_DIRECTORY, TABLE_FILE_PREFIX, TABLE_FILE_SUFFIX, + TABLE_ROOT, ) -def table_path() -> Path: - """Return the Rust file holding the table of the running Python's Unicode version.""" +def table_path(*, method: str) -> Path: + """Return the Rust file holding one method's table for the running Python's Unicode version.""" version: str = unicodedata.unidata_version.replace(".", "_") - return TABLE_DIRECTORY / f"{TABLE_FILE_PREFIX}{version}{TABLE_FILE_SUFFIX}" + directory: Path = TABLE_ROOT / f"{method.removeprefix('is')}_ranges" + return directory / f"{TABLE_FILE_PREFIX}{version}{TABLE_FILE_SUFFIX}" -def alnum_ranges() -> list[tuple[int, int]]: - """Return the inclusive code point ranges where `str.isalnum()` is true.""" +def character_ranges(*, method: str) -> list[tuple[int, int]]: + """Return the inclusive code point ranges where the `str` method is true.""" ranges: list[tuple[int, int]] = [] start: int | None = None for code_point in range(MAX_CODE_POINT + 1): - alnum: bool = chr(code_point).isalnum() - if alnum and start is None: + matched: bool = getattr(chr(code_point), method)() + if matched and start is None: start = code_point - elif not alnum and start is not None: + elif not matched and start is not None: ranges.append((start, code_point - 1)) start = None if start is not None: @@ -37,12 +38,12 @@ def alnum_ranges() -> list[tuple[int, int]]: return ranges -def render_table(ranges: list[tuple[int, int]]) -> str: - """Render the Rust expression of the range table, which `alnum_tables.rs` includes.""" +def render_table(*, method: str, ranges: list[tuple[int, int]]) -> str: + """Render the Rust expression of one range table, which `alnum_tables.rs` includes.""" lines: list[str] = [ "// Generated by `scripts/generate_python_char_table.py`; do not edit.", - f"// Inclusive code point ranges where `str.isalnum()` is true in Unicode " + f"// Inclusive code point ranges where `str.{method}()` is true in Unicode " f"{unicodedata.unidata_version}, in order.", "&[", ] diff --git a/scripts/python_char_table/constants.py b/scripts/python_char_table/constants.py index 2b55fcb5f9..a378a3dbb6 100644 --- a/scripts/python_char_table/constants.py +++ b/scripts/python_char_table/constants.py @@ -2,14 +2,8 @@ from pathlib import Path -TABLE_DIRECTORY: Path = ( - Path(__file__).resolve().parents[2] - / "crates" - / "sqlbuild-core" - / "src" - / "text" - / "alnum_ranges" -) +TABLE_ROOT: Path = Path(__file__).resolve().parents[2] / "crates" / "sqlbuild-core" / "src" / "text" TABLE_FILE_PREFIX: str = "unicode_" TABLE_FILE_SUFFIX: str = ".in" MAX_CODE_POINT: int = 0x10FFFF +TABLE_METHODS: tuple[str, ...] = ("isalnum", "isalpha") diff --git a/scripts/python_char_table/main/generate.py b/scripts/python_char_table/main/generate.py index 47877b4291..41a0e34a00 100644 --- a/scripts/python_char_table/main/generate.py +++ b/scripts/python_char_table/main/generate.py @@ -1,4 +1,4 @@ -"""Focused entry for generating the native `str.isalnum()` table of the running Python.""" +"""Focused entry for generating the native `str` character tables of the running Python.""" from __future__ import annotations @@ -6,13 +6,18 @@ import unicodedata from pathlib import Path -from scripts.python_char_table._helpers.table import alnum_ranges, render_table, table_path +from scripts.python_char_table._helpers.table import character_ranges, render_table, table_path +from scripts.python_char_table.constants import TABLE_METHODS def generate_python_char_table() -> int: - """Write the table of this Python's Unicode version next to the native text helpers.""" + """Write this Python's Unicode version tables next to the native text helpers.""" - path: Path = table_path() - _ = path.write_text(render_table(alnum_ranges()), encoding="utf-8") - print(f"wrote {path} (Unicode {unicodedata.unidata_version})", file=sys.stderr) + for method in TABLE_METHODS: + path: Path = table_path(method=method) + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text( + render_table(method=method, ranges=character_ranges(method=method)), encoding="utf-8" + ) + print(f"wrote {path} (Unicode {unicodedata.unidata_version})", file=sys.stderr) return 0 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5d5f96029c..cbeecee66f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,6 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Any, TypedDict from sqlbuild.compiler.sql_test_glue.models import ( @@ -301,9 +301,19 @@ def render_attached_generic_audit( def scan_sql_declaration_references( sqls: list[str], ) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None] | None]: ... -def substitute_static_project_vars( - sqls: list[str], variables: list[tuple[str, str]] -) -> list[tuple[int, str | None]]: ... +def interpolate_sql_batch( + sqls: list[tuple[str, str]], + sources: tuple[ + Mapping[str, object], + Mapping[str, str], + Mapping[str, str | None] | None, + Callable[[object, str], str], + ], + python_version: tuple[int, int], + unicode_version: str, +) -> list[ + tuple[str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None] +]: ... def extract_static_sql_references( sql: str, ) -> list[tuple[str, str, str | None, int | None]] | None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 523b117c29..5465555df0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -81,9 +81,9 @@ expand_sql_macros_result, ) from sqlbuild.compiler.compile._helpers.render.sql_vars import ( + applied_interpolation, expand_authored_sql_result, - prepare_static_project_vars_batch, - substitute_sql_vars, + interpolate_sql_batch, ) from sqlbuild.compiler.compile._helpers.render.templating import ( contains_template_data, @@ -121,6 +121,7 @@ NativeModelConfigInputs, NativeModelConfigSession, SqlAnalysisOptOutRequest, + SqlInterpolation, SqlReferenceOrigin, SqlReferenceScan, SqlReferenceSourceMap, @@ -537,11 +538,13 @@ def _build_model_inputs( discovered_inputs.sql_hook_files ) render_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - prepared_var_substituted_sqls: dict[Path, str | None] = dict( + interpolations: dict[Path, SqlInterpolation] = dict( zip( (model_file.file_path for model_file in render_files), - prepare_static_project_vars_batch( - sqls=tuple(model_file.query_sql for model_file in render_files), + interpolate_sql_batch( + sqls=tuple( + (model_file.query_sql, model_file.file_path) for model_file in render_files + ), effective_vars=effective_vars, ), strict=True, @@ -551,7 +554,7 @@ def _build_model_inputs( tuple( model_file for model_file in render_files - if prepared_var_substituted_sqls[model_file.file_path] is not None + if interpolations[model_file.file_path].error is None ) if native_stage_enabled(NativeStage.MODEL_LOOP) else () @@ -581,8 +584,7 @@ def _build_model_inputs( (model_file.file_path for model_file in prepared_files), scan_native_declaration_references( sqls=tuple( - cast(str, prepared_var_substituted_sqls[model_file.file_path]) - for model_file in prepared_files + interpolations[model_file.file_path].sql for model_file in prepared_files ) ), strict=True, @@ -596,9 +598,7 @@ def _build_model_inputs( _build_model_input( loop=loop, model_file=model_file, - prepared_var_substituted_sql=prepared_var_substituted_sqls.get( - model_file.file_path - ), + interpolation=interpolations[model_file.file_path], ) ) @@ -613,7 +613,7 @@ def _build_model_input( *, loop: _ModelInputLoop, model_file: DiscoveredSqlModelFile, - prepared_var_substituted_sql: str | None, + interpolation: SqlInterpolation, ) -> CompileModelInput: discovered_inputs: DiscoveredProjectInputs = loop.discovered_inputs context: ModelInputBuildContext = loop.context @@ -717,15 +717,7 @@ def _build_model_input( consumer_path=model_file.relative_path, ), ) - var_substituted_sql: str = ( - prepared_var_substituted_sql - if prepared_var_substituted_sql is not None - else substitute_sql_vars( - sql=model_file.query_sql, - file_path=model_file.file_path, - effective_vars=effective_vars, - ) - ) + var_substituted_sql: str = applied_interpolation(interpolation) declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( enums=declarations.enums, constants=declarations.constants, @@ -747,17 +739,14 @@ def _build_model_input( value_renderer=context.value_renderer, collection_rendering=context.collection_rendering, ) - if prepared_var_substituted_sql is not None + if native_stage_enabled(NativeStage.MODEL_LOOP) else None ) if declaration_expansion is not None and native_stage_enabled(NativeStage.MODEL_LOOP): report_native_answer(stage=NativeStage.MODEL_LOOP, kind="declaration_expansions") if declaration_expansion is None: if native_stage_enabled(NativeStage.MODEL_LOOP): - report_native_fallback( - site=NativeFallbackSite.DECLARATION_REFERENCES, - kind="scan" if prepared_var_substituted_sql is not None else "after_variables", - ) + report_native_fallback(site=NativeFallbackSite.DECLARATION_REFERENCES, kind="scan") declaration_expansion = expand_declaration_references_result( sql=var_substituted_sql, file_path=model_file.file_path, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index e6e19e3b0d..5ee7deb01a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -3,12 +3,14 @@ from __future__ import annotations import os -import re +import sys +import unicodedata from collections.abc import Mapping from pathlib import Path import sqlbuild._native as _native from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text +from sqlbuild.compiler.compile._helpers.render.context_templates import record_template_reads from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_declaration_references_result, expand_declaration_references_with_spans, @@ -18,13 +20,6 @@ expand_sql_macros_result, expand_sql_macros_with_spans, ) -from sqlbuild.compiler.compile.constants import ( - COMPILE_INPUT_READS, - SQL_CONTEXT_NAME_EXTRA_TOKENS, - SQL_IDENTIFIER_EXTRA_TOKEN, - SQL_INTERPOLATION_TOKEN, - SQL_QUOTE_TOKENS, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -35,6 +30,7 @@ LoadedMacro, MacroContext, MacroExpansionResult, + SqlInterpolation, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled @@ -43,31 +39,8 @@ from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( scan_native_declaration_references, ) -from sqlbuild.compiler.sql_analysis.main._is_identifier_character import ( - is_identifier_character as _is_identifier_continue, -) -from sqlbuild.compiler.sql_analysis.main._is_identifier_start import ( - is_identifier_start as _is_identifier_start, -) -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text from sqlbuild.sql_values.types import CollectionRendering -_SQL_INTERPOLATION_SPECIAL: re.Pattern[str] = re.compile(r"['\"`$\-/@]") - -_CONTEXT: str = "SQL interpolation" -_NATIVE_UNCHANGED: int = 0 -_NATIVE_SUBSTITUTED: int = 1 -_NATIVE_FALLBACK: int = 2 -_NATIVE_UNKNOWN_VARIABLE: int = 3 -_NATIVE_UNCLOSED_QUOTE: int = 4 -_NATIVE_UNCLOSED_BLOCK_COMMENT: int = 5 -_NATIVE_RESULT_LENGTH: int = 2 -_NATIVE_ERROR_STATUSES: frozenset[int] = frozenset( - {_NATIVE_UNKNOWN_VARIABLE, _NATIVE_UNCLOSED_QUOTE, _NATIVE_UNCLOSED_BLOCK_COMMENT} -) - def expand_authored_sql_result( # noqa: PLR0913 *, @@ -85,22 +58,11 @@ def expand_authored_sql_result( # noqa: PLR0913 """Apply all expansion passes and retain facts emitted by those passes.""" native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) - prepared_sql: str | None = ( - prepare_static_project_vars_batch( - sqls=(sql,), effective_vars=effective_vars, error_file_path=file_path - )[0] - if native - else None - ) - interpolated_sql: str = ( - prepared_sql - if prepared_sql is not None - else substitute_sql_vars( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) + interpolated_sql: str = substitute_sql_vars( + sql=sql, + file_path=file_path, + effective_vars=effective_vars, + context_values=context_values, ) declaration_context: DeclarationResolutionContext = ( declarations or DeclarationResolutionContext() @@ -203,93 +165,12 @@ def substitute_sql_vars( ) -> str: """Replace @@name, @@ENV:NAME, and allowed @@CTX:name references in SQL text.""" - rendered_sql: str - rendered_sql, _spans = substitute_sql_vars_with_spans( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - return rendered_sql - - -def prepare_static_project_vars_batch( - *, - sqls: tuple[str, ...], - effective_vars: dict[str, object], - error_file_path: Path | None = None, -) -> tuple[str | None, ...]: - """Prepare static substitutions in order; with a file path, raise exact errors for it.""" - - scalar_variables: list[tuple[str, str]] = [ - (name, render_project_var_text(value=value, label=f"SQL variable '@@{name}'")) - for name, value in effective_vars.items() - if value is None or isinstance(value, str | int | float | bool) - ] - raw_results: object = _native.substitute_static_project_vars(list(sqls), scalar_variables) - if not isinstance(raw_results, list) or len(raw_results) != len(sqls): - raise CompileInputError("native static SQL interpolation returned an invalid batch") - results: list[str | None] = [] - counted: bool = native_stage_enabled(NativeStage.MODEL_LOOP) - for sql, raw_result in zip(sqls, raw_results, strict=True): - if not ( - isinstance(raw_result, tuple) - and len(raw_result) == _NATIVE_RESULT_LENGTH - and type(raw_result[0]) is int - and (raw_result[1] is None or isinstance(raw_result[1], str)) - ): - raise CompileInputError("native static SQL interpolation returned an invalid result") - status, rendered = raw_result - if status == _NATIVE_UNCHANGED and rendered is None: - results.append(sql) - elif status == _NATIVE_SUBSTITUTED and rendered is not None: - results.append(rendered) - elif status == _NATIVE_FALLBACK and rendered is None: - if counted: - report_native_fallback(site=NativeFallbackSite.SQL_VARIABLES) - results.append(None) - elif status in _NATIVE_ERROR_STATUSES: - if error_file_path is not None: - _raise_native_interpolation_error( - status=status, - name=rendered, - file_path=error_file_path, - effective_vars=effective_vars, - ) - if counted: - report_native_fallback(site=NativeFallbackSite.SQL_VARIABLES, kind="error") - results.append(None) - else: - raise CompileInputError("native static SQL interpolation returned an invalid status") - return tuple(results) - - -def _raise_native_interpolation_error( - *, status: int, name: str | None, file_path: Path, effective_vars: dict[str, object] -) -> None: - if status == _NATIVE_UNCLOSED_QUOTE: - raise CompileInputError( - f"{_CONTEXT} contains an unclosed quoted string", bridge_independent=True - ) - if status == _NATIVE_UNCLOSED_BLOCK_COMMENT: - raise CompileInputError( - f"{_CONTEXT} contains an unclosed block comment", bridge_independent=True - ) - if name is not None and name not in effective_vars: - raise CompileInputError( - _unknown_project_variable_message( - var_name=name, file_path=file_path, effective_vars=effective_vars - ), - bridge_independent=True, - ) - - -def _unknown_project_variable_message( - *, var_name: str, file_path: Path, effective_vars: dict[str, object] -) -> str: - return ( - f"unknown project variable '@@{var_name}' in '{file_path}'. " - f"Available vars: {', '.join(sorted(effective_vars)) or 'none'}" + return applied_interpolation( + interpolate_sql_batch( + sqls=((sql, file_path),), + effective_vars=effective_vars, + context_values=context_values, + )[0] ) @@ -302,251 +183,45 @@ def substitute_sql_vars_with_spans( ) -> tuple[str, tuple[ExpansionSpan, ...]]: """Replace SQL interpolation tokens, returning the span of every substitution.""" - if SQL_INTERPOLATION_TOKEN not in sql: - return sql, () - parts: list[str] = [] - spans: list[ExpansionSpan] = [] - output_length: int = 0 - cursor: int = 0 - while cursor < len(sql): - special: re.Match[str] | None = _SQL_INTERPOLATION_SPECIAL.search(sql, cursor) - if special is None: - parts.append(sql[cursor:]) - break - if special.start() > cursor: - plain: str = sql[cursor : special.start()] - parts.append(plain) - output_length += len(plain) - cursor = special.start() - character: str = sql[cursor] - if character in SQL_QUOTE_TOKENS: - end: int = skip_quoted_text(sql=sql, start=cursor, context=_CONTEXT) - segment_text: str - segment_spans: tuple[ExpansionSpan, ...] - segment_text, segment_spans = _interpolate_sql_segment( - segment=sql[cursor:end], - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - parts.append(segment_text) - segment_span: ExpansionSpan - for segment_span in segment_spans: - spans.append( - _rebased_span(span=segment_span, source_base=cursor, output_base=output_length) - ) - output_length += len(segment_text) - cursor = end - continue - if sql.startswith("--", cursor): - end = skip_line_comment(sql=sql, start=cursor) - parts.append(sql[cursor:end]) - output_length += end - cursor - cursor = end - continue - if sql.startswith("/*", cursor): - end = skip_block_comment(sql=sql, start=cursor, context=_CONTEXT) - parts.append(sql[cursor:end]) - output_length += end - cursor - cursor = end - continue - if sql.startswith("@@", cursor): - rendered_token: str - next_cursor: int - rendered_token, next_cursor = _render_interpolation_token( - sql=sql, - start=cursor, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - parts.append(rendered_token) - spans.append( - ExpansionSpan( - source_start=cursor, - source_end=next_cursor, - output_start=output_length, - output_end=output_length + len(rendered_token), - ) - ) - output_length += len(rendered_token) - cursor = next_cursor - continue - parts.append(character) - output_length += 1 - cursor += 1 - return "".join(parts), tuple(spans) - - -def _rebased_span(*, span: ExpansionSpan, source_base: int, output_base: int) -> ExpansionSpan: - return ExpansionSpan( - source_start=span.source_start + source_base, - source_end=span.source_end + source_base, - output_start=span.output_start + output_base, - output_end=span.output_end + output_base, - ) - - -def _interpolate_sql_segment( - *, - segment: str, - file_path: Path, - effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - if SQL_INTERPOLATION_TOKEN not in segment: - return segment, () - parts: list[str] = [] - spans: list[ExpansionSpan] = [] - output_length: int = 0 - cursor: int = 0 - while cursor < len(segment): - token_start: int = segment.find(SQL_INTERPOLATION_TOKEN, cursor) - if token_start < 0: - parts.append(segment[cursor:]) - break - if token_start > cursor: - plain: str = segment[cursor:token_start] - parts.append(plain) - output_length += len(plain) - rendered_token: str - next_cursor: int - rendered_token, next_cursor = _render_interpolation_token( - sql=segment, - start=token_start, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - ) - parts.append(rendered_token) - spans.append( - ExpansionSpan( - source_start=token_start, - source_end=next_cursor, - output_start=output_length, - output_end=output_length + len(rendered_token), - ) - ) - output_length += len(rendered_token) - cursor = next_cursor - return "".join(parts), tuple(spans) + interpolation: SqlInterpolation = interpolate_sql_batch( + sqls=((sql, file_path),), effective_vars=effective_vars, context_values=context_values + )[0] + return applied_interpolation(interpolation), interpolation.spans -def _render_interpolation_token( +def interpolate_sql_batch( *, - sql: str, - start: int, - file_path: Path, + sqls: tuple[tuple[str, Path], ...], effective_vars: dict[str, object], - context_values: Mapping[str, str | None] | None, -) -> tuple[str, int]: - if sql.startswith("@@@", start): - name_start: int = start + 3 - if name_start < len(sql) and _is_identifier_start(sql[name_start]): - name_end: int = _consume_identifier(sql=sql, start=name_start) - return sql[start:name_end], name_end - return "@@@", start + 3 - - token_start: int = start + 2 - if sql.startswith("ENV:", token_start): - env_name_start: int = token_start + len("ENV:") - env_name_end: int = _consume_env_name(sql=sql, start=env_name_start) - if env_name_end == env_name_start: - raise CompileInputError(f"invalid environment interpolation token in '{file_path}'") - env_name: str = sql[env_name_start:env_name_end] - COMPILE_INPUT_READS.environment_read(env_name) - if env_name not in os.environ: - raise CompileInputError( - f"unknown environment variable '@@ENV:{env_name}' in '{file_path}'" - ) - return os.environ[env_name], env_name_end - - if sql.startswith("CTX:", token_start): - context_name_start: int = token_start + len("CTX:") - context_name_end: int = _consume_context_name(sql=sql, start=context_name_start) - if context_name_end == context_name_start: - raise CompileInputError(f"invalid CTX interpolation token in '{file_path}'") - context_name: str = sql[context_name_start:context_name_end] - if context_values is None: - raise CompileInputError(f"SQL text in '{file_path}' does not allow @@CTX templates") - context_name_end = _known_context_name_end( - sql=sql, - start=context_name_start, - greedy_end=context_name_end, - context_values=context_values, + context_values: Mapping[str, str | None] | None = None, +) -> tuple[SqlInterpolation, ...]: + """Interpolate every `(sql, file path)` natively; errors wait in their result.""" + + rows = _native.interpolate_sql_batch( + [(sql, str(file_path)) for sql, file_path in sqls], + (effective_vars, os.environ, context_values, _render_variable), + (sys.version_info[0], sys.version_info[1]), + unicodedata.unidata_version, + ) + return tuple( + SqlInterpolation( + sql=rendered if rendered is not None else sql, + spans=tuple(ExpansionSpan(*span) for span in spans), + reads=tuple(reads), + error=error, ) - context_name = sql[context_name_start:context_name_end] - COMPILE_INPUT_READS.context_read(context_name) - if context_name not in context_values: - raise CompileInputError( - f"SQL text in '{file_path}' references unknown CTX key '{context_name}'" - ) - context_value: str | None = context_values[context_name] - if context_value is None: - raise CompileInputError( - f"SQL text in '{file_path}' references CTX key '{context_name}' " - "but no value is available" - ) - return context_value, context_name_end - - if token_start < len(sql) and _is_identifier_start(sql[token_start]): - name_end = _consume_identifier(sql=sql, start=token_start) - var_name: str = sql[token_start:name_end] - if var_name not in effective_vars: - raise CompileInputError( - _unknown_project_variable_message( - var_name=var_name, file_path=file_path, effective_vars=effective_vars - ) - ) - try: - return render_project_var_text( - value=effective_vars[var_name], - label=f"SQL variable '@@{var_name}'", - ), name_end - except ValueError as error: - raise CompileInputError(str(error)) from error - - return "@@", start + 2 - - -def _consume_identifier(*, sql: str, start: int) -> int: - cursor: int = start + 1 - while cursor < len(sql) and _is_identifier_continue(sql[cursor]): - cursor += 1 - return cursor - + for (sql, _file_path), (rendered, spans, reads, error) in zip(sqls, rows, strict=True) + ) -def _consume_env_name(*, sql: str, start: int) -> int: - cursor: int = start - while cursor < len(sql) and ( - sql[cursor].isalnum() or sql[cursor] == SQL_IDENTIFIER_EXTRA_TOKEN - ): - cursor += 1 - return cursor +def applied_interpolation(interpolation: SqlInterpolation) -> str: + """Record the interpolation's environment and context reads, then return or raise it.""" -def _consume_context_name(*, sql: str, start: int) -> int: - cursor: int = start - while cursor < len(sql) and ( - sql[cursor].isalnum() or sql[cursor] in SQL_CONTEXT_NAME_EXTRA_TOKENS - ): - cursor += 1 - return cursor + record_template_reads(interpolation.reads) + if interpolation.error is not None: + raise CompileInputError(interpolation.error) + return interpolation.sql -def _known_context_name_end( - *, - sql: str, - start: int, - greedy_end: int, - context_values: Mapping[str, str | None], -) -> int: - greedy_name: str = sql[start:greedy_end] - if greedy_name in context_values: - return greedy_end - matching_names: tuple[str, ...] = tuple( - name for name in context_values if greedy_name.startswith(f"{name}.") - ) - if not matching_names: - return greedy_end - return start + len(max(matching_names, key=len)) +def _render_variable(value: object, label: str) -> str: + return render_project_var_text(value=value, label=label) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 1b3e17de44..069c4ce8d7 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1313,6 +1313,16 @@ class ExpansionSpan: output_end: int +@dataclass(frozen=True) +class SqlInterpolation: + """One SQL text after `@@` interpolation, or the error that stopped it, with its reads.""" + + sql: str + spans: tuple[ExpansionSpan, ...] + reads: tuple[tuple[str, str], ...] + error: str | None + + @dataclass(frozen=True) class CompiledSqlExpansion: """Process-local authored-to-expanded SQL evidence shared with compiler checks.""" diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index e3e38174ff..1f632c24ac 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -56,7 +56,6 @@ class NativeFallbackSite(StrEnum): SCOPE_LOOKUP = "declaration_scopes.lookup" SCOPE_REBIND_LOOKUP = "declaration_scopes.rebind_lookup" SCOPE_RELATIONSHIP_CTES = "declaration_scopes.relationship_ctes" - SQL_VARIABLES = "model_loop.sql_variables" DECLARATION_REFERENCES = "model_loop.declaration_references" DECLARATION_CONTEXT = "model_loop.declaration_context" CURSOR_INTRINSIC_VALIDATION = "model_loop.cursor_intrinsic_validation" diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 623ac34f95..644a2e1f04 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -382,7 +382,7 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: "pair_seed_files", "render_attached_generic_audit", "expand_config_templates", - "substitute_static_project_vars", + "interpolate_sql_batch", "scan_sql_declaration_references", ) _MISSING_ENV: str = "SQB_ATTACHMENTS_UNSET" diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py index 1dd66a3c41..4d02acbe22 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py @@ -45,7 +45,7 @@ "pair_seed_files", "render_attached_generic_audit", "expand_config_templates", - "substitute_static_project_vars", + "interpolate_sql_batch", "scan_sql_declaration_references", "scan_test_parameter_references", "omitted_ceremonial_select", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index f8ed07c0ea..cc27baeb6d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -66,21 +66,12 @@ class SqlReferenceExtractionErrorTestCase: @dataclass(frozen=True) class StaticProjectVarBatchTestCase: - description: str - sqls: tuple[str, ...] - effective_vars: dict[str, object] - expected_sqls: tuple[str | None, ...] + """Model SQL texts and the `(sql, error)` each interpolates to.""" - -@dataclass(frozen=True) -class StaticProjectVarDifferentialTestCase: description: str - seed: int - sql_count: int + sqls: tuple[str, ...] effective_vars: dict[str, object] - fragments: tuple[str, ...] - expected_minimum_dollar_quote_substitutions: int - expected_minimum_doubled_backtick_substitutions: int + expected_results: tuple[tuple[str, str | None], ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index fc8eaeb908..a80f8c53c0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,4 +1,3 @@ -import random import sqlite3 from collections.abc import Callable from dataclasses import dataclass, replace @@ -23,12 +22,7 @@ expand_sql_macros, load_project_macros, ) -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - prepare_static_project_vars_batch, - substitute_sql_vars, -) from sqlbuild.compiler.compile._helpers.sql_tests.native import extract_unexpanded_sql_test -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -826,106 +820,6 @@ def diagnostic_spans( ) -@dataclass(frozen=True) -class StaticProjectVarDifferential: - """Native static substitutions beside the pure-Python scanner's output for the same SQL.""" - - native: dict[str, str] - python: dict[str, str | None] - substituted_dollar_quotes: int - substituted_doubled_backticks: int - - -LEXICAL_SQL_FRAGMENTS: tuple[str, ...] = ( - "$", - "$$", - "$t$", - "$tag$", - "a$", - "$1", - "$1$", - "--", - "/*", - "*/", - "'", - '"', - "`", - "@@revision", - "@@status", - "@@@window_start", - "@@", - "@", - " ", - "\n", - "x", - "_", - "\u00e9", -) -BACKTICK_SQL_FRAGMENTS: tuple[str, ...] = ( - "`", - "``", - "'", - "''", - '"', - "@@revision", - "@@status", - "@@", - " ", - "x", - "--", - "\n", -) -_LEXICAL_SQL_MAX_FRAGMENTS: int = 12 -_DIFFERENTIAL_FILE_PATH: Path = Path("models/generated.sql") - - -def generated_lexical_sqls(*, seed: int, count: int, fragments: tuple[str, ...]) -> tuple[str, ...]: - """Generate SQL-like strings mixing quotes, dollar quotes, comments and variable tokens.""" - - rng: random.Random = random.Random(seed) - sqls: list[str] = [] - for _ in range(count): - fragment_count: int = rng.randint(1, _LEXICAL_SQL_MAX_FRAGMENTS) - sqls.append("".join(rng.choices(fragments, k=fragment_count))) - return tuple(sqls) - - -def native_static_project_var_differential( - *, sqls: tuple[str, ...], effective_vars: dict[str, object] -) -> StaticProjectVarDifferential: - """Pair every native static substitution with the pure-Python scanner's output.""" - - prepared: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=sqls, effective_vars=effective_vars - ) - native: dict[str, str] = cast( - dict[str, str], - dict(filter(lambda pair: pair[1] is not None, zip(sqls, prepared, strict=True))), - ) - python: dict[str, str | None] = { - sql: _python_substituted_sql(sql=sql, effective_vars=effective_vars) for sql in native - } - return StaticProjectVarDifferential( - native=native, - python=python, - substituted_dollar_quotes=sum( - sql != rendered and "$$" in sql for sql, rendered in native.items() - ), - substituted_doubled_backticks=sum( - sql != rendered and "``" in sql for sql, rendered in native.items() - ), - ) - - -def _python_substituted_sql(*, sql: str, effective_vars: dict[str, object]) -> str | None: - try: - return substitute_sql_vars( - sql=sql, file_path=_DIFFERENTIAL_FILE_PATH, effective_vars=effective_vars - ) - except CompileInputError: - return None - - def extract_test_ctes( *, ctes: tuple[tuple[str, str], ...], file_label: str, mode: SqlTestMode ) -> CompileSqlTestCtes: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index ac84cca696..ef4d614ef6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -7,23 +7,16 @@ import pytest from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - prepare_static_project_vars_batch, + interpolate_sql_batch, substitute_sql_vars, ) from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import SqlInterpolation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( StaticProjectVarBatchTestCase, - StaticProjectVarDifferentialTestCase, SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, ) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( - BACKTICK_SQL_FRAGMENTS, - LEXICAL_SQL_FRAGMENTS, - StaticProjectVarDifferential, - generated_lexical_sqls, - native_static_project_var_differential, -) _FILE_PATH: Path = Path("models/test_model.sql") @@ -233,101 +226,43 @@ def test_given_missing_var_when_substituting_then_raises( "test_case", [ StaticProjectVarBatchTestCase( - description="static scalars use native results and dynamic values fall back", + description="each model is interpolated; an error waits in its own result", sqls=( "SELECT @@revision, '@@status', @@@window_start", "-- @@revision\nSELECT 1", - "SELECT '@@ENV:SQLBUILD_TEST_BATCH_USER'", - ), - effective_vars={"revision": 7, "status": "ready"}, - expected_sqls=( - "SELECT 7, 'ready', @@@window_start", - "-- @@revision\nSELECT 1", - None, - ), - ), - StaticProjectVarBatchTestCase( - description="dollar-quoted text is quoted text, not comments", - sqls=( - "SELECT $$--@@revision$$ AS x", - "SELECT $$ /* $$ AS a, '@@status' AS b -- */", + "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", "SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", + "SELECT @@grants", ), - effective_vars={"revision": 7, "status": "ready"}, - expected_sqls=( - "SELECT $$--7$$ AS x", - "SELECT $$ /* $$ AS a, 'ready' AS b -- */", - "SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", + effective_vars={"revision": 7, "status": "ready", "grants": {"role": "analyst"}}, + expected_results=( + ("SELECT 7, 'ready', @@@window_start", None), + ("-- @@revision\nSELECT 1", None), + ( + "SELECT '@@ENV:SQLBUILD_TEST_BATCH_UNSET'", + "unknown environment variable '@@ENV:SQLBUILD_TEST_BATCH_UNSET' in " + "'models/test_model.sql'", + ), + ("SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", None), + ( + "SELECT @@grants", + "SQL variable '@@grants' is an object and cannot be interpolated as text: " + '{"role":"analyst"}. Use a macro to consume structured vars.', + ), ), ), - StaticProjectVarBatchTestCase( - description="unclosed dollar quotes fall back to the Python scanner", - sqls=("SELECT $tag$ @@revision", "-- @@revision\nSELECT $$ open"), - effective_vars={"revision": 7}, - expected_sqls=(None, None), - ), - StaticProjectVarBatchTestCase( - description="structured project variables fall back", - sqls=("SELECT @@grants",), - effective_vars={"grants": {"role": "analyst"}}, - expected_sqls=(None,), - ), ], ids=lambda case: case.description, ) -def test_given_static_model_sql_when_batch_preparing_then_returns_safe_results_and_fallbacks( +def test_given_model_sqls_when_interpolating_a_batch_then_each_holds_its_sql_or_error( test_case: StaticProjectVarBatchTestCase, ) -> None: - result: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=test_case.sqls, + results: tuple[SqlInterpolation, ...] = interpolate_sql_batch( + sqls=tuple((sql, _FILE_PATH) for sql in test_case.sqls), effective_vars=test_case.effective_vars, ) - assert result == test_case.expected_sqls - - -@pytest.mark.parametrize( - "test_case", - [ - StaticProjectVarDifferentialTestCase( - description="generated quotes, dollar quotes, comments and tokens", - seed=550, - sql_count=4000, - effective_vars={"revision": 7, "status": "ready"}, - fragments=LEXICAL_SQL_FRAGMENTS, - expected_minimum_dollar_quote_substitutions=1, - expected_minimum_doubled_backtick_substitutions=0, - ), - StaticProjectVarDifferentialTestCase( - description="generated doubled backticks beside quotes, comments and tokens", - seed=1038, - sql_count=4000, - effective_vars={"revision": 7, "status": "ready"}, - fragments=BACKTICK_SQL_FRAGMENTS, - expected_minimum_dollar_quote_substitutions=0, - expected_minimum_doubled_backtick_substitutions=200, - ), - ], - ids=lambda case: case.description, -) -def test_given_generated_lexical_sql_when_batch_preparing_then_native_matches_python_scanner( - test_case: StaticProjectVarDifferentialTestCase, -) -> None: - sqls: tuple[str, ...] = generated_lexical_sqls( - seed=test_case.seed, count=test_case.sql_count, fragments=test_case.fragments - ) - - differential: StaticProjectVarDifferential = native_static_project_var_differential( - sqls=sqls, effective_vars=test_case.effective_vars - ) - - assert ( - differential.native == differential.python, - differential.substituted_dollar_quotes - >= test_case.expected_minimum_dollar_quote_substitutions, - differential.substituted_doubled_backticks - >= test_case.expected_minimum_doubled_backtick_substitutions, - ) == (True, True, True), test_case.description + assert [(result.sql, result.error) for result in results] == list(test_case.expected_results) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py index ef6d634e54..20c7960c22 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py @@ -20,15 +20,15 @@ NativeFallbackRecorderTestCase( description="counts_per_site_and_kind_sorted", recorded=( - (NativeFallbackSite.SQL_VARIABLES, "deferred"), + (NativeFallbackSite.CONFIG_BUILD, "deferred"), (NativeFallbackSite.REFERENCE_SCAN, "deferred"), - (NativeFallbackSite.SQL_VARIABLES, "deferred"), - (NativeFallbackSite.SQL_VARIABLES, "error"), + (NativeFallbackSite.CONFIG_BUILD, "deferred"), + (NativeFallbackSite.CONFIG_BUILD, "error"), ), expected_files=[ [ - ["model_loop.sql_variables", "deferred", 2], - ["model_loop.sql_variables", "error", 1], + ["model_config.build", "deferred", 2], + ["model_config.build", "error", 1], ["reference_extraction.scan", "deferred", 1], ] ], From 61f2897def1af2446939e8553f6faffee6fa9a78 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:03:40 +0100 Subject: [PATCH 02/40] feat(native): interpolate SQL variables and expand templates natively; delete the Python paths --- src/sqlbuild/_native.pyi | 7 +- .../compile/_helpers/assembly/project.py | 5 - .../compile/_helpers/assembly/targets.py | 7 - .../compile/_helpers/attachment/core.py | 9 - .../compile/_helpers/attachment/functions.py | 2 - .../_helpers/attachment/model_config.py | 2 +- .../compile/_helpers/attachment/sources.py | 3 - .../_helpers/render/context_templates.py | 87 +-- .../compile/_helpers/render/sql_vars.py | 4 +- .../compile/_helpers/render/templating.py | 540 ++---------------- src/sqlbuild/compiler/compile/constants.py | 22 - .../compile/main/expand_template_data.py | 5 - src/sqlbuild/compiler/compile/types.py | 5 - src/sqlbuild/compiler/frontier/types.py | 2 - .../main/_expand_native_config_templates.py | 37 +- .../test_native_attachment_project.py | 1 - .../compiler/model_config/_test_types.py | 15 - .../sqlbuild/compiler/model_config/helpers.py | 259 +-------- .../model_config/test_native_model_config.py | 74 +-- 19 files changed, 64 insertions(+), 1022 deletions(-) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cbeecee66f..da1b80420a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -181,9 +181,10 @@ def expand_config_templates( value: object, sources: tuple[dict[str, object], object, dict[str, str | None]], flags: tuple[bool, bool, bool, str], -) -> ( - tuple[object, list[tuple[str, str]]] | tuple[NativeConfigError, list[tuple[str, str]]] | str -): ... +) -> tuple[object, list[tuple[str, str]]]: ... +def expand_effective_vars( + raw_values: dict[str, object], environment: object +) -> tuple[object, list[tuple[str, str]]]: ... class NativeModelConfigBuilder: def __init__( diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index aa496d3302..998ad6b660 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -171,9 +171,6 @@ DiagnosticSeverity, SqlTestMode, ) -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main._record_cpu import record_compile_cpu_timing @@ -1583,8 +1580,6 @@ def _build_source_relation_entry( def _expand_target_value(*, value: str | None, effective_vars: dict[str, object]) -> str | None: if value is None: return None - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="source_target") return str( expand_template_data( value=value, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py b/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py index 0f535172a1..28bd9e7bed 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py @@ -8,9 +8,6 @@ from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE from sqlbuild.compiler.compile.models import CompiledRelationLocation, CompileModelInput -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.spec.contracts.models import DefaultsConfig, SchemaSeedEntry, TargetConfig @@ -153,8 +150,6 @@ def _expand_seed_environment_value( ) -> str | None: if raw_value == PRESERVE_TARGET_VALUE: return None - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="seed_target") return str( expand_template_data( value=raw_value, @@ -179,8 +174,6 @@ def _expand_seed_target_value( ) -> str | None: if raw_value == PRESERVE_TARGET_VALUE: return None - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="seed_target") return str( expand_template_data( value=raw_value, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 5465555df0..0dc4ba8f0c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -1267,8 +1267,6 @@ def build_effective_connection( connection.update(local_config.connections.get(connection_name, {})) connection.update(target_config.connection) connection.update(local_config.connection) - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="connection") return cast( dict[str, object], expand_template_data( @@ -1311,8 +1309,6 @@ def build_effective_vars( values.update(target_config.vars) values.update(local_config.vars) values.update(cli_vars) - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="effective_vars") return expand_effective_vars(values) @@ -1332,7 +1328,6 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi request.materialization_defaults ) scan_cache: ModelConfigScanCache | None = request.scan_cache - native: bool = scan_cache is not None and scan_cache.native _validate_model_header_tags(model_header_values=model_header_values, model_name=model_name) layered_values: dict[str, object] = build_layered_model_values( @@ -1358,14 +1353,12 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi effective_vars=effective_vars, effective_target_name=effective_target_name, run_id=run_id, - native=native, ) model_resolved_values: dict[str, object] = resolve_chained_model_context_templates( values=early_resolved_values, model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, - native=native, ) else: model_resolved_values = layered_values @@ -1396,7 +1389,6 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi run_id=run_id, include_target_values=False, ), - native=native, ) target_resolved_values: dict[str, object] = ( resolve_target_context_templates( @@ -1404,7 +1396,6 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, - native=native, ) if has_authored_templates or contains_template_data(model_resolved_values.get("database")) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index f88834410f..b79645b360 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -887,7 +887,6 @@ def _expand_function_header_value( allow_context=False, preserve_context_tokens=True, preserve_unknown_context=False, - native=native_stage_enabled(NativeStage.ATTACHMENTS), ) ) @@ -986,6 +985,5 @@ def _expand_function_environment_value( allow_context=False, preserve_context_tokens=True, preserve_unknown_context=False, - native=native_stage_enabled(NativeStage.ATTACHMENTS), ) ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index db35c7d23e..a80106dbd7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -32,9 +32,9 @@ ) from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type -from sqlbuild.compiler.compile._helpers.render.context_templates import record_template_reads from sqlbuild.compiler.compile._helpers.render.templating import ( contains_template_data, + record_template_reads, ) from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py index eee929f819..fc28ee11ec 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py @@ -26,8 +26,6 @@ DiscoveredProjectInputs, DiscoveredSourceFile, ) -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.models import ResourceIdentity, UsageRecord, VisibilityRecord from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind from sqlbuild.spec.contracts.models import ( @@ -328,5 +326,4 @@ def _expand_source_template_object( allow_context=False, preserve_context_tokens=False, preserve_unknown_context=False, - native=native_stage_enabled(NativeStage.ATTACHMENTS), ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index b995a5e940..e81a0725dc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -8,26 +8,9 @@ from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile.constants import ( - COMPILE_INPUT_READS, - MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, - MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, - MISSING_TEMPLATE_VALUE_MESSAGE_PARTS, PRESERVE_TARGET_VALUE, ) from sqlbuild.compiler.compile.types import CompileContextKey -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage -from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ -from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( - expand_native_config_templates, -) -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.model_config.models import ( - NativeTemplateExpansion, - NativeTemplateRejection, - TemplateResolutionFlags, -) from sqlbuild.spec.contracts.models import TargetConfig @@ -37,7 +20,6 @@ def resolve_early_model_templates( effective_vars: dict[str, object], effective_target_name: str | None, run_id: str, - native: bool = False, ) -> dict[str, object]: """Resolve `${name}`, `${ENV:...}`, and early `run.*` model templates.""" @@ -54,7 +36,6 @@ def resolve_early_model_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=True, - native=native, ), ) @@ -65,7 +46,6 @@ def resolve_model_context_templates( model_name: str, effective_target_name: str | None, run_id: str, - native: bool = False, ) -> dict[str, object]: """Resolve model-bound `CTX` values once logical model identity is known.""" @@ -85,7 +65,6 @@ def resolve_model_context_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=True, - native=native, ), ) @@ -96,7 +75,6 @@ def resolve_chained_model_context_templates( model_name: str, effective_target_name: str | None, run_id: str, - native: bool = False, ) -> dict[str, object]: """Resolve twice so ${CTX:model.*} values may chain exactly one level without looping.""" @@ -105,14 +83,12 @@ def resolve_chained_model_context_templates( model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, - native=native, ) return resolve_model_context_templates( values=first_pass_values, model_name=model_name, effective_target_name=effective_target_name, run_id=run_id, - native=native, ) @@ -122,7 +98,6 @@ def resolve_target_context_templates( model_name: str, effective_target_name: str | None, run_id: str, - native: bool = False, ) -> dict[str, object]: """Resolve late `target.*` values after environment overrides finalize naming.""" @@ -142,7 +117,6 @@ def resolve_target_context_templates( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, - native=native, ), ) @@ -208,7 +182,6 @@ def apply_environment_database_schema_overrides( effective_vars: dict[str, object], target_config: TargetConfig | None, model_context_values: dict[str, str | None], - native: bool = False, ) -> dict[str, object]: """Return values with environment database/schema overrides applied.""" @@ -225,7 +198,6 @@ def apply_environment_database_schema_overrides( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, - native=native, ) if target_config.schema is not None and target_config.schema != PRESERVE_TARGET_VALUE: overridden["schema"] = expand_config_templates( @@ -236,7 +208,6 @@ def apply_environment_database_schema_overrides( allow_context=True, preserve_context_tokens=False, preserve_unknown_context=False, - native=native, ) return overridden @@ -260,40 +231,9 @@ def expand_config_templates( allow_context: bool, preserve_context_tokens: bool, preserve_unknown_context: bool, - native: bool, ) -> object: - """Expand like `expand_template_data`, natively when set; unsupported values run Python.""" - - outcome: NativeTemplateExpansion | NativeTemplateRejection | str | None = ( - expand_native_config_templates( - value=value, - variables=variables, - context_values=context_values, - flags=TemplateResolutionFlags( - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ), - context_label=context_label, - ) - if native - else None - ) - if isinstance(outcome, NativeTemplateExpansion): - record_template_reads(outcome.reads) - return outcome.value - if isinstance(outcome, NativeTemplateRejection) and not _names_missing_template_value( - context_label - ): - record_template_reads(outcome.reads) - raise native_config_error(error=outcome.error, bridge_independent=True) - if native: - report_native_fallback( - site=NativeFallbackSite.CONFIG_TEMPLATES, - kind="unsupported" if outcome is None or isinstance(outcome, str) else "missing_value", - ) - elif native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="config") + """Expand `${...}` templates in a config value natively, recording what it reads.""" + return expand_template_data( value=value, variables=variables, @@ -303,26 +243,3 @@ def expand_config_templates( preserve_context_tokens=preserve_context_tokens, preserve_unknown_context=preserve_unknown_context, ) - - -def _names_missing_template_value(context_label: str) -> bool: - """Whether Python's `coalesce` would read errors naming this label as missing values.""" - - return any( - part in context_label - for part in ( - *MISSING_TEMPLATE_VALUE_MESSAGE_PARTS, - MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, - MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, - ) - ) - - -def record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: - """Record native template environment and context reads for project reuse, in order.""" - - for kind, name in reads: - if kind == ENVIRONMENT_READ: - COMPILE_INPUT_READS.environment_read(name) - else: - COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 5ee7deb01a..859cbf948c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -10,7 +10,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile._helpers.render.context_templates import record_template_reads from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_declaration_references_result, expand_declaration_references_with_spans, @@ -20,6 +19,7 @@ expand_sql_macros_result, expand_sql_macros_with_spans, ) +from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -219,7 +219,7 @@ def applied_interpolation(interpolation: SqlInterpolation) -> str: record_template_reads(interpolation.reads) if interpolation.error is not None: - raise CompileInputError(interpolation.error) + raise CompileInputError(interpolation.error, bridge_independent=True) return interpolation.sql diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 54dd2dd0eb..f75b39ba2b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -1,109 +1,34 @@ -"""Template expansion helpers for compile input resolution.""" +"""Template expansion for compile inputs, natively, with the reads it makes recorded.""" from __future__ import annotations import os -import re -from collections.abc import Callable, Mapping -from dataclasses import dataclass +from typing import cast -from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile.constants import ( - COMPILE_INPUT_READS, - MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, - MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, - MISSING_TEMPLATE_VALUE_MESSAGE_PARTS, - TEMPLATE_COALESCE_FUNCTION_NAME, - TEMPLATE_EQ_FUNCTION_NAME, - TEMPLATE_ESCAPE_TOKEN, - TEMPLATE_FALSE_LITERAL, - TEMPLATE_FALSE_VALUES, - TEMPLATE_IF_FUNCTION_NAME, - TEMPLATE_NAMESPACE_SEPARATOR, - TEMPLATE_NE_FUNCTION_NAME, - TEMPLATE_NULL_LITERAL, - TEMPLATE_OPEN_TOKEN, - TEMPLATE_PATTERN, - TEMPLATE_TRUE_LITERAL, +import sqlbuild._native as _native +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, TEMPLATE_OPEN_TOKEN +from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ +from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( + expand_native_config_templates, +) +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error +from sqlbuild.compiler.model_config.models import ( + NativeTemplateExpansion, + NativeTemplateRejection, + TemplateResolutionFlags, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import TemplateNamespace - - -@dataclass(frozen=True) -class _TemplateToken: - kind: str - value: str - position: int - - -@dataclass(frozen=True) -class _TemplateStringExpr: - value: str - - -@dataclass(frozen=True) -class _TemplateReferenceExpr: - value: str - - -@dataclass(frozen=True) -class _TemplateFunctionExpr: - name: str - arguments: tuple[object, ...] - - -_TEMPLATE_END_TOKEN: str = "end" -_TEMPLATE_WORD_TOKEN: str = "word" -_TEMPLATE_STRING_TOKEN: str = "string" -_TEMPLATE_SYMBOL_TOKEN: str = "symbol" -_TEMPLATE_SYMBOLS: frozenset[str] = frozenset({"(", ")", ","}) -_TEMPLATE_QUOTE_NAMES: dict[str, str] = {"'": "single", '"': "double"} - - -class _EffectiveVarsResolver: - def __init__(self, raw_values: dict[str, object]) -> None: - self.raw_values = raw_values - self.resolved_values: dict[str, object] = {} - self.resolving_keys: tuple[str, ...] = () - - def resolve(self, key: str) -> object: - if key not in self.raw_values: - raise CompileInputError(f"effective vars references unknown variable '{key}'") - if key in self.resolved_values: - return self.resolved_values[key] - if key in self.resolving_keys: - cycle: str = " -> ".join((*self.resolving_keys, key)) - raise CompileInputError(f"effective vars contain a cyclic reference: {cycle}") - self.resolving_keys = (*self.resolving_keys, key) - raw_value: object = self.raw_values[key] - resolved_value: object = ( - expand_template_string( - value=raw_value, - variables=self.raw_values, - resolve_variable=self.resolve, - context_values={}, - context_label="effective vars", - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) - if isinstance(raw_value, str) - else raw_value - ) - self.resolving_keys = self.resolving_keys[:-1] - self.resolved_values[key] = resolved_value - return resolved_value def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: """Resolve merged effective vars with recursive `${name}` expansion.""" - resolver: _EffectiveVarsResolver = _EffectiveVarsResolver(raw_values) - key: str - for key in raw_values: - resolver.resolve(key) - return resolver.resolved_values + outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( + raw_values, os.environ + ) + record_template_reads(tuple(outcome[1])) + if isinstance(outcome[0], _native.NativeConfigError): + raise native_config_error(error=outcome[0], bridge_independent=True) + return cast(dict[str, object], outcome[0]) def contains_template_data(value: object) -> bool: @@ -130,419 +55,28 @@ def expand_template_data( ) -> object: """Recursively expand template strings inside supported Python container values.""" - if isinstance(value, str): - if TEMPLATE_OPEN_TOKEN not in value: - return value - resolver: _TemplateResolver = _TemplateResolver( - variables=variables, - resolve_variable=lambda name: _lookup_variable( - name=name, - variables=variables, - context_label=context_label, - ), - context_values=context_values, - context_label=context_label, + outcome: NativeTemplateExpansion | NativeTemplateRejection = expand_native_config_templates( + value=value, + variables=variables, + context_values=context_values, + flags=TemplateResolutionFlags( allow_context=allow_context, preserve_context_tokens=preserve_context_tokens, preserve_unknown_context=preserve_unknown_context, - ) - full_match: re.Match[str] | None = TEMPLATE_PATTERN.fullmatch(value) - if full_match is not None: - return resolver.evaluate_token(full_match.group(1)) - return resolver.expand_string(value) - if isinstance(value, dict): - return { - key: expand_template_data( - value=item_value, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) - for key, item_value in value.items() - } - if isinstance(value, list): - return [ - expand_template_data( - value=item, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) - for item in value - ] - if isinstance(value, tuple): - return tuple( - expand_template_data( - value=item, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) - for item in value - ) - return value - - -def expand_template_string( - *, - value: str, - variables: dict[str, object], - resolve_variable: Callable[[str], object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> str: - """Expand `${...}` templates within a single string value.""" - - resolver: _TemplateResolver = _TemplateResolver( - variables=variables, - resolve_variable=resolve_variable, - context_values=context_values, + ), context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) - return resolver.expand_string(value) - - -class _TemplateResolver: - def __init__( - self, - *, - variables: dict[str, object], - resolve_variable: Callable[[str], object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, - ) -> None: - self.variables = variables - self.resolve_variable = resolve_variable - self.context_values = context_values - self.context_label = context_label - self.allow_context = allow_context - self.preserve_context_tokens = preserve_context_tokens - self.preserve_unknown_context = preserve_unknown_context - - def expand_string(self, value: str) -> str: - def replace_match(match: re.Match[str]) -> str: - token: str = match.group(1) - return _stringify_template_value( - value=self.evaluate_token(token), - label=f"{self.context_label} variable '{token}'", - ) - - return TEMPLATE_PATTERN.sub(replace_match, value) - - def evaluate_token(self, token: str) -> object: - parser: _TemplateExpressionParser = _TemplateExpressionParser(token) - expression: object = parser.parse() - return self._evaluate_expression(expression) - - def _evaluate_expression(self, expression: object) -> object: - if isinstance(expression, _TemplateStringExpr): - return expression.value - if isinstance(expression, _TemplateReferenceExpr): - return self._evaluate_reference(expression.value) - if isinstance(expression, _TemplateFunctionExpr): - return self._evaluate_function(expression) - raise CompileInputError(f"{self.context_label} contains unsupported template expression") - - def _evaluate_reference(self, value: str) -> object: - if value == TEMPLATE_TRUE_LITERAL: - return True - if value == TEMPLATE_FALSE_LITERAL: - return False - if value == TEMPLATE_NULL_LITERAL: - return None - if TEMPLATE_NAMESPACE_SEPARATOR not in value: - return self.resolve_variable(value) - - namespace: str - name: str - namespace, name = value.split(":", 1) - if namespace == TemplateNamespace.ENV: - return _lookup_environment_variable(name=name, context_label=self.context_label) - if namespace == TemplateNamespace.CTX: - if not self.allow_context: - if self.preserve_context_tokens: - return f"${{{value}}}" - raise CompileInputError(f"{self.context_label} does not allow CTX templates") - return _lookup_context_value( - name=name, - context_values=self.context_values, - context_label=self.context_label, - preserve_unknown_context=self.preserve_unknown_context, - ) - raise CompileInputError( - f"{self.context_label} references unsupported template namespace '{namespace}'" - ) - - def _evaluate_function(self, expression: _TemplateFunctionExpr) -> object: - function_name: str = expression.name - conditional_argument_count: int = 3 - comparison_argument_count: int = 2 - if function_name == TEMPLATE_IF_FUNCTION_NAME: - if len(expression.arguments) != conditional_argument_count: - raise CompileInputError(f"{self.context_label} if(...) expects 3 arguments") - condition_value: object = self._evaluate_expression(expression.arguments[0]) - if _is_truthy_template_value(condition_value): - return self._evaluate_expression(expression.arguments[1]) - return self._evaluate_expression(expression.arguments[2]) - if function_name == TEMPLATE_EQ_FUNCTION_NAME: - if len(expression.arguments) != comparison_argument_count: - raise CompileInputError(f"{self.context_label} eq(...) expects 2 arguments") - left: object = self._evaluate_expression(expression.arguments[0]) - right: object = self._evaluate_expression(expression.arguments[1]) - return _normalize_template_comparison_value( - left - ) == _normalize_template_comparison_value(right) - if function_name == TEMPLATE_NE_FUNCTION_NAME: - if len(expression.arguments) != comparison_argument_count: - raise CompileInputError(f"{self.context_label} ne(...) expects 2 arguments") - left = self._evaluate_expression(expression.arguments[0]) - right = self._evaluate_expression(expression.arguments[1]) - return _normalize_template_comparison_value( - left - ) != _normalize_template_comparison_value(right) - if function_name == TEMPLATE_COALESCE_FUNCTION_NAME: - if not expression.arguments: - raise CompileInputError( - f"{self.context_label} coalesce(...) expects at least 1 argument" - ) - argument: object - for argument in expression.arguments: - value: object | None = self._evaluate_optional_expression(argument) - if _is_truthy_template_value(value): - return value - return self._evaluate_expression(expression.arguments[-1]) - raise CompileInputError( - f"{self.context_label} references unsupported template function '{function_name}'" - ) - - def _evaluate_optional_expression(self, expression: object) -> object | None: - try: - return self._evaluate_expression(expression) - except CompileInputError as error: - if _is_missing_template_value_error(str(error)): - return None - raise - - -class _TemplateExpressionParser: - def __init__(self, token: str) -> None: - self._tokens: list[_TemplateToken] = _tokenize_template_expression(token) - self._index: int = 0 - - def parse(self) -> object: - expression: object = self._parse_expression() - self._expect_end() - return expression - - def _parse_expression(self) -> object: - token: _TemplateToken = self._peek() - if token.kind == _TEMPLATE_STRING_TOKEN: - self._advance() - return _TemplateStringExpr(token.value) - if token.kind == _TEMPLATE_WORD_TOKEN: - self._advance() - if self._match_symbol("("): - return self._parse_function_call(token.value) - return _TemplateReferenceExpr(token.value) - raise CompileInputError( - "template expression contains unexpected token " - f"'{token.value}' at position {token.position}" - ) - - def _parse_function_call(self, name: str) -> _TemplateFunctionExpr: - arguments: list[object] = [] - while not self._is_at_end_symbol(")"): - arguments.append(self._parse_expression()) - if not self._match_symbol(","): - break - self._consume_symbol(")") - return _TemplateFunctionExpr(name=name, arguments=tuple(arguments)) - - def _is_at_end_symbol(self, symbol: str) -> bool: - token: _TemplateToken = self._peek() - return token.kind == _TEMPLATE_SYMBOL_TOKEN and token.value == symbol - - def _match_symbol(self, symbol: str) -> bool: - token: _TemplateToken = self._peek() - if token.kind == _TEMPLATE_SYMBOL_TOKEN and token.value == symbol: - self._advance() - return True - return False - - def _consume_symbol(self, symbol: str) -> None: - if self._match_symbol(symbol): - return - token: _TemplateToken = self._peek() - raise CompileInputError( - f"template expression expected '{symbol}' at position {token.position}" - ) - - def _expect_end(self) -> None: - token: _TemplateToken = self._peek() - if token.kind != _TEMPLATE_END_TOKEN: - raise CompileInputError( - "template expression contains unexpected token " - f"'{token.value}' at position {token.position}" - ) - - def _peek(self) -> _TemplateToken: - return self._tokens[self._index] - - def _advance(self) -> _TemplateToken: - token: _TemplateToken = self._tokens[self._index] - self._index += 1 - return token - - -def _tokenize_template_expression(value: str) -> list[_TemplateToken]: - tokens: list[_TemplateToken] = [] - index: int = 0 - while index < len(value): - character: str = value[index] - if character.isspace(): - index += 1 - continue - if character in _TEMPLATE_SYMBOLS: - tokens.append( - _TemplateToken(kind=_TEMPLATE_SYMBOL_TOKEN, value=character, position=index) - ) - index += 1 - continue - if character in _TEMPLATE_QUOTE_NAMES: - string_value: str - next_index: int - string_value, next_index = _read_template_quoted_string(value=value, start=index) - tokens.append( - _TemplateToken(kind=_TEMPLATE_STRING_TOKEN, value=string_value, position=index) - ) - index = next_index - continue - next_index = index - while next_index < len(value): - next_character: str = value[next_index] - if ( - next_character.isspace() - or next_character in _TEMPLATE_SYMBOLS - or next_character in _TEMPLATE_QUOTE_NAMES - ): - break - next_index += 1 - word: str = value[index:next_index] - if not word: - raise CompileInputError( - "template expression contains unexpected character " - f"'{character}' at position {index}" - ) - tokens.append(_TemplateToken(kind=_TEMPLATE_WORD_TOKEN, value=word, position=index)) - index = next_index - tokens.append(_TemplateToken(kind=_TEMPLATE_END_TOKEN, value="", position=len(value))) - return tokens - - -def _read_template_quoted_string(*, value: str, start: int) -> tuple[str, int]: - parts: list[str] = [] - quote: str = value[start] - quote_name: str = _TEMPLATE_QUOTE_NAMES[quote] - index: int = start + 1 - while index < len(value): - character: str = value[index] - if character == TEMPLATE_ESCAPE_TOKEN: - if index + 1 >= len(value): - raise CompileInputError( - f"template expression has unterminated escape at position {index}" - ) - parts.append(value[index + 1]) - index += 2 - continue - if character == quote: - return "".join(parts), index + 1 - parts.append(character) - index += 1 - raise CompileInputError( - f"template expression has unterminated {quote_name}-quoted string at position {start}" - ) - - -def _normalize_template_comparison_value(value: object) -> str: - if value is None: - return "" - if isinstance(value, bool): - return "true" if value else "false" - return str(value) - - -def _is_truthy_template_value(value: object) -> bool: - if value is None: - return False - if isinstance(value, bool): - return value - normalized: str = str(value).strip().lower() - if normalized in TEMPLATE_FALSE_VALUES: - return False - return True - - -def _stringify_template_value(*, value: object, label: str) -> str: - try: - return render_project_var_text(value=value, label=label) - except ValueError as error: - raise CompileInputError(str(error)) from error - - -def _is_missing_template_value_error(message: str) -> bool: - return ( - any(part in message for part in MISSING_TEMPLATE_VALUE_MESSAGE_PARTS) - or MISSING_TEMPLATE_CONTEXT_MESSAGE_PART in message - and MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART in message ) + record_template_reads(outcome.reads) + if isinstance(outcome, NativeTemplateRejection): + raise native_config_error(error=outcome.error, bridge_independent=True) + return outcome.value -def _lookup_variable(*, name: str, variables: dict[str, object], context_label: str) -> object: - if name not in variables: - raise CompileInputError(f"{context_label} references unknown variable '{name}'") - return variables[name] - - -def _lookup_environment_variable(*, name: str, context_label: str) -> str: - COMPILE_INPUT_READS.environment_read(name) - if name not in os.environ: - raise CompileInputError(f"{context_label} references missing ENV variable '{name}'") - return os.environ[name] +def record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: + """Record native template environment and context reads for project reuse, in order.""" - -def _lookup_context_value( - *, - name: str, - context_values: Mapping[str, str | None], - context_label: str, - preserve_unknown_context: bool, -) -> str: - COMPILE_INPUT_READS.context_read(name) - if name not in context_values: - if preserve_unknown_context: - return f"${{{TemplateNamespace.CTX}:{name}}}" - raise CompileInputError(f"{context_label} references unknown CTX key '{name}'") - context_value: str | None = context_values.get(name) - if context_value is None: - raise CompileInputError( - f"{context_label} references CTX key '{name}' but no value is available" - ) - return context_value + for kind, name in reads: + if kind == ENVIRONMENT_READ: + COMPILE_INPUT_READS.environment_read(name) + else: + COMPILE_INPUT_READS.context_read(name) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index d4777ac0ee..63c94a9167 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -91,7 +91,6 @@ SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) -SQL_IDENTIFIER_EXTRA_TOKEN: str = "_" SQL_WITH_KEYWORD: str = "WITH" SQL_CEREMONIAL_SELECT_VALUE: str = "1" OMITTED_CEREMONIAL_SELECT_SQL: str = f"\nSELECT {SQL_CEREMONIAL_SELECT_VALUE}" @@ -113,34 +112,13 @@ MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" PYTHON_LITERAL_NAMES: frozenset[str] = frozenset({"True", "False", "None"}) SQL_INTERPOLATION_TOKEN: str = "@@" -SQL_CONTEXT_NAME_EXTRA_TOKENS: frozenset[str] = frozenset({"_", "."}) -TEMPLATE_TRUE_LITERAL: str = "true" -TEMPLATE_FALSE_LITERAL: str = "false" -TEMPLATE_NULL_LITERAL: str = "null" -TEMPLATE_NAMESPACE_SEPARATOR: str = ":" -TEMPLATE_IF_FUNCTION_NAME: str = "if" -TEMPLATE_EQ_FUNCTION_NAME: str = "eq" -TEMPLATE_NE_FUNCTION_NAME: str = "ne" -TEMPLATE_COALESCE_FUNCTION_NAME: str = "coalesce" -TEMPLATE_ESCAPE_TOKEN: str = "\\" TEMPLATE_OPEN_TOKEN: str = "${" -TEMPLATE_FALSE_VALUES: frozenset[str] = frozenset({"", "0", "false"}) POLYGLOT_LITERAL_KIND: str = "literal" POLYGLOT_ARRAY_KIND: str = "array_func" POLYGLOT_STRUCTURED_KINDS: frozenset[str] = frozenset({"parse_json", "struct"}) POLYGLOT_FUNCTION_KIND: str = "function" -MISSING_TEMPLATE_VALUE_MESSAGE_PARTS: frozenset[str] = frozenset( - { - "references missing ENV variable", - "references unknown variable", - "references unknown CTX key", - } -) -MISSING_TEMPLATE_CONTEXT_MESSAGE_PART: str = "references CTX key" -MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART: str = "no value is available" -TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{([^{}]+)\}") MACRO_CALL_PATTERN: re.Pattern[str] = re.compile(r"@[A-Za-z_][A-Za-z0-9_]*\s*\(") SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN: re.Pattern[str] = re.compile( r"(?[A-Za-z_][A-Za-z0-9_]*)'" diff --git a/src/sqlbuild/compiler/compile/main/expand_template_data.py b/src/sqlbuild/compiler/compile/main/expand_template_data.py index ac93265bdf..cd173d3d2f 100644 --- a/src/sqlbuild/compiler/compile/main/expand_template_data.py +++ b/src/sqlbuild/compiler/compile/main/expand_template_data.py @@ -5,9 +5,6 @@ from sqlbuild.compiler.compile._helpers.render.templating import ( expand_template_data as _expand_template_data, ) -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage def expand_template_data( @@ -22,8 +19,6 @@ def expand_template_data( ) -> object: """Recursively expand template strings inside supported Python container values.""" - if native_stage_enabled(NativeStage.MODEL_CONFIG): - report_native_fallback(site=NativeFallbackSite.PYTHON_TEMPLATES, kind="api") return _expand_template_data( value=value, variables=variables, diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index cb9e238230..bc6f801e0b 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -75,11 +75,6 @@ class CompileContextKey(StrEnum): DESTINATION_QUALIFIED = "destination.qualified" -class TemplateNamespace(StrEnum): - ENV = "ENV" - CTX = "CTX" - - class CompiledResourceType(StrEnum): MODEL = "model" SOURCE = "source" diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 1f632c24ac..dd825586bd 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -70,10 +70,8 @@ class NativeFallbackSite(StrEnum): REFERENCE_SCAN = "reference_extraction.scan" CONFIG_BUILD = "model_config.build" CONFIG_HEADER_METADATA = "model_config.header_metadata" - CONFIG_TEMPLATES = "model_config.templates" CONFIG_PRESENCE_SCAN = "model_config.presence_scan" CONFIG_VALIDATORS = "model_config.validators" - PYTHON_TEMPLATES = "model_config.python_templates" AUDIT_RENDERING = "attachments.audit_rendering" FUNCTION_HEADER = "attachments.function_header" CURSOR_INTRINSIC_REJECTION = "attachments.cursor_intrinsic_rejection" diff --git a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py index 013d618892..405a2e575f 100644 --- a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py +++ b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py @@ -1,12 +1,10 @@ -"""Expand `${...}` templates in model config natively for the preview compiler engine.""" +"""Expand `${...}` templates natively, returning the value or Python's error with its reads.""" from __future__ import annotations import os -from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.model_config.constants import UNSUPPORTED_OUTCOME from sqlbuild.compiler.model_config.models import ( NativeTemplateExpansion, NativeTemplateRejection, @@ -21,26 +19,19 @@ def expand_native_config_templates( context_values: dict[str, str | None], flags: TemplateResolutionFlags, context_label: str, -) -> NativeTemplateExpansion | NativeTemplateRejection | str: - """Return the expansion, the error Python raises, or `unsupported` (Python must run).""" +) -> NativeTemplateExpansion | NativeTemplateRejection: + """Return the expanded value, or the error expansion stopped at; both carry their reads.""" - try: - outcome: tuple[object, list[tuple[str, str]]] | str = _native.expand_config_templates( - value, - (variables, os.environ, context_values), - ( - flags.allow_context, - flags.preserve_context_tokens, - flags.preserve_unknown_context, - context_label, - ), - ) - except TypeError: - return UNSUPPORTED_OUTCOME - if isinstance(outcome, str): - return outcome + outcome: tuple[object, list[tuple[str, str]]] = _native.expand_config_templates( + value, + (variables, os.environ, context_values), + ( + flags.allow_context, + flags.preserve_context_tokens, + flags.preserve_unknown_context, + context_label, + ), + ) if isinstance(outcome[0], _native.NativeConfigError): - return NativeTemplateRejection( - error=cast(_native.NativeConfigError, outcome[0]), reads=tuple(outcome[1]) - ) + return NativeTemplateRejection(error=outcome[0], reads=tuple(outcome[1])) return NativeTemplateExpansion(value=outcome[0], reads=tuple(outcome[1])) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py index 4d02acbe22..da7f8f4a2f 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py @@ -31,7 +31,6 @@ { "pair_seed_files", "render_attached_generic_audit", - "expand_config_templates", "scan_test_parameter_references", "omitted_ceremonial_select", "extract_sql_scenario_json", diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index 4841e40750..d9f326ba3d 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -35,18 +35,3 @@ class ModelConfigTierTestCase: description: str engine: str expected_native_calls: dict[str, int] - - -@dataclass(frozen=True) -class ConfigTemplateParityTestCase: - """Seeded config values whose templates expand natively and in Python.""" - - description: str - seed: int - count: int - allow_context: bool - preserve_context_tokens: bool - preserve_unknown_context: bool - expected_minimum_expanded: int - expected_minimum_rejected: int - expected_minimum_unsupported: int diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 0bcab3f19b..48806a06c3 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import random from collections.abc import Callable from dataclasses import dataclass, fields @@ -15,12 +14,10 @@ import sqlbuild._native as _native from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances -from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates from sqlbuild.compiler.compile._helpers.render.templating import ( contains_template_data, - expand_template_data, ) -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, MACRO_CALL_PATTERN +from sqlbuild.compiler.compile.constants import MACRO_CALL_PATTERN from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs @@ -28,7 +25,6 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.compiler.model_config.constants import UNSUPPORTED_OUTCOME from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( native_config_contains_macro_call, ) @@ -139,25 +135,6 @@ class HeaderMetadataParity: unsupported: int -@dataclass(frozen=True) -class ConfigTemplateParity: - """How native template expansion compared with Python's over one corpus.""" - - mismatches: list[tuple[object, object, object]] - expanded: int - rejected: int - unsupported: int - - -@dataclass(frozen=True) -class TemplateFlags: - """The resolver flags one expansion runs with.""" - - allow_context: bool - preserve_context_tokens: bool - preserve_unknown_context: bool - - @dataclass(frozen=True) class ConfigPresenceParity: """How the native presence scans compared with Python's over one corpus.""" @@ -239,125 +216,6 @@ def counted(*args: object) -> object: return counted -def generated_template_values(*, rng: random.Random, count: int) -> list[object]: - """Return seeded config values holding valid, invalid and unusual `${...}` templates.""" - - return [_template_value(rng=rng, depth=0) for _ in range(count)] - - -def config_template_parity(*, values: list[object], flags: TemplateFlags) -> ConfigTemplateParity: - """Compare native expansion, its reads and its rejections with Python's expansion.""" - - answered: list[object] = list( - compress( - values, - [ - _native_classification(value=value, flags=flags) != UNSUPPORTED_OUTCOME - for value in values - ], - ) - ) - outcomes: list[tuple[object, object, object]] = [ - (value, _python_expansion(value=value, flags=flags), _native_expansion(value, flags)) - for value in answered - ] - return ConfigTemplateParity( - mismatches=mismatches( - inputs=[value for value, _, _ in outcomes], - expected=[expected for _, expected, _ in outcomes], - actual=[actual for _, _, actual in outcomes], - ), - expanded=sum(not _is_error(actual) for _, _, actual in outcomes), - rejected=sum(_is_error(actual) for _, _, actual in outcomes), - unsupported=len(values) - len(outcomes), - ) - - -def _python_expansion(*, value: object, flags: TemplateFlags) -> object: - with COMPILE_INPUT_READS.recording() as reads: - try: - result: object = expand_template_data( - value=value, - variables=TEMPLATE_VARIABLES, - context_values=TEMPLATE_CONTEXT, - context_label="model config", - allow_context=flags.allow_context, - preserve_context_tokens=flags.preserve_context_tokens, - preserve_unknown_context=flags.preserve_unknown_context, - ) - except CompileInputError as error: - return _python_error_shape(error) - return (_expansion_shape(result), reads.environment_names, reads.read_run_id) - - -def _python_error_shape(error: Exception) -> tuple[object, ...]: - return (*error_shape(error)[:4], True) - - -def _is_error(outcome: object) -> bool: - return isinstance(outcome, tuple) and len(outcome) == len(error_shape(ValueError())) - - -def _native_classification(*, value: object, flags: TemplateFlags) -> object: - return _native.expand_config_templates( - value, - (TEMPLATE_VARIABLES, os.environ, TEMPLATE_CONTEXT), - ( - flags.allow_context, - flags.preserve_context_tokens, - flags.preserve_unknown_context, - "model config", - ), - ) - - -def _native_expansion(value: object, flags: TemplateFlags) -> object: - with COMPILE_INPUT_READS.recording() as reads: - try: - result: object = expand_config_templates( - value=value, - variables=TEMPLATE_VARIABLES, - context_values=TEMPLATE_CONTEXT, - context_label="model config", - allow_context=flags.allow_context, - preserve_context_tokens=flags.preserve_context_tokens, - preserve_unknown_context=flags.preserve_unknown_context, - native=True, - ) - except CompileInputError as error: - return error_shape(error) - return (_expansion_shape(result), reads.environment_names, reads.read_run_id) - - -def _expansion_shape(value: object) -> object: - return (repr(value), _type_tree(value)) - - -def _type_tree(value: object) -> object: - children: tuple[object, ...] = _CHILDREN.get(type(value), _no_children)(value) - return (type(value).__name__, tuple(_type_tree(item) for item in children)) - - -def _no_children(value: object) -> tuple[object, ...]: - del value - return () - - -def _mapping_children(value: object) -> tuple[object, ...]: - return tuple(cast(dict[object, object], value).values()) - - -def _sequence_children(value: object) -> tuple[object, ...]: - return tuple(cast(tuple[object, ...], value)) - - -_CHILDREN: dict[type, Callable[[object], tuple[object, ...]]] = { - dict: _mapping_children, - list: _sequence_children, - tuple: _sequence_children, -} - - def header_metadata_parity(*, headers: list[tuple[object, object]]) -> HeaderMetadataParity: """Compare the native parse or rejection with Python's wherever native answers.""" @@ -625,121 +483,6 @@ def _audit(*, rng: random.Random) -> object: )[0] -def _template_value(*, rng: random.Random, depth: int) -> object: - factories: tuple[Callable[[], object], ...] = ( - lambda: _template_text(rng=rng), - lambda: rng.choice(_TEXTS), - lambda: [_template_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))], - lambda: tuple(_template_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 2))), - lambda: { - rng.choice(("schema", "database", "${key}", 1)): _template_value( - rng=rng, depth=depth + 1 - ) - for _ in range(rng.randint(0, 3)) - }, - ) - return rng.choices(factories, weights=(60 + 1000 * (depth > 1), 10, 10, 5, 15))[0]() - - -def _template_text(*, rng: random.Random) -> str: - parts: list[str] = [ - rng.choices( - (f"${{{_template_expression(rng=rng, depth=0)}}}", rng.choice(_TEMPLATE_NOISE)), - weights=(4, 1), - )[0] - for _ in range(rng.randint(1, 3)) - ] - text: str = "".join(parts) - noise: str = rng.choices((rng.choice(_TEMPLATE_NOISE), ""), weights=(15, 85))[0] - position: int = rng.randrange(len(text) + 1) - return text[:position] + noise + text[position:] - - -def _template_expression(*, rng: random.Random, depth: int) -> str: - factories: tuple[Callable[[], str], ...] = ( - lambda: rng.choice(_TEMPLATE_REFERENCES), - lambda: rng.choice(_TEMPLATE_LITERALS), - lambda: "{}({})".format( - rng.choice(_TEMPLATE_FUNCTIONS), - rng.choice((",", ", ", " ,\t", "\x1f,")).join( - _template_expression(rng=rng, depth=depth + 1) - for _ in range(rng.choice((0, 1, 2, 2, 3, 3, 4))) - ), - ), - ) - return rng.choices(factories, weights=(5, 3, 4 - 3 * (depth > 2)))[0]() - - -TEMPLATE_VARIABLES: dict[str, object] = { - "flag": True, - "zero": 0, - "count": 12, - "big": 2**70, - "env": "prod", - "spaced": " FALSE ", - "ratio": 1.5, - "items": [1, "a"], - "mapping": {"a": 1}, - "none": None, - "unicode": "caf\u00e9", -} -TEMPLATE_CONTEXT: dict[str, str | None] = { - "run.id": "20261007T000000Z_abc", - "run.target": "prod", - "model.name": "orders", - "model.schema": None, -} -TEMPLATE_ENVIRONMENT: dict[str, str] = { - "SQB_TEMPLATE_SCHEMA": "analytics", - "SQB_TEMPLATE_EMPTY": "", - "SQB_TEMPLATE_ZERO": "0", -} -_TEMPLATE_REFERENCES: tuple[str, ...] = ( - *TEMPLATE_VARIABLES, - "missing", - "ENV:SQB_TEMPLATE_SCHEMA", - "ENV:SQB_TEMPLATE_EMPTY", - "ENV:SQB_TEMPLATE_ZERO", - "ENV:SQB_TEMPLATE_MISSING", - "CTX:run.id", - "CTX:run.target", - "CTX:model.name", - "CTX:model.schema", - "CTX:model.alias", - "OTHER:name", - "true", - "false", - "null", -) -_TEMPLATE_LITERALS: tuple[str, ...] = ( - "'prod'", - '"0"', - "''", - "' False '", - "'a\\'b'", - "'caf\u00e9'", - "'unterminated", - "'trailing\\", -) -_TEMPLATE_FUNCTIONS: tuple[str, ...] = ("if", "eq", "ne", "coalesce", "upper") -_TEMPLATE_NOISE: tuple[str, ...] = ( - "orders_", - "${", - "}", - "{", - "$", - "${}", - "(", - ")", - ",", - "\u00a0", - "\u2003", - "\x1c", - " ", - "caf\u00e9", -) - - def _config_value(*, rng: random.Random, depth: int) -> object: factories: tuple[Callable[[], object], ...] = ( lambda: rng.choice(_TEXTS), diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 7104829642..823e323941 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -9,21 +9,15 @@ from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( ConfigPresenceParityTestCase, - ConfigTemplateParityTestCase, HeaderMetadataParityTestCase, ModelConfigTierTestCase, ) from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( - TEMPLATE_ENVIRONMENT, ConfigPresenceParity, - ConfigTemplateParity, HeaderMetadataParity, - TemplateFlags, config_presence_parity, - config_template_parity, generated_config_values, generated_header_metadata, - generated_template_values, header_metadata_parity, model_config_engine_outcome, ) @@ -85,68 +79,6 @@ def test_given_generated_config_when_scanning_presence_then_native_matches_pytho ) == ([], True, True), (parity.present, parity.deferred) -@pytest.mark.parametrize( - "test_case", - [ - ConfigTemplateParityTestCase( - description="model config resolution preserving unknown context", - seed=20261009, - count=4000, - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - expected_minimum_expanded=800, - expected_minimum_rejected=400, - expected_minimum_unsupported=100, - ), - ConfigTemplateParityTestCase( - description="target resolution rejecting unknown context", - seed=20261010, - count=4000, - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=False, - expected_minimum_expanded=800, - expected_minimum_rejected=400, - expected_minimum_unsupported=100, - ), - ConfigTemplateParityTestCase( - description="context disallowed but preserved", - seed=20261011, - count=4000, - allow_context=False, - preserve_context_tokens=True, - preserve_unknown_context=False, - expected_minimum_expanded=800, - expected_minimum_rejected=400, - expected_minimum_unsupported=100, - ), - ], - ids=lambda case: case.description, -) -def test_given_generated_templates_when_expanding_then_native_matches_python_or_defers( - test_case: ConfigTemplateParityTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - for name, value in TEMPLATE_ENVIRONMENT.items(): - monkeypatch.setenv(name, value) - - parity: ConfigTemplateParity = config_template_parity( - values=generated_template_values(rng=random.Random(test_case.seed), count=test_case.count), - flags=TemplateFlags( - allow_context=test_case.allow_context, - preserve_context_tokens=test_case.preserve_context_tokens, - preserve_unknown_context=test_case.preserve_unknown_context, - ), - ) - - assert ( - parity.mismatches, - parity.expanded >= test_case.expected_minimum_expanded, - parity.rejected >= test_case.expected_minimum_rejected, - parity.unsupported >= test_case.expected_minimum_unsupported, - ) == ([], True, True, True), (parity.expanded, parity.rejected, parity.unsupported) - - @pytest.mark.parametrize( "test_case", [ @@ -155,7 +87,7 @@ def test_given_generated_templates_when_expanding_then_native_matches_python_or_ engine="python", expected_native_calls={ "parse_model_header_metadata": 0, - "expand_config_templates": 0, + "expand_config_templates": 14, "config_contains_template": 0, "config_contains_macro_call": 0, }, @@ -165,7 +97,7 @@ def test_given_generated_templates_when_expanding_then_native_matches_python_or_ engine="native", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 1, + "expand_config_templates": 8, "config_contains_template": 0, "config_contains_macro_call": 0, }, @@ -175,7 +107,7 @@ def test_given_generated_templates_when_expanding_then_native_matches_python_or_ engine="native-preview", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 1, + "expand_config_templates": 8, "config_contains_template": 0, "config_contains_macro_call": 0, }, From 3d5ef0f3ebd72eaa49c48a241906a1ffb803e155 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:30:19 +0100 Subject: [PATCH 03/40] feat(native): declaration files, scopes and @enum/@const scans run natively only; delete their Python fallbacks --- .../src/text/_helpers/alnum_tables.rs | 9 +- .../src/text/decimal_ranges/unicode_15_0_0.in | 68 +++ .../src/text/decimal_ranges/unicode_15_1_0.in | 68 +++ .../src/text/decimal_ranges/unicode_16_0_0.in | 75 ++++ .../src/text/main/is_python_decimal.rs | 12 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + .../src/text/main/python_text.rs | 7 +- crates/sqlbuild-core/src/text/models.rs | 2 + .../src/text/tests/test_python_compat.rs | 12 + .../src/text/tests/test_types.rs | 1 + .../_helpers/checks/python_values.rs | 58 ++- .../_helpers/checks/stops.rs | 3 +- .../_helpers/listing/collection_reads.rs | 6 +- .../_helpers/listing/collections.rs | 7 +- .../_helpers/parsing/audit_files.rs | 13 +- .../_helpers/parsing/constant_files.rs | 17 +- .../_helpers/parsing/enum_files.rs | 28 +- .../_helpers/parsing/hook_files.rs | 5 +- .../_helpers/parsing/schema_files.rs | 20 +- .../main/parse_declaration_text.rs | 7 +- .../tests/test_parse_declaration_text.rs | 13 +- crates/sqlbuild-discovery/src/models.rs | 2 - .../_helpers/discovery/declaration_files.rs | 1 - .../_helpers/discovery/project_files.rs | 1 - .../sqltext/declaration_references.rs | 17 +- .../reference_syntax.rs | 24 +- .../declaration_references/scanner.rs | 20 +- .../compiler/main/declaration_references.rs | 10 +- .../src/compiler/tests/helpers.rs | 6 +- .../tests/test_declaration_references.rs | 11 +- .../_helpers/corpus/failure_cases.py | 16 + scripts/python_char_table/constants.py | 2 +- src/sqlbuild/_native.pyi | 4 +- .../compile/_helpers/attachment/core.py | 46 +- .../_helpers/attachment/declaration_scope.py | 80 +--- .../compile/_helpers/render/declarations.py | 179 +------- .../compile/_helpers/render/sql_vars.py | 58 +-- .../discovery/_helpers/filesystem/core.py | 397 +----------------- .../_helpers/native/declaration_files.py | 27 +- .../discovery/_helpers/native/declarations.py | 10 - .../compiler/discovery/_helpers/sql/audits.py | 397 ------------------ .../discovery/_helpers/sql/declarations.py | 221 +--------- .../discovery/_helpers/sql/functions.py | 51 --- .../compiler/discovery/_helpers/sql/hooks.py | 127 ------ src/sqlbuild/compiler/discovery/constants.py | 3 - src/sqlbuild/compiler/discovery/types.py | 1 - src/sqlbuild/compiler/frontier/types.py | 5 - .../_scan_native_declaration_references.py | 13 +- .../scopes/classes/native_scope_index.py | 14 +- src/sqlbuild/compiler/scopes/exceptions.py | 11 + .../scopes/main/_open_native_scope_index.py | 30 +- ...ine-error-non-ascii-digit-bare-number.json | 19 + .../compiler/discovery/_test_types.py | 24 +- .../sqlbuild/compiler/discovery/helpers.py | 40 -- .../test_native_declaration_files.py | 81 +--- .../test_native_undecodable_paths.py | 75 ++-- .../compiler/model_loop/_test_types.py | 14 - .../sqlbuild/compiler/model_loop/helpers.py | 87 ---- .../test_native_declaration_references.py | 61 --- .../src/sqlbuild/compiler/scopes/helpers.py | 8 +- .../scopes/test_native_scope_parity.py | 6 +- .../compiler/discovery/_helpers/helpers.py | 55 ++- .../_helpers/test_function_header_keys.py | 31 +- .../discovery/_helpers/test_sql_audits.py | 43 +- .../_helpers/test_sql_declarations.py | 85 ++-- .../discovery/_helpers/test_sql_hooks.py | 22 +- .../_helpers/test_statement_header_keys.py | 23 +- 67 files changed, 788 insertions(+), 2102 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in create mode 100644 crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in create mode 100644 crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in create mode 100644 crates/sqlbuild-core/src/text/main/is_python_decimal.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/audits.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/functions.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py create mode 100644 tests/goldens/compiler/failure/engine-error-non-ascii-digit-bare-number.json delete mode 100644 tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs index 38b3a6b9a5..4c81042156 100644 --- a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs @@ -1,4 +1,4 @@ -//! The generated `str.isalnum()` and `str.isalpha()` tables, one per supported Unicode version. +//! The generated `str.isalnum()`, `str.isalpha()` and `str.isdecimal()` tables, per Unicode version. /// `(unicodedata.unidata_version, ranges)` for every table native text helpers can select. pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ @@ -13,3 +13,10 @@ pub(crate) const ALPHA_TABLES: [(&str, &[(u32, u32)]); 3] = [ ("15.1.0", include!("../alpha_ranges/unicode_15_1_0.in")), ("16.0.0", include!("../alpha_ranges/unicode_16_0_0.in")), ]; + +/// `(unicodedata.unidata_version, ranges)` of `str.isdecimal()` (`\d`), in the same order. +pub(crate) const DECIMAL_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../decimal_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../decimal_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../decimal_ranges/unicode_16_0_0.in")), +]; diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in new file mode 100644 index 0000000000..237a5d10d5 --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in @@ -0,0 +1,68 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 15.0.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in new file mode 100644 index 0000000000..2715a72404 --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in @@ -0,0 +1,68 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 15.1.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in new file mode 100644 index 0000000000..b6964c3b9c --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in @@ -0,0 +1,75 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 16.0.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x10D40, 0x10D49), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11BF0, 0x11BF9), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16130, 0x16139), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x16D70, 0x16D79), + (0x1CCF0, 0x1CCF9), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E5F1, 0x1E5FA), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/main/is_python_decimal.rs b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs new file mode 100644 index 0000000000..e24c260d67 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs @@ -0,0 +1,12 @@ +//! Python's `str.isdecimal()`, the regex `\d` of `str` patterns, from a generated table. + +use crate::text::main::is_python_alnum::in_ranges; +use crate::text::models::PythonText; + +/// Whether Python's `str.isdecimal()` is true for one character. +pub fn is_python_decimal(python: PythonText, character: char) -> bool { + if character.is_ascii() { + return character.is_ascii_digit(); + } + in_ranges(python.decimal_ranges, character) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index 3944a5830d..2b20809e36 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -2,6 +2,7 @@ pub mod close_matches; pub mod decode_python_text; pub mod is_python_alnum; pub mod is_python_alpha; +pub mod is_python_decimal; pub mod is_python_space; pub mod is_python_word; pub mod python_cleandoc; diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs index 1208a686b7..5253b9853c 100644 --- a/crates/sqlbuild-core/src/text/main/python_text.rs +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -1,6 +1,6 @@ //! Select the Python string semantics of one CPython release. -use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES}; +use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES}; use crate::text::models::{CleandocMargin, PythonText}; /// The semantics of one Python and its `unicodedata.unidata_version`, or `None` if unknown. @@ -18,9 +18,14 @@ pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option { + pub(crate) python: PythonText, + pub(crate) file_path: &'file str, +} + /// The Python type a header value projects to. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PythonType { @@ -19,12 +28,12 @@ pub(crate) enum PythonType { Other, } -/// The projected type; a bare word with non-ASCII text may be a Unicode number, so it defers. -pub(crate) fn python_type(value: &AuthoredValue) -> Result { +/// The projected type; a bare number written with non-ASCII digits is rejected. +pub(crate) fn python_type(value: &AuthoredValue, words: WordRules) -> Result { Ok(match value { AuthoredValue::Null => PythonType::None, AuthoredValue::Boolean(_) => PythonType::Bool, - AuthoredValue::BareWord(word) => word_type(word)?, + AuthoredValue::BareWord(word) => word_type(word, words)?, AuthoredValue::String(_) => PythonType::Str, AuthoredValue::List(_) => PythonType::List, AuthoredValue::Map(_) => PythonType::Dict, @@ -38,17 +47,23 @@ pub(crate) fn python_type(value: &AuthoredValue) -> Result Result, ParseStop> { +pub(crate) fn python_str<'value>( + value: &'value AuthoredValue, + words: WordRules, +) -> Result, ParseStop> { Ok(match value { AuthoredValue::String(text) => Some(text), - AuthoredValue::BareWord(word) if word_type(word)? == PythonType::Str => Some(word), + AuthoredValue::BareWord(word) if word_type(word, words)? == PythonType::Str => Some(word), _ => None, }) } /// `isinstance(value, str) and value.strip()`. -pub(crate) fn non_empty_str(value: &AuthoredValue) -> Result, ParseStop> { - Ok(python_str(value)?.filter(|text| !python_strip(text).is_empty())) +pub(crate) fn non_empty_str<'value>( + value: &'value AuthoredValue, + words: WordRules, +) -> Result, ParseStop> { + Ok(python_str(value, words)?.filter(|text| !python_strip(text).is_empty())) } /// Python's `dict.get(key)`. @@ -89,9 +104,22 @@ pub(crate) fn failure(kind: FailureKind, message: String) -> ParseStop { } /// `_parse_word_value`: `true`, `false` and `null` are parsed as values; numbers become numbers. -fn word_type(word: &str) -> Result { +fn word_type(word: &str, words: WordRules) -> Result { if !word.is_ascii() { - return Err(ParseStop::Deferred); + if is_unicode_number(word, words.python) { + return Err(ParseStop::Failed(DiscoveryFailure { + kind: FailureKind::Declaration, + message: format!( + "{} has the bare number '{word}', written with non-ASCII digits", + words.file_path + ), + help: Some(format!( + "Quote it to keep it as text (\"{word}\"), or write the number with ASCII \ + digits 0-9" + )), + })); + } + return Ok(PythonType::Str); } Ok(match word { "true" | "false" => PythonType::Bool, @@ -120,3 +148,15 @@ fn is_float(word: &str) -> bool { let digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit()); digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty()) } + +/// Whether Python's `^[+-]?\d+$` or float pattern, with Unicode `\d`, matches a non-ASCII word. +fn is_unicode_number(word: &str, python: PythonText) -> bool { + let digits = |part: &str| part.chars().all(|character| is_python_decimal(python, character)); + let unsigned = unsigned(word); + if !unsigned.is_empty() && digits(unsigned) { + return true; + } + unsigned.split_once('.').is_some_and(|(whole, fraction)| { + digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty()) + }) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs index 3ce19f339b..b08389b284 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs @@ -2,11 +2,10 @@ use crate::models::DiscoveryFailure; -/// The first failure Python raises for the file, or a file only the Python parser reproduces. +/// The first failure Python raises for the file. #[derive(Clone, Debug, PartialEq)] pub(crate) enum ParseStop { Failed(DiscoveryFailure), - Deferred, } impl From for ParseStop { diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs index 0c6eceefd3..b41b9b9674 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs @@ -37,14 +37,14 @@ pub(crate) fn read_collection( CollectionKind::Enums => DeclarationCollection::Enums( declaration_files(source, DeclarationKind::Enum, isolate_kind).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_enum_file(file_path, contents) + parse_enum_file(file_path, contents, options.python) }) }), ), CollectionKind::Constants => DeclarationCollection::Constants( declaration_files(source, DeclarationKind::Constant, isolate_kind).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_constant_file(file_path, contents) + parse_constant_file(file_path, contents, options.python) }) }), ), @@ -58,7 +58,7 @@ pub(crate) fn read_collection( CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas( named_declaration_files(source, &[NamedKind::Schema], false).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_schema_file(file_path, contents) + parse_schema_file(file_path, contents, options.python) }) }), ), diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs index 7d51f06e03..c21c648495 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs @@ -1,6 +1,6 @@ //! The files of each collection in Python's discovery order, read and parsed in parallel. -use crate::_helpers::reading::read_authored_text; +use crate::_helpers::reading::{read_authored_text, undecodable_path_failure}; use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::listing::collection_reads::CollectionSource; @@ -258,14 +258,13 @@ fn read_one( relative_path: &str, parse: &impl Fn(&str, &str, String) -> Result, ) -> FileOutcome { - if tree.is_undecodable(relative_path) { - return FileOutcome::Deferred; + if let Some(failure) = undecodable_path_failure(root, tree, relative_path) { + return FileOutcome::Failed(failure); } match read_authored_text(&tree.absolute(relative_path)) { Ok(contents) => match parse(&root.display_path(relative_path), relative_path, contents) { Ok(parsed) => FileOutcome::Parsed(parsed), Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), - Err(ParseStop::Deferred) => FileOutcome::Deferred, }, Err(failure) => FileOutcome::Unreadable(failure), } diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs index ad089ae598..0b45e63cde 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs @@ -2,12 +2,14 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ + WordRules, PythonType, failure, get, non_empty_str, python_str, python_type, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::audit_text::{AuditText, chars, text}; use crate::declaration_files::models::{AuditBlock, AuditFile, DeclarationFileOptions}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::main::python_strip::python_strip; @@ -113,7 +115,7 @@ impl AuditParse<'_> { .filter(|character| **character == '\n') .count(); let header_values: Vec<(String, AuthoredValue)> = self.header(&header, header_line)?; - let evaluation_mode: &'static str = evaluation_mode(&header_values, file_path)?; + let evaluation_mode: &'static str = evaluation_mode(&header_values, file_path, self.options.python)?; let body: &[char] = &block_chars[body_start..]; let mut measure_sql: Option = None; let mut evidence_sql: Option = None; @@ -143,7 +145,7 @@ impl AuditParse<'_> { sql_body }; let name: Option = match get(&header_values, "name") { - Some(value) => python_str(value)?.map(str::to_owned), + Some(value) => python_str(value, WordRules { python: self.options.python, file_path })?.map(str::to_owned), None => None, }; Ok(AuditBlock { @@ -177,7 +179,7 @@ impl AuditParse<'_> { )?; for key in STRING_KEYS { if let Some(value) = get(&values, key) - && non_empty_str(value)?.is_none() + && non_empty_str(value, WordRules { python: self.options.python, file_path })?.is_none() { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a non-empty string" @@ -186,7 +188,7 @@ impl AuditParse<'_> { } for key in BOOLEAN_KEYS { if let Some(value) = get(&values, key) - && python_type(value)? != PythonType::Bool + && python_type(value, WordRules { python: self.options.python, file_path })? != PythonType::Bool { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a boolean" @@ -343,9 +345,10 @@ fn keyword_at_slice_start( fn evaluation_mode( header_values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result<&'static str, ParseStop> { let mode: &str = match get(header_values, "evaluation") { - Some(value) => python_str(value)?.unwrap_or_default(), + Some(value) => python_str(value, WordRules { python, file_path })?.unwrap_or_default(), None => VIOLATIONS, }; let mode: &'static str = match mode { diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs index d6e61a5d28..cebb8abc7f 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs @@ -1,6 +1,7 @@ //! Python's `parse_constant_declaration_file` up to value normalisation, which Python performs. use crate::declaration_files::_helpers::checks::python_values::{ + WordRules, failure, get, python_str, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; @@ -8,6 +9,7 @@ use crate::declaration_files::_helpers::parsing::declaration_headers::declaratio use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; use crate::declaration_files::models::{ConstantDeclaration, ConstantFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const CONSTANT_KEYS: [&str; 4] = ["name", "value", "type", "render_as"]; @@ -17,18 +19,18 @@ const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; pub(crate) fn parse_constant_file( file_path: &str, contents: String, + python: PythonText, ) -> Result { let headers = declaration_headers(&contents, file_path, "CONSTANT")?; let mut declarations: Vec = Vec::with_capacity(headers.len()); let mut failure: Option<_> = None; for header in headers { - match parse_constant(&header.values, file_path) { + match parse_constant(&header.values, file_path, python) { Ok(declaration) => declarations.push(declaration), Err(ParseStop::Failed(stopped)) => { failure = Some(stopped); break; } - Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), } } Ok(ConstantFile { @@ -45,6 +47,7 @@ fn declaration(message: String) -> ParseStop { fn parse_constant( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &CONSTANT_KEYS); if !unknown.is_empty() { @@ -53,7 +56,7 @@ fn parse_constant( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "constant")?; + let name: String = declaration_name(get(values, "name"), file_path, "constant", python)?; let mut value: Option<&AuthoredValue> = get(values, "value"); let mut explicit_type: Option<&AuthoredValue> = get(values, "type"); let mut render_as: Option<&AuthoredValue> = get(values, "render_as"); @@ -80,12 +83,11 @@ fn parse_constant( ))); }; let explicit_type: Option = - constant_option(explicit_type, file_path, &format!("constant '{name}' type"))?; + constant_option(explicit_type, file_path, &format!("constant '{name}' type"), python)?; let render_as: Option = constant_option( render_as, file_path, - &format!("constant '{name}' render_as"), - )?; + &format!("constant '{name}' render_as"), python)?; if let Some(rendering) = &render_as && !COLLECTION_RENDERINGS.contains(&rendering.as_str()) { @@ -106,11 +108,12 @@ fn constant_option( raw_value: Option<&AuthoredValue>, file_path: &str, label: &str, + python: PythonText, ) -> Result, ParseStop> { let Some(raw_value) = raw_value else { return Ok(None); }; - match python_str(raw_value)? { + match python_str(raw_value, WordRules { python, file_path })? { Some(text) if !text.is_empty() => Ok(Some(text.to_owned())), _ => Err(declaration(format!( "{file_path} {label} must be an identifier" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs index f19e8e8e2a..5e9cf8d3d4 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs @@ -2,22 +2,28 @@ use crate::declaration_files::_helpers::checks::identities::validate_public_identity; use crate::declaration_files::_helpers::checks::python_values::{ + WordRules, PythonType, failure, get, is_identifier, python_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; use crate::declaration_files::models::{EnumDeclaration, EnumFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const ENUM_KEYS: [&str; 2] = ["name", "members"]; const VARCHAR_TYPE: &str = "VARCHAR"; const INTEGER_TYPE: &str = "INTEGER"; -pub(crate) fn parse_enum_file(file_path: &str, contents: String) -> Result { +pub(crate) fn parse_enum_file( + file_path: &str, + contents: String, + python: PythonText, +) -> Result { let declarations: Vec = declaration_headers(&contents, file_path, "ENUM")? .into_iter() - .map(|header| parse_enum(&header.values, file_path)) + .map(|header| parse_enum(&header.values, file_path, python)) .collect::>()?; Ok(EnumFile { contents, @@ -32,6 +38,7 @@ fn declaration(message: String) -> ParseStop { fn parse_enum( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &ENUM_KEYS); if !unknown.is_empty() { @@ -40,10 +47,10 @@ fn parse_enum( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "enum")?; + let name: String = declaration_name(get(values, "name"), file_path, "enum", python)?; let members: Vec<(String, AuthoredValue)> = match get(values, "members") { - Some(AuthoredValue::List(items)) => shorthand_members(items, file_path, &name)?, - Some(AuthoredValue::Map(items)) => explicit_members(items, file_path, &name)?, + Some(AuthoredValue::List(items)) => shorthand_members(items, file_path, &name, python)?, + Some(AuthoredValue::Map(items)) => explicit_members(items, file_path, &name, python)?, _ => { return Err(declaration(format!( "{file_path} enum '{name}' members must use [...] or (...)" @@ -65,7 +72,7 @@ fn parse_enum( } let mut types: Vec = Vec::with_capacity(members.len()); for (_, value) in &members { - types.push(python_type(value)?); + types.push(python_type(value, WordRules { python, file_path })?); } if types.iter().any(|value_type| *value_type != types[0]) { return Err(declaration(format!( @@ -88,9 +95,10 @@ pub(crate) fn declaration_name( raw_name: Option<&AuthoredValue>, file_path: &str, kind: &str, + python: PythonText, ) -> Result { let name: Option<&str> = match raw_name { - Some(value) => python_str(value)?, + Some(value) => python_str(value, WordRules { python, file_path })?, None => None, }; let Some(name) = name.filter(|name| is_identifier(name)) else { @@ -106,10 +114,11 @@ fn shorthand_members( items: &[AuthoredValue], file_path: &str, enum_name: &str, + python: PythonText, ) -> Result, ParseStop> { let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); for item in items { - let Some(member) = python_str(item)?.filter(|member| is_identifier(member)) else { + let Some(member) = python_str(item, WordRules { python, file_path })?.filter(|member| is_identifier(member)) else { return Err(declaration(format!( "{file_path} enum '{enum_name}' shorthand members must be identifiers" ))); @@ -128,6 +137,7 @@ fn explicit_members( items: &[(String, AuthoredValue)], file_path: &str, enum_name: &str, + python: PythonText, ) -> Result, ParseStop> { let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); for (member, value) in items { @@ -136,7 +146,7 @@ fn explicit_members( "{file_path} enum '{enum_name}' member name must be a SQL identifier" ))); } - if !matches!(python_type(value)?, PythonType::Str | PythonType::Int) { + if !matches!(python_type(value, WordRules { python, file_path })?, PythonType::Str | PythonType::Int) { return Err(declaration(format!( "{file_path} enum '{enum_name}' member '{member}' value must be a string or \ integer" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs index 05b1c6dbc0..b52ec5bf9b 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs @@ -2,6 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ + WordRules, failure, get, non_empty_str, python_str, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; @@ -41,12 +42,12 @@ pub(crate) fn parse_hook_file( )?; let description: Option = match get(&header_values, DESCRIPTION_KEY) { Some(value) => { - if non_empty_str(value)?.is_none() { + if non_empty_str(value, WordRules { python: options.python, file_path })?.is_none() { return Err(hook_failure(format!( "HOOK() description in '{file_path}' must be a non-empty string" ))); } - python_str(value)?.map(str::to_owned) + python_str(value, WordRules { python: options.python, file_path })?.map(str::to_owned) } None => None, }; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs index 8e1054486f..e632f288f3 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs @@ -2,6 +2,7 @@ use crate::_helpers::locations::header_column_locations; use crate::declaration_files::_helpers::checks::python_values::{ + WordRules, PythonType, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; @@ -9,6 +10,7 @@ use crate::declaration_files::_helpers::parsing::declaration_headers::declaratio use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; use crate::declaration_files::models::{SchemaDeclaration, SchemaFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const SCHEMA_KEYS: [&str; 4] = ["name", "description", "extends", "columns"]; @@ -16,12 +18,13 @@ const SCHEMA_KEYS: [&str; 4] = ["name", "description", "extends", "columns"]; pub(crate) fn parse_schema_file( file_path: &str, contents: String, + python: PythonText, ) -> Result { let headers = declaration_headers(&contents, file_path, "SCHEMA")?; let mut declarations: Vec = Vec::with_capacity(headers.len()); let mut failure: Option<_> = None; for header in headers { - match parse_schema(&header.values, file_path) { + match parse_schema(&header.values, file_path, python) { Ok((name, description, extends)) => declarations.push(SchemaDeclaration { name, description, @@ -37,7 +40,6 @@ pub(crate) fn parse_schema_file( failure = Some(stopped); break; } - Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), } } Ok(SchemaFile { @@ -56,6 +58,7 @@ type SchemaFacts = (String, Option, Option); fn parse_schema( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &SCHEMA_KEYS); if !unknown.is_empty() { @@ -64,17 +67,15 @@ fn parse_schema( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "schema")?; + let name: String = declaration_name(get(values, "name"), file_path, "schema", python)?; let description: Option = optional_string( get(values, "description"), file_path, - &format!("schema '{name}' description"), - )?; + &format!("schema '{name}' description"), python)?; let extends: Option = optional_string( get(values, "extends"), file_path, - &format!("schema '{name}' extends"), - )?; + &format!("schema '{name}' extends"), python)?; if let Some(parent) = &extends && !is_identifier(parent) { @@ -90,14 +91,15 @@ fn optional_string( raw_value: Option<&AuthoredValue>, file_path: &str, label: &str, + python: PythonText, ) -> Result, ParseStop> { let Some(raw_value) = raw_value else { return Ok(None); }; - if python_type(raw_value)? == PythonType::None { + if python_type(raw_value, WordRules { python, file_path })? == PythonType::None { return Ok(None); } - match non_empty_str(raw_value)? { + match non_empty_str(raw_value, WordRules { python, file_path })? { Some(text) => Ok(Some(text.to_owned())), None => Err(declaration(format!( "{file_path} {label} must be a non-empty string" diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs index 749733c062..67c0936bfd 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs @@ -31,13 +31,13 @@ fn parse_text( ) -> Option> { let parsed: Result = match kind { CollectionKind::Enums => { - parse_enum_file(file_path, contents).map(ParsedDeclarationText::Enum) + parse_enum_file(file_path, contents, options.python).map(ParsedDeclarationText::Enum) } CollectionKind::Constants => { - parse_constant_file(file_path, contents).map(ParsedDeclarationText::Constant) + parse_constant_file(file_path, contents, options.python).map(ParsedDeclarationText::Constant) } CollectionKind::ModelSchemas => { - parse_schema_file(file_path, contents).map(ParsedDeclarationText::ModelSchema) + parse_schema_file(file_path, contents, options.python).map(ParsedDeclarationText::ModelSchema) } CollectionKind::SqlFunctions => parse_function_file(file_path, contents, options) .map(ParsedDeclarationText::SqlFunction), @@ -51,6 +51,5 @@ fn parse_text( Some(match parsed { Ok(parsed) => FileOutcome::Parsed(parsed), Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), - Err(ParseStop::Deferred) => FileOutcome::Deferred, }) } diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs index 5f38e8839b..27090fada5 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs @@ -44,10 +44,19 @@ fn given_declaration_files_when_parsing_then_values_and_failures_match_python() ], }, DeclarationTextTestCase { - description: "a non-ASCII bare word may be a Unicode number", + description: "a bare number in non-ASCII digits is rejected with help", kind: CollectionKind::Enums, contents: "ENUM (name status, members (A \u{661}));", - expected_fragments: &["Some(Deferred)"], + expected_fragments: &[ + "/project/declarations/orders.sql has the bare number '\u{661}', written with non-ASCII digits", + "Quote it to keep it as text", + ], + }, + DeclarationTextTestCase { + description: "a non-ASCII bare word that is not a number is text", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members (A münchen, B \u{2167}));", + expected_fragments: &["Parsed", "münchen"], }, DeclarationTextTestCase { description: "a camel-case name suggests snake case", diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 3f8be1e5de..b38c79864f 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -84,8 +84,6 @@ pub enum FileOutcome { Failed(DiscoveryFailure), /// The file could not be read or decoded as Python's `read_text` would. Unreadable(ReadFailure), - /// Native parsing cannot reproduce Python for this file; the Python parser reads it. - Deferred, } /// One discovered file, by `/`-separated path relative to the project directory. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs index 5a961a34d8..9c02672b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_files.rs @@ -214,7 +214,6 @@ fn outcome_object( FileOutcome::Parsed(value) => parsed(py, value), FileOutcome::Failed(failure) => failure_object(py, failure.clone()), FileOutcome::Unreadable(failure) => read_object(py, failure.clone()), - FileOutcome::Deferred => tuple_object(py, vec![object(py, "defer")?]), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index d851204dd3..b1f14b5875 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -152,7 +152,6 @@ fn outcome_object( FileOutcome::Parsed(value) => parsed(py, value), FileOutcome::Unreadable(failure) => read_object(py, failure), FileOutcome::Failed(failure) => failure_object(py, failure), - FileOutcome::Deferred => tuple_object(py, vec![object(py, "defer")?]), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index 7580cc5c9d..6313a42b6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -1,6 +1,8 @@ //! `@enum`/`@const` reference offsets scanned natively for the preview model loop. +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use sqlbuild_core::text::main::python_text::python_text; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ @@ -16,16 +18,21 @@ type ReferenceRow = (u8, String, Option, usize, usize); /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. type ScanRow = (Vec, Option); -/// Return each SQL string's references and stopping error, or `None` where Python must expand it. +/// Return each SQL string's references and stopping error under this Python's `\\b`. #[pyfunction] fn scan_sql_declaration_references( py: Python<'_>, sqls: Vec, -) -> PyResult>> { - let scanned: Vec> = py - .compiler_detach(|| Ok(scan_declaration_references(&sqls))) + python_version: (u8, u8), + unicode_version: &str, +) -> PyResult> { + let python = python_text(python_version, unicode_version).ok_or_else(|| { + PyRuntimeError::new_err("declaration scanning needs a supported Python (see D017)") + })?; + let scanned: Vec = py + .compiler_detach(|| Ok(scan_declaration_references(python, &sqls))) .map_err(compiler_error)?; - Ok(scanned.into_iter().map(|scan| scan.map(scan_row)).collect()) + Ok(scanned.into_iter().map(scan_row).collect()) } fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs index 0f4803a502..079401bff8 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs @@ -1,6 +1,8 @@ //! Python's `@enum("name").MEMBER` and `@const("name")` patterns at one reference start. +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; use crate::compiler::models::DeclarationReferenceKind; @@ -19,12 +21,10 @@ pub(crate) enum ReferenceSyntax<'sql> { Matched(MatchedReference<'sql>), /// A reference start whose full pattern does not match: Python raises an invalid reference. Malformed(DeclarationReferenceKind), - /// A non-ASCII character after the keyword, whose word boundary only Python decides. - Deferred, } /// Match Python's `@(enum|const)\b` start and the full reference pattern at `start`. -pub(crate) fn match_reference(sql: &str, start: usize) -> ReferenceSyntax<'_> { +pub(crate) fn match_reference(python: PythonText, sql: &str, start: usize) -> ReferenceSyntax<'_> { let bytes: &[u8] = sql.as_bytes(); let (kind, keyword_end) = if bytes[start..].starts_with(b"@enum") { (DeclarationReferenceKind::Enum, start + 5) @@ -33,18 +33,12 @@ pub(crate) fn match_reference(sql: &str, start: usize) -> ReferenceSyntax<'_> { } else { return ReferenceSyntax::NotReference; }; - match bytes.get(keyword_end) { - Some(byte) - if !byte.is_ascii() - && !sql[keyword_end..] - .chars() - .next() - .is_some_and(is_python_space) => - { - return ReferenceSyntax::Deferred; - } - Some(byte) if is_word(*byte) => return ReferenceSyntax::NotReference, - _ => {} + if sql[keyword_end..] + .chars() + .next() + .is_some_and(|next| next == '_' || is_python_alnum(python, next)) + { + return ReferenceSyntax::NotReference; } match_arguments(sql, kind, keyword_end) .map_or(ReferenceSyntax::Malformed(kind), ReferenceSyntax::Matched) diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs index 8fa714ab15..4648692a08 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs @@ -6,6 +6,7 @@ use crate::compiler::_helpers::declaration_references::reference_syntax::{ use crate::compiler::models::{ DeclarationReference, DeclarationReferenceScan, DeclarationReferenceStop, }; +use sqlbuild_core::text::models::PythonText; const SPECIAL_BYTES: &[u8] = b"'\"`$@/-"; @@ -14,32 +15,30 @@ enum NextStart { Reference(usize), End, Stop(DeclarationReferenceStop), - Deferred, } -/// Python's references in `sql` (code points) and its stopping error; None defers to Python. -pub(crate) fn scan_references(sql: &str) -> Option { +/// Python's references in `sql` (code points) and its stopping error. +pub(crate) fn scan_references(python: PythonText, sql: &str) -> DeclarationReferenceScan { let mut references: Vec = Vec::new(); let mut offsets: CharOffsets = CharOffsets::default(); let mut cursor: usize = 0; let mut stop: Option = None; while cursor < sql.len() { - let start: usize = match next_reference_start(sql, cursor) { + let start: usize = match next_reference_start(python, sql, cursor) { NextStart::Reference(start) => start, NextStart::End => break, NextStart::Stop(found) => { stop = Some(found); break; } - NextStart::Deferred => return None, }; - let matched = match match_reference(sql, start) { + let matched = match match_reference(python, sql, start) { ReferenceSyntax::Matched(matched) => matched, ReferenceSyntax::Malformed(kind) => { stop = Some(DeclarationReferenceStop::Malformed(kind)); break; } - ReferenceSyntax::NotReference | ReferenceSyntax::Deferred => return None, + ReferenceSyntax::NotReference => unreachable!("the walk stops only at a reference"), }; let start_char: usize = offsets.advance(sql, start); let end_char: usize = offsets.advance(sql, matched.end); @@ -52,11 +51,11 @@ pub(crate) fn scan_references(sql: &str) -> Option { }); cursor = matched.end; } - Some(DeclarationReferenceScan { references, stop }) + DeclarationReferenceScan { references, stop } } /// The next reference start at or after `start`, as Python's walk finds it. -fn next_reference_start(sql: &str, start: usize) -> NextStart { +fn next_reference_start(python: PythonText, sql: &str, start: usize) -> NextStart { let rest: &str = &sql[start..]; if !rest.contains("@enum") && !rest.contains("@const") { return NextStart::End; @@ -82,12 +81,11 @@ fn next_reference_start(sql: &str, start: usize) -> NextStart { Some(close) => Some(index + 2 + close + 2), None => return NextStart::Stop(DeclarationReferenceStop::UnclosedBlockComment), }, - b'@' => match match_reference(sql, index) { + b'@' => match match_reference(python, sql, index) { ReferenceSyntax::NotReference => Some(index + 1), ReferenceSyntax::Matched(_) | ReferenceSyntax::Malformed(_) => { return NextStart::Reference(index); } - ReferenceSyntax::Deferred => return NextStart::Deferred, }, _ => Some(index + 1), }; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs index fa87a2efdc..6880c5aa0c 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs @@ -4,9 +4,13 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use crate::compiler::_helpers::declaration_references::scanner::scan_references; use crate::compiler::models::DeclarationReferenceScan; +use sqlbuild_core::text::models::PythonText; -/// Each SQL string's references and stopping error, or None where Python must expand it. +/// Each SQL string's references and stopping error, with `python`'s word boundaries. #[must_use] -pub fn scan_declaration_references(sqls: &[String]) -> Vec> { - sqls.par_iter().map(|sql| scan_references(sql)).collect() +pub fn scan_declaration_references( + python: PythonText, + sqls: &[String], +) -> Vec { + sqls.par_iter().map(|sql| scan_references(python, sql)).collect() } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index c833ee1d5e..5ef7cf4b4e 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -189,9 +189,9 @@ pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { /// Spell a scan as `kind:name[.member]@start..end, ...` plus ` | stop:` when it stops. pub(crate) fn scanned_references(sql: &str) -> Option { - let scan: DeclarationReferenceScan = scan_declaration_references(&[sql.to_owned()]) - .pop() - .flatten()?; + let python = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); + let scan: DeclarationReferenceScan = + scan_declaration_references(python, &[sql.to_owned()]).pop()?; let references: String = scan .references .iter() diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs index 97ea5606aa..8f09c181d9 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs @@ -55,9 +55,14 @@ fn given_sql_when_scanning_declaration_references_then_python_offsets_are_return expected_references: Some("enum:order_status.PLACED@7..40"), }, DeclarationReferenceScanTestCase { - description: "a non-ASCII character after the keyword defers its word boundary to Python", - sql: "SELECT @consté", - expected_references: None, + description: "a non-ASCII letter or number after the keyword continues the word", + sql: "SELECT @consté, @enum\u{2167}, @const(\"a\")", + expected_references: Some("const:a@24..35"), + }, + DeclarationReferenceScanTestCase { + description: "a non-ASCII symbol after the keyword ends the word, so the reference is malformed", + sql: "SELECT @const\u{2192}", + expected_references: Some(" | stop:invalid const"), }, DeclarationReferenceScanTestCase { description: "an unclosed block comment before a reference stops the walk", diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index ed800bcd5f..e5f5ac19e0 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -626,6 +626,22 @@ def engine_error_cases() -> tuple[FailureCase, ...]: "(column names are case-insensitive)" ), ), + failure_case( + name="engine-error-non-ascii-digit-bare-number", + expected_code="D013", + files={ + "enums/order_priority.sql": ( + "ENUM (\n name order_priority,\n members (LOW 1, HIGH \u0663),\n);\n" + ) + }, + expected_message=( + "/enums/order_priority.sql has the bare number '\u0663', written with " + "non-ASCII digits" + ), + expected_help=( + 'Quote it to keep it as text ("\u0663"), or write the number with ASCII digits 0-9' + ), + ), failure_case( name="engine-error-week-date-cursor-start", expected_code="P001", diff --git a/scripts/python_char_table/constants.py b/scripts/python_char_table/constants.py index a378a3dbb6..84818e990a 100644 --- a/scripts/python_char_table/constants.py +++ b/scripts/python_char_table/constants.py @@ -6,4 +6,4 @@ TABLE_FILE_PREFIX: str = "unicode_" TABLE_FILE_SUFFIX: str = ".in" MAX_CODE_POINT: int = 0x10FFFF -TABLE_METHODS: tuple[str, ...] = ("isalnum", "isalpha") +TABLE_METHODS: tuple[str, ...] = ("isalnum", "isalpha", "isdecimal") diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index da1b80420a..8f235cefdc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -300,8 +300,8 @@ def render_attached_generic_audit( policies: dict[str, object], ) -> tuple[str | None, str, str | None, str, str, str | None] | None: ... def scan_sql_declaration_references( - sqls: list[str], -) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None] | None]: ... + sqls: list[str], python_version: tuple[int, int], unicode_version: str +) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... def interpolate_sql_batch( sqls: list[tuple[str, str]], sources: tuple[ diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 0dc4ba8f0c..5df4244686 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -72,7 +72,6 @@ ) from sqlbuild.compiler.compile._helpers.render.declarations import ( build_model_declaration_indexes, - expand_declaration_references_result, expand_scanned_declaration_references, resolve_declaration_context, resolve_enum_contract_columns, @@ -286,7 +285,7 @@ class _ModelInputLoop: reusable_config_cache: _ReusableModelConfigCache declaration_cache: _VisibleModelDeclarationCache native_header_metadata: dict[Path, NativeHeaderMetadata] - native_declaration_references: dict[Path, NativeDeclarationScan | None] + native_declaration_references: dict[Path, NativeDeclarationScan] @dataclass(frozen=True) @@ -550,14 +549,10 @@ def _build_model_inputs( strict=True, ) ) - prepared_files: tuple[DiscoveredSqlModelFile, ...] = ( - tuple( - model_file - for model_file in render_files - if interpolations[model_file.file_path].error is None - ) - if native_stage_enabled(NativeStage.MODEL_LOOP) - else () + prepared_files: tuple[DiscoveredSqlModelFile, ...] = tuple( + model_file + for model_file in render_files + if interpolations[model_file.file_path].error is None ) report_native_answer( stage=NativeStage.MODEL_LOOP, kind="variable_substitutions", units=len(prepared_files) @@ -730,30 +725,15 @@ def _build_model_input( inaccessible_macros=declarations.inaccessible_macros, consumer=model_identity, ) - declaration_expansion: DeclarationExpansionResult | None = ( - expand_scanned_declaration_references( - sql=var_substituted_sql, - references=loop.native_declaration_references.get(model_file.file_path), - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - if native_stage_enabled(NativeStage.MODEL_LOOP) - else None + declaration_expansion: DeclarationExpansionResult = expand_scanned_declaration_references( + sql=var_substituted_sql, + references=loop.native_declaration_references[model_file.file_path], + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, ) - if declaration_expansion is not None and native_stage_enabled(NativeStage.MODEL_LOOP): - report_native_answer(stage=NativeStage.MODEL_LOOP, kind="declaration_expansions") - if declaration_expansion is None: - if native_stage_enabled(NativeStage.MODEL_LOOP): - report_native_fallback(site=NativeFallbackSite.DECLARATION_REFERENCES, kind="scan") - declaration_expansion = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) + report_native_answer(stage=NativeStage.MODEL_LOOP, kind="declaration_expansions") declaration_expanded_sql: str = declaration_expansion.sql macro_expansion: MacroExpansionResult = expand_sql_macros_result( sql=declaration_expanded_sql, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 768429f794..c835e3f5a1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -4,12 +4,10 @@ import inspect from collections.abc import Mapping -from dataclasses import replace from pathlib import Path from types import CodeType from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( - build_scope_relationship_grants, extract_scope_relationship_facts, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( @@ -24,20 +22,15 @@ from sqlbuild.compiler.compile.models import ( DeclarationScopeBuild, LoadedMacro, - ScopeRelationshipBuild, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.models import RelationshipFact, ScopeIndex, ScopeLookup -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -58,55 +51,11 @@ def build_declaration_scope( validate_authored_scenario_cte_names( scenario_files=discovered_inputs.scenario_files, syntax=sql_lexical_syntax ) - if native_stage_enabled(NativeStage.DECLARATION_SCOPES): - native_scope: DeclarationScopeBuild | None = _build_native_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, - ) - if native_scope is not None: - return native_scope - index: ScopeIndex = build_scope_index( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - try: - validate_scope_index(index=index) - except ScopeValidationError as error: - raise CompileInputError(str(error)) from error - has_scoped_relationship_declarations: bool = any( - declaration.scope is not ScopeKind.GLOBAL - and declaration.identity.kind - in {DeclarationKind.ENUM, DeclarationKind.CONSTANT, DeclarationKind.MACRO} - for declaration in index.declarations - ) - relationships: ScopeRelationshipBuild = ( - build_scope_relationship_grants( - discovered_inputs=discovered_inputs, - index=index, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, - ) - if has_scoped_relationship_declarations - and (discovered_inputs.test_files or discovered_inputs.scenario_files) - else ScopeRelationshipBuild() - ) - if relationships.faults: - raise CompileInputError(relationships.faults[0].message) - index = replace( - index, - grants=tuple(dict.fromkeys((*index.grants, *relationships.grants))), - completeness=replace(index.completeness, relationships=True), - ) - return DeclarationScopeBuild( + return _build_native_declaration_scope( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, - index=index, - resolver=build_declaration_scope_resolver( - discovered_inputs=discovered_inputs, - scope_index=index, - loaded_macros=loaded_macros, - ), - sql_test_scans=scan_cache, + sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) @@ -116,15 +65,12 @@ def _build_native_declaration_scope( loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, scan_cache: SqlTestScanCache, -) -> DeclarationScopeBuild | None: - """Build the scope natively with Python's errors and order, or None for the Python stage.""" +) -> DeclarationScopeBuild: + """Build and validate the scope index natively, with its relationship grants and lookup.""" - native: NativeScopeIndex | None = open_native_scope_index( + native: NativeScopeIndex = open_native_scope_index( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) - if native is None: - report_native_fallback(site=NativeFallbackSite.SCOPE_INDEX) - return None try: validate_scope_index(index=native.index) except ScopeValidationError as error: @@ -141,14 +87,9 @@ def _build_native_declaration_scope( ) if fault is not None: raise CompileInputError(fault, bridge_independent=True) - if not native.grant(facts): - report_native_fallback(site=NativeFallbackSite.SCOPE_GRANTS) - return None + native.grant(facts) index: ScopeIndex = native.index_with_relationships() - lookup: ScopeLookup | None = native.lookup(index=index) - if lookup is None: - report_native_fallback(site=NativeFallbackSite.SCOPE_LOOKUP) - return None + lookup: ScopeLookup = native.lookup(index=index) report_native_answer(stage=NativeStage.DECLARATION_SCOPES, kind="scope_indexes") return DeclarationScopeBuild( loaded_macros=loaded_macros, @@ -180,6 +121,7 @@ def rebind_declaration_scope( discovered_inputs=discovered_inputs, scope_index=scope.index, loaded_macros=loaded_macros, + lookup=scope.resolver.lookup, ), sql_test_scans=scope.sql_test_scans, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index 644bb14747..3fdbfd73b3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -584,128 +584,6 @@ def _validate_no_inherited_column_overrides( ) -def expand_declaration_references_with_spans( - *, - sql: str, - file_path: Path, - enums: dict[str, EnumDeclaration], - constants: dict[str, ConstantDeclaration], - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - inaccessible_enums: dict[str, DeclarationRecord] | None = None, - inaccessible_constants: dict[str, DeclarationRecord] | None = None, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Resolve declaration references, returning the span of every substitution.""" - - result: DeclarationExpansionResult = expand_declaration_references_result( - sql=sql, - file_path=file_path, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - declarations=DeclarationResolutionContext( - enums=enums, - constants=constants, - inaccessible_enums=inaccessible_enums or {}, - inaccessible_constants=inaccessible_constants or {}, - ), - ) - return result.sql, result.spans - - -def expand_declaration_references_result( - *, - sql: str, - file_path: Path, - declarations: DeclarationResolutionContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, -) -> DeclarationExpansionResult: - """Resolve references and retain usage facts from the resolving token walk.""" - - rendered_parts: list[str] = [] - spans: list[ExpansionSpan] = [] - usages: list[UsageRecord] = [] - output_length: int = 0 - cursor: int = 0 - while cursor < len(sql): - reference_start: int | None = _find_next_reference_start(sql=sql, start=cursor) - if reference_start is None: - rendered_parts.append(sql[cursor:]) - break - leading_literal: str = sql[cursor:reference_start] - rendered_parts.append(leading_literal) - output_length += len(leading_literal) - start_match: re.Match[str] | None = _DECLARATION_REFERENCE_START_PATTERN.match( - sql, reference_start - ) - if start_match is None: - raise CompileInputError( - f"Invalid declaration reference in '{file_path}'", bridge_independent=True - ) - kind: str = start_match.group("kind") - if kind == _ENUM_REFERENCE_KIND: - enum_match: re.Match[str] | None = _ENUM_REFERENCE_PATTERN.match(sql, reference_start) - if enum_match is None: - raise CompileInputError( - _invalid_reference_message(kind=kind, file_path=file_path), - bridge_independent=True, - ) - replacement: str - next_cursor: int - replacement, next_cursor = _resolve_enum_reference( - sql=sql, - reference_start=reference_start, - file_path=file_path, - enums=declarations.enums, - inaccessible_enums=declarations.inaccessible_enums, - ) - visibility: tuple[VisibilityRecord, ...] = declarations.enum_visibility.get( - enum_match.group("name"), () - ) - member: str | None = enum_match.group("member") - else: - constant_match: re.Match[str] | None = _CONSTANT_REFERENCE_PATTERN.match( - sql, reference_start - ) - if constant_match is None: - raise CompileInputError( - _invalid_reference_message(kind=kind, file_path=file_path), - bridge_independent=True, - ) - replacement, next_cursor = _resolve_constant_reference( - sql=sql, - reference_start=reference_start, - file_path=file_path, - constants=declarations.constants, - inaccessible_constants=declarations.inaccessible_constants, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) - visibility = declarations.constant_visibility.get(constant_match.group("name"), ()) - member = None - usages.extend( - declaration_reference_usages( - declarations=declarations, visibility=visibility, enum_member=member - ) - ) - rendered_parts.append(replacement) - spans.append( - ExpansionSpan( - source_start=reference_start, - source_end=next_cursor, - output_start=output_length, - output_end=output_length + len(replacement), - ) - ) - output_length += len(replacement) - cursor = next_cursor - return DeclarationExpansionResult( - sql="".join(rendered_parts), - spans=tuple(spans), - usages=tuple(dict.fromkeys(usages)), - ) - - def declaration_reference_usages( *, declarations: DeclarationResolutionContext, @@ -731,16 +609,14 @@ def declaration_reference_usages( def expand_scanned_declaration_references( # noqa: PLR0913 *, sql: str, - references: NativeDeclarationScan | None, + references: NativeDeclarationScan, file_path: Path, declarations: DeclarationResolutionContext, value_renderer: TypedSqlValueRenderer, collection_rendering: CollectionRendering, -) -> DeclarationExpansionResult | None: - """Splice scanned references and raise Python's first error, or None for Python to expand.""" +) -> DeclarationExpansionResult: + """Splice natively scanned references and raise the first error the scan stopped at.""" - if references is None: - return None scanned, stop = references if not scanned and stop is None: return DeclarationExpansionResult(sql=sql, spans=(), usages=()) @@ -874,29 +750,6 @@ def _with_declaration[T: EnumDeclaration | ConstantDeclaration]( return declarations | {declaration.name: declaration} -def _resolve_enum_reference( - *, - sql: str, - reference_start: int, - file_path: Path, - enums: dict[str, EnumDeclaration], - inaccessible_enums: dict[str, DeclarationRecord], -) -> tuple[str, int]: - match: re.Match[str] | None = _ENUM_REFERENCE_PATTERN.match(sql, reference_start) - if match is None: - raise CompileInputError( - f"Invalid enum reference in '{file_path}'; use @enum(\"name\").MEMBER", - bridge_independent=True, - ) - return _enum_member_text( - name=match.group("name"), - member_name=match.group("member"), - file_path=file_path, - enums=enums, - inaccessible_enums=inaccessible_enums, - ), match.end() - - def _enum_member_text( *, name: str, @@ -935,32 +788,6 @@ def _enum_member_text( return render_enum_member_value(value=member.value) -def _resolve_constant_reference( - *, - sql: str, - reference_start: int, - file_path: Path, - constants: dict[str, ConstantDeclaration], - inaccessible_constants: dict[str, DeclarationRecord], - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, -) -> tuple[str, int]: - match: re.Match[str] | None = _CONSTANT_REFERENCE_PATTERN.match(sql, reference_start) - if match is None: - raise CompileInputError( - f"Invalid constant reference in '{file_path}'; use @const(\"name\")", - bridge_independent=True, - ) - return _constant_text( - name=match.group("name"), - file_path=file_path, - constants=constants, - inaccessible_constants=inaccessible_constants, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ), match.end() - - def _constant_text( *, name: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 859cbf948c..41ebdfee10 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -11,8 +11,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile._helpers.render.declarations import ( - expand_declaration_references_result, - expand_declaration_references_with_spans, expand_scanned_declaration_references, ) from sqlbuild.compiler.compile._helpers.render.macros import ( @@ -33,9 +31,6 @@ SqlInterpolation, ) from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( scan_native_declaration_references, ) @@ -57,7 +52,6 @@ def expand_authored_sql_result( # noqa: PLR0913 ) -> AuthoredSqlExpansionResult: """Apply all expansion passes and retain facts emitted by those passes.""" - native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) interpolated_sql: str = substitute_sql_vars( sql=sql, file_path=file_path, @@ -67,30 +61,13 @@ def expand_authored_sql_result( # noqa: PLR0913 declaration_context: DeclarationResolutionContext = ( declarations or DeclarationResolutionContext() ) - scanned_result: DeclarationExpansionResult | None = ( - expand_scanned_declaration_references( - sql=interpolated_sql, - references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], - file_path=file_path, - declarations=declaration_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) - if native - else None - ) - if native and scanned_result is None: - report_native_fallback(site=NativeFallbackSite.DECLARATION_REFERENCES, kind="scan") - declaration_result: DeclarationExpansionResult = ( - scanned_result - if scanned_result is not None - else expand_declaration_references_result( - sql=interpolated_sql, - file_path=file_path, - declarations=declaration_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - ) + declaration_result: DeclarationExpansionResult = expand_scanned_declaration_references( + sql=interpolated_sql, + references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], + file_path=file_path, + declarations=declaration_context, + value_renderer=value_renderer, + collection_rendering=collection_rendering, ) macro_result: MacroExpansionResult = expand_sql_macros_result( sql=declaration_result.sql, @@ -130,20 +107,23 @@ def expand_authored_sql_with_spans( effective_vars=effective_vars, context_values=context_values, ) - declaration_expanded_sql: str - declaration_spans: tuple[ExpansionSpan, ...] - declaration_expanded_sql, declaration_spans = expand_declaration_references_with_spans( + declaration_result: DeclarationExpansionResult = expand_scanned_declaration_references( sql=interpolated_sql, + references=scan_native_declaration_references(sqls=(interpolated_sql,))[0], file_path=file_path, - enums=declarations.enums if declarations is not None else {}, - constants=declarations.constants if declarations is not None else {}, + declarations=DeclarationResolutionContext( + enums=declarations.enums if declarations is not None else {}, + constants=declarations.constants if declarations is not None else {}, + inaccessible_enums=declarations.inaccessible_enums if declarations is not None else {}, + inaccessible_constants=( + declarations.inaccessible_constants if declarations is not None else {} + ), + ), value_renderer=value_renderer, collection_rendering=collection_rendering, - inaccessible_enums=(declarations.inaccessible_enums if declarations is not None else None), - inaccessible_constants=( - declarations.inaccessible_constants if declarations is not None else None - ), ) + declaration_expanded_sql: str = declaration_result.sql + declaration_spans: tuple[ExpansionSpan, ...] = declaration_result.spans macro_expanded_sql: str macro_spans: tuple[ExpansionSpan, ...] macro_expanded_sql, macro_spans = expand_sql_macros_with_spans( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 56d4d62865..90c3091091 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -6,12 +6,12 @@ import importlib.util import inspect import sys -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable from dataclasses import dataclass, field from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, cast, get_type_hints +from typing import TYPE_CHECKING, get_type_hints from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( @@ -19,7 +19,6 @@ named_declaration_roots, ) from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( - is_in_scoped_declaration_tree, project_relative_path, ) from sqlbuild.compiler.discovery._helpers.native.declaration_files import ( @@ -32,21 +31,11 @@ native_sql_function_files, native_sql_hook_files, ) -from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_file_facts from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function -from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file -from sqlbuild.compiler.discovery._helpers.sql.declarations import ( - parse_constant_declaration_file, - parse_enum_declaration_file, - parse_model_schema_declaration_file, -) -from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql -from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - SEED_FILE_SUFFIX, ) from sqlbuild.compiler.discovery.exceptions import ( EventExporterDiscoveryError, @@ -80,13 +69,10 @@ DiscoveryFileFault, NamedDeclarationRoot, ) -from sqlbuild.compiler.discovery.types import NativeFileScope, ScopedDeclarationFile -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind +from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.provider.exceptions import ProviderInputError from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_audit_factory_definition import ( @@ -120,7 +106,6 @@ from sqlbuild.provider.classes.provider import Provider _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" -_FACTS_MEMO_KEY: str = "declaration_file_facts" @dataclass @@ -147,45 +132,6 @@ def add_audit_factory(self, item: DiscoveredAuditFactory) -> None: self.audit_factories.append(item) -@dataclass(frozen=True) -class _DeclarationFileFacts: - file_path: Path - relative_path: Path - declaration_kind: DeclarationKind - scope_kind: ScopeKind - ownership_root: Path - owning_path: Path | None - declaration_root: Path - - -def _discover_declaration_file_facts( - *, project_dir: Path, declaration_kind: DeclarationKind | None = None -) -> tuple[_DeclarationFileFacts, ...]: - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - memo_key: tuple[str, DeclarationKind | None] = (_FACTS_MEMO_KEY, declaration_kind) - cached: object = tree.memo.get(memo_key) - if cached is not None: - return cast(tuple[_DeclarationFileFacts, ...], cached) - facts: tuple[_DeclarationFileFacts, ...] = tuple( - _DeclarationFileFacts( - file_path=project_dir / relative_path, - relative_path=Path(relative_path), - declaration_kind=DeclarationKind(kind), - scope_kind=ScopeKind(scope_kind), - ownership_root=Path(ownership_root), - owning_path=None if owning_path is None else Path(owning_path), - declaration_root=Path(declaration_root), - ) - for relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root in ( - native_declaration_file_facts( - project_dir=project_dir, declaration_kind=declaration_kind - ) - ) - ) - tree.memo[memo_key] = facts - return facts - - def discover_enum_files( *, project_dir: Path, @@ -194,14 +140,10 @@ def discover_enum_files( ) -> tuple[DiscoveredEnumFile, ...]: """Discover global and scoped enum declaration files.""" - return _discover_declaration_kind_files( + return native_enum_files( project_dir=project_dir, - declaration_kind=DeclarationKind.ENUM, isolate_declaration_kind=isolate_declaration_kind, - parse_declarations=parse_enum_declaration_file, - discovered_file_type=DiscoveredEnumFile, on_fault=on_fault, - native_files=native_enum_files, ) @@ -213,217 +155,19 @@ def discover_constant_files( ) -> tuple[DiscoveredConstantFile, ...]: """Discover global and scoped constant declaration files.""" - return _discover_declaration_kind_files( + return native_constant_files( project_dir=project_dir, - declaration_kind=DeclarationKind.CONSTANT, isolate_declaration_kind=isolate_declaration_kind, - parse_declarations=parse_constant_declaration_file, - discovered_file_type=DiscoveredConstantFile, - on_fault=on_fault, - native_files=native_constant_files, - ) - - -def _discover_declaration_kind_files[DeclarationT, DiscoveredT]( - *, - project_dir: Path, - declaration_kind: DeclarationKind, - isolate_declaration_kind: bool, - parse_declarations: Callable[..., tuple[DeclarationT, ...]], - discovered_file_type: Callable[..., DiscoveredT], - on_fault: Callable[[DiscoveryFileFault], None] | None, - native_files: Callable[..., tuple[DiscoveredT, ...]], -) -> tuple[DiscoveredT, ...]: - def parse(facts: _DeclarationFileFacts) -> DiscoveredT: - contents: str = facts.file_path.read_text(encoding="utf-8") - declarations: tuple[DeclarationT, ...] = parse_declarations( - contents=contents, - file_path=facts.file_path, - relative_path=facts.relative_path, - ) - return discovered_file_type( - file_path=facts.file_path, - relative_path=facts.relative_path, - contents=contents, - declarations=declarations, - scope_kind=facts.scope_kind, - ownership_root=facts.ownership_root, - owning_path=facts.owning_path, - declaration_root=facts.declaration_root, - ) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - on_fault=on_fault, - parse_with_python=_scoped_parser(project_dir=project_dir, parse=parse), - ) - return _parse_discovered_files( - project_dir=project_dir, - items=( - facts - for facts in _discover_declaration_file_facts( - project_dir=project_dir, - declaration_kind=declaration_kind if isolate_declaration_kind else None, - ) - if facts.declaration_kind is declaration_kind - ), - item_path=lambda facts: facts.file_path, - parse=parse, - on_fault=on_fault, - ) - - -def _scoped_parser[DiscoveredT]( - *, project_dir: Path, parse: Callable[[_DeclarationFileFacts], DiscoveredT] -) -> Callable[..., DiscoveredT]: - """Parse a macro, enum or constant file native discovery listed as the Python path does.""" - - def parse_listed(*, relative_path: Path, scope: NativeFileScope | None) -> DiscoveredT: - kind, scope_kind, ownership_root, owning_path, declaration_root = cast( - NativeFileScope, scope - ) - return parse( - _DeclarationFileFacts( - file_path=project_dir / relative_path, - relative_path=relative_path, - declaration_kind=DeclarationKind(kind), - scope_kind=ScopeKind(scope_kind), - ownership_root=Path(cast(str, ownership_root)), - owning_path=None if owning_path is None else Path(owning_path), - declaration_root=Path(cast(str, declaration_root)), - ) - ) - - return parse_listed - - -def _named_parser[DiscoveredT: ScopedDeclarationFile]( - *, project_dir: Path, parse: Callable[[tuple[NamedDeclarationRoot, Path]], DiscoveredT] -) -> Callable[..., DiscoveredT]: - """Parse a named declaration file native discovery listed as its Python role places it.""" - - def parse_listed(*, relative_path: Path, scope: NativeFileScope | None) -> DiscoveredT: - kind, scope_kind, ownership_root, owning_path, declaration_root = cast( - NativeFileScope, scope - ) - relative_directory: Path = Path(cast(str, declaration_root)) - root: NamedDeclarationRoot = NamedDeclarationRoot( - directory=project_dir / relative_directory, - relative_directory=relative_directory, - kind=DeclarationKind(kind), - scope_kind=ScopeKind(scope_kind), - ownership_root=None if ownership_root is None else Path(ownership_root), - owning_path=None if owning_path is None else Path(owning_path), - ) - return root.place(parse((root, project_dir / relative_path))) - - return parse_listed - - -def _parse_discovered_files[ItemT, DiscoveredT]( - *, - project_dir: Path, - items: Iterable[ItemT], - item_path: Callable[[ItemT], Path], - parse: Callable[[ItemT], DiscoveredT], - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredT, ...]: - """Parse each discovered file, isolating per-file read and parse faults when requested.""" - - discovered: list[DiscoveredT] = [] - item: ItemT - for item in items: - try: - discovered.append(parse(item)) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - if on_fault is None: - raise - on_fault(_discovery_fault(project_dir=project_dir, path=item_path(item), error=error)) - return tuple(discovered) - - -def _parse_discovered_paths[DiscoveredT]( - *, - project_dir: Path, - file_paths: Iterable[Path], - parse: Callable[[Path], DiscoveredT], - on_fault: Callable[[DiscoveryFileFault], None] | None, -) -> tuple[DiscoveredT, ...]: - return _parse_discovered_files( - project_dir=project_dir, - items=file_paths, - item_path=lambda file_path: file_path, - parse=parse, on_fault=on_fault, ) -def _unscoped_files(*, root: Path, pattern: str, project_dir: Path) -> Iterator[Path]: - file_path: Path - for file_path in _sorted_glob(project_dir=project_dir, root=root, pattern=pattern): - if not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): - yield file_path - - def discover_model_schema_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None ) -> tuple[DiscoveredModelSchemaFile, ...]: """Discover project-wide and scoped reusable model schemas.""" - def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredModelSchemaFile: - file_path: Path = item[1] - contents: str = file_path.read_text(encoding="utf-8") - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - return DiscoveredModelSchemaFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - declarations=parse_model_schema_declaration_file( - contents=contents, file_path=file_path, relative_path=relative_path - ), - ) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_model_schema_files( - project_dir=project_dir, - on_fault=on_fault, - parse_with_python=_named_parser(project_dir=project_dir, parse=parse), - ) - return _parse_named_declaration_files( - project_dir=project_dir, - kinds=frozenset({DeclarationKind.SCHEMA}), - parse=parse, - on_fault=on_fault, - ) - - -def _parse_named_declaration_files[DiscoveredT: ScopedDeclarationFile]( - *, - project_dir: Path, - kinds: frozenset[DeclarationKind], - parse: Callable[[tuple[NamedDeclarationRoot, Path]], DiscoveredT], - on_fault: Callable[[DiscoveryFileFault], None] | None, - skip_underscored: bool = False, -) -> tuple[DiscoveredT, ...]: - """Parse each SQL file in the requested named declaration roles with its role's scope facts.""" - - return _parse_discovered_files( - project_dir=project_dir, - items=( - item - for item in named_declaration_files( - project_dir=project_dir, - roots=named_declaration_roots(project_dir=project_dir, kinds=kinds), - pattern="*.sql", - ) - if not (skip_underscored and item[1].name.startswith("_")) - ), - item_path=lambda item: item[1], - parse=lambda item: item[0].place(parse(item)), - on_fault=on_fault, - ) + return native_model_schema_files(project_dir=project_dir, on_fault=on_fault) def discover_sql_function_files( @@ -431,38 +175,7 @@ def discover_sql_function_files( ) -> tuple[DiscoveredSqlFunctionFile, ...]: """Discover SQL function files under functions/sql/.""" - def parse(file_path: Path) -> DiscoveredSqlFunctionFile: - contents: str = file_path.read_text(encoding="utf-8") - header_values: dict[str, object] - body_sql: str - header_values, body_sql = parse_function_sql(contents=contents, file_path=file_path) - return DiscoveredSqlFunctionFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - header_values=header_values, - body_sql=body_sql, - ) - - def parse_listed( - *, relative_path: Path, scope: NativeFileScope | None - ) -> DiscoveredSqlFunctionFile: - _ = scope - return parse(project_dir / relative_path) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_sql_function_files( - project_dir=project_dir, on_fault=on_fault, parse_with_python=parse_listed - ) - function_root: Path = project_dir / "functions" / "sql" - if not function_root.is_dir(): - return () - return _parse_discovered_paths( - project_dir=project_dir, - file_paths=_unscoped_files(root=function_root, pattern="*.sql", project_dir=project_dir), - parse=parse, - on_fault=on_fault, - ) + return native_sql_function_files(project_dir=project_dir, on_fault=on_fault) def discover_python_function_files( @@ -500,20 +213,7 @@ def discover_python_function_files( def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: """Discover seed CSV files under seeds/.""" - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_seed_files(project_dir=project_dir) - seeds_root: Path = project_dir / "seeds" - if not seeds_root.is_dir(): - return () - - return tuple( - DiscoveredSeedFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - ) - for file_path in _sorted_glob(project_dir=project_dir, root=seeds_root, pattern="*") - if file_path.is_file() and file_path.suffix == SEED_FILE_SUFFIX - ) + return native_seed_files(project_dir=project_dir) def discover_audit_files( @@ -521,29 +221,7 @@ def discover_audit_files( ) -> tuple[DiscoveredAuditFile, ...]: """Discover generic and singular audit SQL files in their project-wide and scoped roles.""" - def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredAuditFile: - root, file_path = item - contents: str = file_path.read_text(encoding="utf-8") - return DiscoveredAuditFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - blocks=parse_sql_audit_file(contents=contents, file_path=file_path), - declaration_kind=root.kind, - ) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_audit_files( - project_dir=project_dir, - on_fault=on_fault, - parse_with_python=_named_parser(project_dir=project_dir, parse=parse), - ) - return _parse_named_declaration_files( - project_dir=project_dir, - kinds=frozenset({DeclarationKind.AUDIT, DeclarationKind.SINGULAR_AUDIT}), - parse=parse, - on_fault=on_fault, - ) + return native_audit_files(project_dir=project_dir, on_fault=on_fault) def discover_macro_files( @@ -551,42 +229,11 @@ def discover_macro_files( ) -> tuple[DiscoveredMacroFile, ...]: """Discover global and scoped project macro Python files.""" - def read(facts: _DeclarationFileFacts) -> DiscoveredMacroFile: - return DiscoveredMacroFile( - file_path=facts.file_path, - relative_path=facts.relative_path, - contents=facts.file_path.read_text(encoding="utf-8"), - scope_kind=facts.scope_kind, - ownership_root=facts.ownership_root, - owning_path=facts.owning_path, - declaration_root=facts.declaration_root, - ) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_macro_files( - project_dir=project_dir, - isolate_declaration_kind=isolate_declaration_kind, - read_with_python=_scoped_parser(project_dir=project_dir, parse=read), - ) - return tuple( - read(facts) - for facts in _discover_declaration_file_facts( - project_dir=project_dir, - declaration_kind=DeclarationKind.MACRO if isolate_declaration_kind else None, - ) - if facts.declaration_kind is DeclarationKind.MACRO + return native_macro_files( + project_dir=project_dir, isolate_declaration_kind=isolate_declaration_kind ) -def _discovery_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault: - try: - relative_path: Path | None = project_relative_path(path=path, project_dir=project_dir) - except ValueError: - relative_path = None - message: str = str(error).replace(str(project_dir), ".") - return DiscoveryFileFault(path=relative_path, message=message) - - def discover_materialization_files( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> tuple[DiscoveredMaterializationFile, ...]: @@ -702,27 +349,7 @@ def discover_sql_hook_files( ) -> tuple[DiscoveredSqlHookFile, ...]: """Discover named SQL lifecycle hooks in their project-wide and scoped roles.""" - def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredSqlHookFile: - file_path: Path = item[1] - return parse_sql_hook_file( - contents=file_path.read_text(encoding="utf-8"), - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - ) - - if native_stage_enabled(NativeStage.DECLARATION_FILES): - return native_sql_hook_files( - project_dir=project_dir, - on_fault=on_fault, - parse_with_python=_named_parser(project_dir=project_dir, parse=parse), - ) - return _parse_named_declaration_files( - project_dir=project_dir, - kinds=frozenset({DeclarationKind.SQL_HOOK}), - parse=parse, - on_fault=on_fault, - skip_underscored=True, - ) + return native_sql_hook_files(project_dir=project_dir, on_fault=on_fault) def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py index b7bdf895a6..6c1bfba3d9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py @@ -24,7 +24,6 @@ from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( - NATIVE_DEFERRED_TAG, NATIVE_FAILED_TAG, NATIVE_PARSED_TAG, SQL_AUDIT_HEADER_KEYS, @@ -50,12 +49,10 @@ ) from sqlbuild.compiler.discovery.types import NativeFileScope, NativeLocation, NativeScopeFields from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind type _NativeDeclarationFile = tuple[str, NativeFileScope | None, tuple[object, ...]] -type _PythonParse[RecordT] = Callable[..., RecordT] type _Build[RecordT] = Callable[..., RecordT] type _EnumPayload = tuple[str, dict[str, object], str] type _ConstantPayload = tuple[str, dict[str, object], str | None, str | None] @@ -69,7 +66,6 @@ def native_enum_files( project_dir: Path, isolate_declaration_kind: bool, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredEnumFile], ) -> tuple[DiscoveredEnumFile, ...]: """Discover the enum files natively; Python parses the files native parsing defers.""" @@ -78,7 +74,6 @@ def native_enum_files( kind=DeclarationKind.ENUM, isolate_kind=isolate_declaration_kind, build=_enum_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -88,7 +83,6 @@ def native_constant_files( project_dir: Path, isolate_declaration_kind: bool, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredConstantFile], ) -> tuple[DiscoveredConstantFile, ...]: """Discover the constant files natively; Python normalises each constant's value.""" @@ -97,7 +91,6 @@ def native_constant_files( kind=DeclarationKind.CONSTANT, isolate_kind=isolate_declaration_kind, build=_constant_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -106,7 +99,6 @@ def native_model_schema_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredModelSchemaFile], ) -> tuple[DiscoveredModelSchemaFile, ...]: """Discover the reusable model schema files natively; Python parses each schema's columns.""" @@ -114,7 +106,6 @@ def native_model_schema_files( project_dir=project_dir, kind="model_schema", build=_schema_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -123,7 +114,6 @@ def native_sql_function_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredSqlFunctionFile], ) -> tuple[DiscoveredSqlFunctionFile, ...]: """Discover the SQL function files natively.""" @@ -131,7 +121,6 @@ def native_sql_function_files( project_dir=project_dir, kind="sql_function", build=_function_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -140,7 +129,6 @@ def native_sql_hook_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredSqlHookFile], ) -> tuple[DiscoveredSqlHookFile, ...]: """Discover the named SQL hook files natively.""" @@ -148,7 +136,6 @@ def native_sql_hook_files( project_dir=project_dir, kind="sql_hook", build=_hook_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -157,7 +144,6 @@ def native_audit_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None, - parse_with_python: _PythonParse[DiscoveredAuditFile], ) -> tuple[DiscoveredAuditFile, ...]: """Discover the generic and singular audit files natively.""" @@ -165,7 +151,6 @@ def native_audit_files( project_dir=project_dir, kind="audit", build=_audit_file, - parse_with_python=parse_with_python, on_fault=on_fault, ) @@ -174,7 +159,6 @@ def native_macro_files( *, project_dir: Path, isolate_declaration_kind: bool, - read_with_python: _PythonParse[DiscoveredMacroFile], ) -> tuple[DiscoveredMacroFile, ...]: """Discover and read the macro files natively; Python reads the files native reading defers.""" @@ -183,7 +167,6 @@ def native_macro_files( kind=DeclarationKind.MACRO, isolate_kind=isolate_declaration_kind, build=_macro_file, - parse_with_python=read_with_python, on_fault=None, ) @@ -211,7 +194,6 @@ def _native_records[RecordT]( project_dir: Path, kind: str, build: _Build[RecordT], - parse_with_python: _PythonParse[RecordT], on_fault: Callable[[DiscoveryFileFault], None] | None, isolate_kind: bool = False, ) -> tuple[RecordT, ...]: @@ -226,7 +208,6 @@ def _native_records[RecordT]( relative_path=relative_path, item=item, build=build, - parse_with_python=parse_with_python, ), on_fault=on_fault, ) @@ -272,15 +253,11 @@ def _record[RecordT]( relative_path: Path, item: tuple[NativeFileScope | None, tuple[object, ...]], build: _Build[RecordT], - parse_with_python: _PythonParse[RecordT], ) -> RecordT: - """Build a parsed file or raise Python's error for it; Python parses the files native defers.""" + """Build a parsed file or raise Python's error for it.""" scope, payload = item tag: object = payload[0] - if tag == NATIVE_DEFERRED_TAG: - report_native_fallback(site=NativeFallbackSite.DECLARATION_FILE) - return parse_with_python(relative_path=relative_path, scope=scope) if tag == NATIVE_PARSED_TAG: report_native_answer(stage=NativeStage.DECLARATION_FILES, kind="parsed_files") return build( diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py index 29b613e0ed..f8f83add05 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py @@ -14,22 +14,12 @@ ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import NATIVE_FAILED_TAG -from sqlbuild.compiler.discovery.types import NativeDeclarationFact from sqlbuild.compiler.scopes.types import DeclarationKind _LAYOUT_MEMO_KEY: str = "native_declaration_layout" _GROUPS_MEMO_KEY: str = "native_declaration_groups" -def native_declaration_file_facts( - *, project_dir: Path, declaration_kind: DeclarationKind | None -) -> list[NativeDeclarationFact]: - """Return the declaration files of one kind (or every kind), raising an invalid layout.""" - - facts, _groups = _layout(project_dir=project_dir, declaration_kind=declaration_kind) - return cast(list[NativeDeclarationFact], _rows(facts)) - - def native_declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: """Return the `(canonical root, group directory)` pairs of a valid named layout.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py b/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py deleted file mode 100644 index e7ff307bbc..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py +++ /dev/null @@ -1,397 +0,0 @@ -"""Parsing helpers for authored SQL audit files.""" - -from __future__ import annotations - -from collections.abc import Iterator -from inspect import cleandoc -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.auditing.models import MeasurementContract -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY, SQL_AUDIT_HEADER_KEYS -from sqlbuild.compiler.discovery.exceptions import SqlAuditParseError -from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock - -_AUDIT_NAME_HEADER_KEY: str = "name" -_AUDIT_SEVERITY_HEADER_KEY: str = "severity" -_AUDIT_RUN_SCOPE_HEADER_KEY: str = "run_scope" -_AUDIT_ALWAYS_RUN_HEADER_KEY: str = "always_run" -_AUDIT_EVALUATION_HEADER_KEY: str = "evaluation" -_AUDIT_VALUE_HEADER_KEY: str = "value" -_AUDIT_SAMPLE_COUNT_HEADER_KEY: str = "sample_count" -_AUDIT_SAMPLE_UNIT_HEADER_KEY: str = "sample_unit" -_AUDIT_THRESHOLDS_HEADER_KEY: str = "thresholds" -_AUDIT_MINIMUM_SAMPLES_HEADER_KEY: str = "minimum_samples" -_OPEN_PAREN: str = "(" -_CLOSE_PAREN: str = ")" -_STATEMENT_DELIMITER: str = ";" -_IDENTIFIER_SEPARATOR: str = "_" -_ESCAPE_CHARACTER: str = "\\" -_SQL_QUOTES: frozenset[str] = frozenset({"'", '"', "`"}) - - -def parse_sql_audit_file(*, contents: str, file_path: Path) -> tuple[DiscoveredAuditBlock, ...]: - """Parse one SQL audit file into one or more raw AUDIT(...) blocks.""" - - starts: tuple[int, ...] = _find_top_level_keywords(text=contents, keyword="AUDIT") - if not starts or contents[: starts[0]].strip(): - raise SqlAuditParseError( - f"SQL audit '{file_path}' must start with an AUDIT() header as the first " - "non-whitespace content" - ) - blocks: list[DiscoveredAuditBlock] = [] - for index, start in enumerate(starts, start=1): - end: int = starts[index] if index < len(starts) else len(contents) - blocks.append( - _parse_single_sql_audit_block( - file_path=file_path, - raw_audit_block=contents[start:end].strip(), - block_line=contents.count("\n", 0, start) + 1, - audit_index=index, - ) - ) - result: tuple[DiscoveredAuditBlock, ...] = tuple(blocks) - _validate_audit_names(file_path=file_path, blocks=result) - return result - - -def _parse_single_sql_audit_block( - *, file_path: Path, raw_audit_block: str, block_line: int, audit_index: int -) -> DiscoveredAuditBlock: - open_index: int = _keyword_open_paren( - text=raw_audit_block, keyword="AUDIT", file_path=file_path - ) - close_index: int = _find_matching_parenthesis( - text=raw_audit_block, - open_index=open_index, - file_path=file_path, - label="AUDIT", - ) - body_start: int = _consume_delimiter( - text=raw_audit_block, - start=close_index + 1, - file_path=file_path, - label="AUDIT(...) header", - ) - header_values: dict[str, object] = _parse_audit_header( - header=raw_audit_block[open_index + 1 : close_index], - header_line=block_line + raw_audit_block.count("\n", 0, open_index + 1), - file_path=file_path, - ) - evaluation_mode: AuditEvaluationMode = _parse_evaluation_mode( - header_values=header_values, file_path=file_path - ) - body: str = raw_audit_block[body_start:] - measure_sql: str | None = None - evidence_sql: str | None = None - contract: MeasurementContract | None = None - if evaluation_mode == AuditEvaluationMode.MEASUREMENT: - measure_sql, evidence_sql = _parse_measurement_body(body=body, file_path=file_path) - contract = MeasurementContract( - value_column=cast(str, header_values[_AUDIT_VALUE_HEADER_KEY]), - sample_count_column=cast(str | None, header_values.get(_AUDIT_SAMPLE_COUNT_HEADER_KEY)), - sample_unit=cast(str | None, header_values.get(_AUDIT_SAMPLE_UNIT_HEADER_KEY)), - ) - sql_body: str = measure_sql - else: - sql_body = cleandoc(body) - if not sql_body: - raise SqlAuditParseError(f"SQL audit '{file_path}' must define SQL after AUDIT(...)") - body_code_start: int = _skip_space_and_comments(text=sql_body, start=0) - if body_code_start in ( - *_find_top_level_keywords(text=sql_body, keyword="MEASURE"), - *_find_top_level_keywords(text=sql_body, keyword="EVIDENCE"), - ): - raise SqlAuditParseError( - f"Violation audit '{file_path}' must use a bare SELECT body, not MEASURE/EVIDENCE" - ) - - return DiscoveredAuditBlock( - audit_index=audit_index, - header_values=header_values, - sql_body=sql_body, - name=cast(str | None, header_values.get(_AUDIT_NAME_HEADER_KEY)), - evaluation_mode=evaluation_mode, - measurement_contract=contract, - measure_sql=measure_sql, - evidence_sql=evidence_sql, - ) - - -def _parse_audit_header(*, header: str, header_line: int, file_path: Path) -> dict[str, object]: - parsed_header: dict[str, object] = parse_header_values( - header=header, - file_path=file_path, - statement_name="AUDIT", - header_line=header_line, - error_class=SqlAuditParseError, - ) - reject_unsupported_header_keys( - header_values=parsed_header, - supported_keys=SQL_AUDIT_HEADER_KEYS, - statement="AUDIT()", - header=header, - header_line=header_line, - file_path=file_path, - error_class=SqlAuditParseError, - ) - for key in ( - _AUDIT_NAME_HEADER_KEY, - _AUDIT_SEVERITY_HEADER_KEY, - _AUDIT_RUN_SCOPE_HEADER_KEY, - _AUDIT_EVALUATION_HEADER_KEY, - _AUDIT_VALUE_HEADER_KEY, - _AUDIT_SAMPLE_COUNT_HEADER_KEY, - _AUDIT_SAMPLE_UNIT_HEADER_KEY, - ): - value: object | None = parsed_header.get(key) - if key in parsed_header and (not isinstance(value, str) or not value.strip()): - raise SqlAuditParseError(f"AUDIT() {key} in '{file_path}' must be a non-empty string") - always_run: object | None = parsed_header.get(_AUDIT_ALWAYS_RUN_HEADER_KEY) - if _AUDIT_ALWAYS_RUN_HEADER_KEY in parsed_header and not isinstance(always_run, bool): - raise SqlAuditParseError(f"AUDIT() always_run in '{file_path}' must be a boolean") - sql_analysis: object | None = parsed_header.get(SQL_ANALYSIS_CONFIG_KEY) - if SQL_ANALYSIS_CONFIG_KEY in parsed_header and not isinstance(sql_analysis, bool): - raise SqlAuditParseError(f"AUDIT() sql_analysis in '{file_path}' must be a boolean") - return parsed_header - - -def _parse_evaluation_mode( - *, header_values: dict[str, object], file_path: Path -) -> AuditEvaluationMode: - raw_mode: object = header_values.get( - _AUDIT_EVALUATION_HEADER_KEY, AuditEvaluationMode.VIOLATIONS.value - ) - try: - mode: AuditEvaluationMode = AuditEvaluationMode(cast(str, raw_mode)) - except ValueError as error: - raise SqlAuditParseError( - f"AUDIT() evaluation in '{file_path}' must be one of: violations, measurement" - ) from error - measurement_only_keys: tuple[str, ...] = ( - _AUDIT_VALUE_HEADER_KEY, - _AUDIT_SAMPLE_COUNT_HEADER_KEY, - _AUDIT_SAMPLE_UNIT_HEADER_KEY, - _AUDIT_THRESHOLDS_HEADER_KEY, - _AUDIT_MINIMUM_SAMPLES_HEADER_KEY, - ) - if mode == AuditEvaluationMode.VIOLATIONS: - invalid: tuple[str, ...] = tuple( - key for key in measurement_only_keys if key in header_values - ) - if invalid: - raise SqlAuditParseError( - f"Violation audit '{file_path}' must not define measurement keys: " - f"{', '.join(invalid)}" - ) - elif _AUDIT_VALUE_HEADER_KEY not in header_values: - raise SqlAuditParseError(f"Measurement audit '{file_path}' must define `value`") - return mode - - -def _parse_measurement_body(*, body: str, file_path: Path) -> tuple[str, str | None]: - position: int = _skip_space_and_comments(text=body, start=0) - measure_sql, position = _extract_delimited_query( - text=body, position=position, keyword="MEASURE", file_path=file_path, required=True - ) - position = _skip_space_and_comments(text=body, start=position) - evidence_sql: str | None = None - if _keyword_at(text=body, keyword="EVIDENCE", position=position): - evidence_sql, position = _extract_delimited_query( - text=body, position=position, keyword="EVIDENCE", file_path=file_path, required=False - ) - position = _skip_space_and_comments(text=body, start=position) - if position != len(body): - label: str = ( - "duplicate MEASURE/EVIDENCE block" - if _keyword_at_any(text=body, position=position, keywords=("MEASURE", "EVIDENCE")) - else "bare SQL outside MEASURE/EVIDENCE blocks" - ) - raise SqlAuditParseError(f"Measurement audit '{file_path}' has {label}") - return cast(str, measure_sql), evidence_sql - - -def _extract_delimited_query( - *, text: str, position: int, keyword: str, file_path: Path, required: bool -) -> tuple[str | None, int]: - if not _keyword_at(text=text, keyword=keyword, position=position): - if required: - raise SqlAuditParseError( - f"Measurement audit '{file_path}' must define exactly one MEASURE(...) block" - ) - return None, position - open_index: int = ( - _keyword_open_paren(text=text[position:], keyword=keyword, file_path=file_path) + position - ) - close_index: int = _find_matching_parenthesis( - text=text, open_index=open_index, file_path=file_path, label=keyword - ) - query: str = cleandoc(text[open_index + 1 : close_index]) - _validate_single_query(query=query, file_path=file_path, label=keyword) - end: int = _consume_delimiter( - text=text, start=close_index + 1, file_path=file_path, label=f"{keyword}(...) block" - ) - return query, end - - -def _validate_single_query(*, query: str, file_path: Path, label: str) -> None: - if not query: - raise SqlAuditParseError(f"{label}(...) in '{file_path}' must contain one SELECT query") - start: int = _skip_space_and_comments(text=query, start=0) - if not ( - _keyword_at(text=query, keyword="SELECT", position=start) - or _keyword_at(text=query, keyword="WITH", position=start) - ): - raise SqlAuditParseError(f"{label}(...) in '{file_path}' must contain one SELECT query") - semicolons: tuple[int, ...] = _top_level_semicolons(query) - if semicolons and (len(semicolons) > 1 or query[semicolons[0] + 1 :].strip()): - raise SqlAuditParseError(f"{label}(...) in '{file_path}' must contain exactly one query") - - -def _keyword_open_paren(*, text: str, keyword: str, file_path: Path) -> int: - start: int = _skip_space_and_comments(text=text, start=0) - if not _keyword_at(text=text, keyword=keyword, position=start): - raise SqlAuditParseError(f"SQL audit '{file_path}' must start with {keyword}(...)") - position: int = _skip_space_and_comments(text=text, start=start + len(keyword)) - if position >= len(text) or text[position] != _OPEN_PAREN: - raise SqlAuditParseError(f"SQL audit '{file_path}' must start with {keyword}(...)") - return position - - -def _consume_delimiter(*, text: str, start: int, file_path: Path, label: str) -> int: - position: int = _skip_space_and_comments(text=text, start=start) - if position >= len(text) or text[position] != _STATEMENT_DELIMITER: - raise SqlAuditParseError(f"{label} in '{file_path}' must end with `);`") - return position + 1 - - -def _find_matching_parenthesis(*, text: str, open_index: int, file_path: Path, label: str) -> int: - depth: int = 1 - for index, character in _code_characters(text=text, start=open_index + 1): - if character == _OPEN_PAREN: - depth += 1 - elif character == _CLOSE_PAREN: - depth -= 1 - if depth == 0: - return index - raise SqlAuditParseError(f"Unclosed {label}(...) block in '{file_path}'") - - -def _find_top_level_keywords(*, text: str, keyword: str) -> tuple[int, ...]: - depth: int = 0 - found: list[int] = [] - for index, character in _code_characters(text=text, start=0): - if character == _OPEN_PAREN: - depth += 1 - elif character == _CLOSE_PAREN: - depth = max(0, depth - 1) - elif ( - depth == 0 - and character.upper() == keyword[0] - and _keyword_at(text=text, keyword=keyword, position=index) - ): - after: int = _skip_space_and_comments(text=text, start=index + len(keyword)) - if after < len(text) and text[after] == _OPEN_PAREN: - found.append(index) - return tuple(found) - - -def _top_level_semicolons(text: str) -> tuple[int, ...]: - depth: int = 0 - found: list[int] = [] - for index, character in _code_characters(text=text, start=0): - if character == _OPEN_PAREN: - depth += 1 - elif character == _CLOSE_PAREN: - depth = max(0, depth - 1) - elif character == _STATEMENT_DELIMITER and depth == 0: - found.append(index) - return tuple(found) - - -def _code_characters(*, text: str, start: int) -> Iterator[tuple[int, str]]: - """Yield code characters while skipping quoted text and SQL comments.""" - - index: int = start - while index < len(text): - character: str = text[index] - if text.startswith("--", index): - newline: int = text.find("\n", index + 2) - index = len(text) if newline < 0 else newline + 1 - continue - if text.startswith("/*", index): - close: int = text.find("*/", index + 2) - index = len(text) if close < 0 else close + 2 - continue - if character in _SQL_QUOTES: - quote: str = character - index += 1 - while index < len(text): - if text[index] == quote: - if index + 1 < len(text) and text[index + 1] == quote: - index += 2 - continue - index += 1 - break - if text[index] == _ESCAPE_CHARACTER: - index += 2 - else: - index += 1 - continue - yield index, character - index += 1 - - -def _skip_space_and_comments(*, text: str, start: int) -> int: - position: int = start - while position < len(text): - if text[position].isspace(): - position += 1 - elif text.startswith("--", position): - newline: int = text.find("\n", position + 2) - position = len(text) if newline < 0 else newline + 1 - elif text.startswith("/*", position): - close: int = text.find("*/", position + 2) - position = len(text) if close < 0 else close + 2 - else: - break - return position - - -def _keyword_at(*, text: str, keyword: str, position: int) -> bool: - end: int = position + len(keyword) - if text[position:end].upper() != keyword: - return False - before: str = text[position - 1] if position > 0 else " " - after: str = text[end] if end < len(text) else " " - return not (before.isalnum() or before == _IDENTIFIER_SEPARATOR) and not ( - after.isalnum() or after == _IDENTIFIER_SEPARATOR - ) - - -def _keyword_at_any(*, text: str, position: int, keywords: tuple[str, ...]) -> bool: - return any(_keyword_at(text=text, keyword=keyword, position=position) for keyword in keywords) - - -def _validate_audit_names(*, file_path: Path, blocks: tuple[DiscoveredAuditBlock, ...]) -> None: - if len(blocks) <= 1: - return - unnamed_indexes: tuple[int, ...] = tuple( - block.audit_index for block in blocks if block.name is None - ) - if unnamed_indexes: - missing_indexes: str = ", ".join(str(index) for index in unnamed_indexes) - raise SqlAuditParseError( - f"SQL audit '{file_path}' contains multiple AUDIT blocks; every block must define " - f"a unique `name`. Missing names for blocks: {missing_indexes}" - ) - seen_names: set[str] = set() - for block in blocks: - if block.name in seen_names: - raise SqlAuditParseError( - f"SQL audit '{file_path}' defines duplicate AUDIT() name '{block.name}'" - ) - seen_names.add(cast(str, block.name)) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py index e31455d249..6549ea85c7 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py @@ -4,21 +4,14 @@ import re from collections.abc import Sequence -from dataclasses import dataclass from pathlib import Path from typing import cast -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - header_column_locations, - parse_header_values, -) -from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError, ModelSqlParseError +from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, EnumDeclaration, EnumMember, - ModelSchemaDeclaration, ) from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, @@ -30,146 +23,10 @@ from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind _IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -_DECLARATION_START_PATTERN: re.Pattern[str] = re.compile(r"(?PENUM|CONSTANT|SCHEMA)\s*\(") _VARCHAR_TYPE: str = "VARCHAR" _INTEGER_TYPE: str = "INTEGER" _STATEMENT_TERMINATOR: str = ";" _ESCAPE_CHARACTER: str = "\\" -_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) -_OPEN_PARENTHESIS: str = "(" -_CLOSE_PARENTHESIS: str = ")" - - -@dataclass(frozen=True) -class _ParsedDeclarationHeader: - values: dict[str, object] - header: str - header_start: int - - -def parse_enum_declaration_file( - *, contents: str, file_path: Path, relative_path: Path -) -> tuple[EnumDeclaration, ...]: - """Parse all public enum declarations in one file.""" - - return tuple( - _parse_enum_declaration( - values=parsed_header.values, - file_path=file_path, - relative_path=relative_path, - model_name=None, - ) - for parsed_header in _parse_declaration_headers( - contents=contents, - file_path=file_path, - expected_kind="ENUM", - ) - ) - - -def parse_constant_declaration_file( - *, contents: str, file_path: Path, relative_path: Path -) -> tuple[ConstantDeclaration, ...]: - """Parse all public constant declarations in one file.""" - - return tuple( - _parse_constant_declaration( - name=parsed_header.values.get("name"), - value=parsed_header.values.get("value", MISSING_SQL_VALUE), - explicit_type=parsed_header.values.get("type", MISSING_SQL_VALUE), - raw_render_as=parsed_header.values.get("render_as", MISSING_SQL_VALUE), - file_path=file_path, - relative_path=relative_path, - model_name=None, - unknown_keys=set(parsed_header.values) - {"name", "value", "type", "render_as"}, - ) - for parsed_header in _parse_declaration_headers( - contents=contents, - file_path=file_path, - expected_kind="CONSTANT", - ) - ) - - -def parse_model_schema_declaration_file( - *, contents: str, file_path: Path, relative_path: Path -) -> tuple[ModelSchemaDeclaration, ...]: - """Parse all public reusable model schemas in one file.""" - - declarations: list[ModelSchemaDeclaration] = [] - parsed_header: _ParsedDeclarationHeader - for parsed_header in _parse_declaration_headers( - contents=contents, - file_path=file_path, - expected_kind="SCHEMA", - ): - values: dict[str, object] = parsed_header.values - unknown_keys: set[str] = set(values) - {"name", "description", "extends", "columns"} - if unknown_keys: - raise DeclarationParseError( - f"{file_path} schema has unknown keys: {', '.join(sorted(unknown_keys))}" - ) - name: str = _parse_declaration_name( - raw_name=values.get("name"), - file_path=file_path, - kind="schema", - model_name=None, - ) - description: str | None = _parse_optional_declaration_string( - raw_value=values.get("description"), - file_path=file_path, - label=f"schema '{name}' description", - ) - extends: str | None = _parse_optional_declaration_identifier( - raw_value=values.get("extends"), - file_path=file_path, - label=f"schema '{name}' extends", - ) - declarations.append( - ModelSchemaDeclaration( - name=name, - description=description, - extends=extends, - columns=parse_schema_columns( - raw_columns=values.get("columns"), - file_path=file_path, - label=f"schema '{name}'", - error_class=DeclarationParseError, - column_locations=header_column_locations( - contents=contents, - header=parsed_header.header, - header_start=parsed_header.header_start, - relative_path=relative_path, - ), - require_columns=True, - ), - relative_path=relative_path, - ) - ) - return tuple(declarations) - - -def _parse_optional_declaration_string( - *, raw_value: object | None, file_path: Path, label: str -) -> str | None: - if raw_value is None: - return None - if not isinstance(raw_value, str) or not raw_value.strip(): - raise DeclarationParseError(f"{file_path} {label} must be a non-empty string") - return raw_value - - -def _parse_optional_declaration_identifier( - *, raw_value: object | None, file_path: Path, label: str -) -> str | None: - value: str | None = _parse_optional_declaration_string( - raw_value=raw_value, - file_path=file_path, - label=label, - ) - if value is not None and not _IDENTIFIER_PATTERN.fullmatch(value): - raise DeclarationParseError(f"{file_path} {label} must be an identifier") - return value def parse_model_enum_declarations( @@ -224,56 +81,6 @@ def parse_model_constant_declarations( return tuple(declarations) -def _parse_declaration_headers( - *, contents: str, file_path: Path, expected_kind: str -) -> tuple[_ParsedDeclarationHeader, ...]: - headers: list[_ParsedDeclarationHeader] = [] - cursor: int = 0 - while cursor < len(contents): - cursor = _skip_whitespace(contents=contents, start=cursor) - if cursor == len(contents): - break - match: re.Match[str] | None = _DECLARATION_START_PATTERN.match(contents, cursor) - if match is None: - raise DeclarationParseError( - f"{file_path} must contain only {expected_kind}(...) declarations" - ) - actual_kind: str = match.group("kind") - if actual_kind != expected_kind: - raise DeclarationParseError( - f"{file_path} contains {actual_kind}(...) under the {expected_kind.lower()}s root" - ) - open_index: int = match.end() - 1 - close_index: int = _find_closing_parenthesis( - contents=contents, - open_index=open_index, - file_path=file_path, - ) - cursor = _skip_whitespace(contents=contents, start=close_index + 1) - if cursor >= len(contents) or contents[cursor] != _STATEMENT_TERMINATOR: - raise DeclarationParseError(f"{expected_kind}(...) in '{file_path}' must end with ';'") - try: - header: str = contents[open_index + 1 : close_index] - headers.append( - _ParsedDeclarationHeader( - values=parse_header_values( - header=header, - file_path=file_path, - statement_name=expected_kind, - header_line=contents.count("\n", 0, open_index + 1) + 1, - ), - header=header, - header_start=open_index + 1, - ) - ) - except ModelSqlParseError as error: - raise DeclarationParseError(str(error), help=error.help) from error - cursor += 1 - if not headers: - raise DeclarationParseError(f"{file_path} contains no {expected_kind}(...) declarations") - return tuple(headers) - - def _parse_enum_declaration( *, values: dict[str, object], @@ -516,32 +323,6 @@ def _scalar_type(*, value: str | int) -> str: return _VARCHAR_TYPE if isinstance(value, str) else _INTEGER_TYPE -def _find_closing_parenthesis(*, contents: str, open_index: int, file_path: Path) -> int: - depth: int = 1 - quote: str | None = None - index: int = open_index + 1 - while index < len(contents): - character: str = contents[index] - if quote is not None: - if character == _ESCAPE_CHARACTER: - index += 2 - continue - if character == quote: - quote = None - index += 1 - continue - if character in _QUOTE_TOKENS: - quote = character - elif character == _OPEN_PARENTHESIS: - depth += 1 - elif character == _CLOSE_PARENTHESIS: - depth -= 1 - if depth == 0: - return index - index += 1 - raise DeclarationParseError(f"{file_path} has an unterminated declaration header") - - def _skip_whitespace(*, contents: str, start: int) -> int: index: int = start while index < len(contents) and contents[index].isspace(): diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py b/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py deleted file mode 100644 index 06cab4dd39..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/functions.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Parsing helpers for authored SQL function files.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import ( - SQL_FUNCTION_HEADER_KEYS, - STATEMENT_HEADER_BODY_PATTERN, -) -from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError - -_FUNCTION_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*FUNCTION\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", - re.DOTALL, -) - - -def parse_function_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: - """Parse a raw SQL function file into header values and SQL body.""" - - header_match: re.Match[str] | None = _FUNCTION_HEADER_PATTERN.match(contents) - if header_match is None: - raise ModelSqlParseError( - f"SQL function '{file_path}' must start with a FUNCTION(...) header as the first " - "non-whitespace content" - ) - - header_line: int = contents.count("\n", 0, header_match.start("header")) + 1 - header_values: dict[str, object] = parse_header_values( - header=header_match.group("header"), - file_path=file_path, - statement_name="FUNCTION", - header_line=header_line, - ) - reject_unsupported_header_keys( - header_values=header_values, - supported_keys=SQL_FUNCTION_HEADER_KEYS, - statement="FUNCTION()", - header=header_match.group("header"), - header_line=header_line, - file_path=file_path, - error_class=ModelSqlParseError, - ) - body_sql: str = header_match.group("sql").strip() - if not body_sql: - raise ModelSqlParseError(f"SQL function '{file_path}' must contain SQL after FUNCTION(...)") - return header_values, body_sql diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py b/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py deleted file mode 100644 index cd2c7e473d..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/hooks.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Parsing helpers for authored SQL hook resources.""" - -from __future__ import annotations - -from inspect import cleandoc -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values -from sqlbuild.compiler.discovery.constants import SQL_HOOK_HEADER_KEYS -from sqlbuild.compiler.discovery.exceptions import SqlHookParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile - -_HOOK_DESCRIPTION_HEADER_KEY: str = "description" -_SQL_ESCAPE_CHARACTER: str = "\\" -_SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"', "`"}) -_SQL_STATEMENT_TERMINATOR: str = ";" -_HOOK_HEADER_OPEN: str = "(" -_HOOK_HEADER_CLOSE: str = ")" - - -def parse_sql_hook_file( - *, contents: str, file_path: Path, relative_path: Path -) -> DiscoveredSqlHookFile: - """Parse exactly one named SQL hook resource from a file.""" - - header: str - raw_sql_body: str - header_start: int - header, header_start, raw_sql_body = _split_hook_file(contents=contents, file_path=file_path) - header_values: dict[str, object] = _parse_hook_header( - header=header, - header_line=contents.count("\n", 0, header_start) + 1, - file_path=file_path, - ) - sql_body: str = cleandoc(raw_sql_body) - if not sql_body: - raise SqlHookParseError(f"SQL hook '{file_path}' must define SQL after HOOK(...)") - description_value: object | None = header_values.get("description") - return DiscoveredSqlHookFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - header_values=header_values, - sql_body=sql_body, - name=file_path.stem, - description=cast(str | None, description_value), - ) - - -def _split_hook_file(*, contents: str, file_path: Path) -> tuple[str, int, str]: - index: int = 0 - while index < len(contents) and contents[index].isspace(): - index += 1 - if not contents.startswith("HOOK", index): - raise SqlHookParseError( - f"SQL hook '{file_path}' must start with a HOOK() header as the first " - "non-whitespace content" - ) - index += len("HOOK") - while index < len(contents) and contents[index].isspace(): - index += 1 - if index >= len(contents) or contents[index] != _HOOK_HEADER_OPEN: - raise SqlHookParseError(f"SQL hook '{file_path}' must define a HOOK(...) header") - - header_start: int = index + 1 - depth: int = 1 - quote: str | None = None - index = header_start - while index < len(contents): - character: str = contents[index] - if quote is not None: - if character == _SQL_ESCAPE_CHARACTER: - index += 2 - continue - if character == quote: - if index + 1 < len(contents) and contents[index + 1] == quote: - index += 2 - continue - quote = None - index += 1 - continue - if character in _SQL_QUOTE_CHARACTERS: - quote = character - elif character == _HOOK_HEADER_OPEN: - depth += 1 - elif character == _HOOK_HEADER_CLOSE: - depth -= 1 - if depth == 0: - break - index += 1 - if depth != 0: - raise SqlHookParseError(f"SQL hook '{file_path}' has an unterminated HOOK(...) header") - - header_end: int = index - index += 1 - while index < len(contents) and contents[index].isspace(): - index += 1 - if index >= len(contents) or contents[index] != _SQL_STATEMENT_TERMINATOR: - raise SqlHookParseError(f"SQL hook '{file_path}' HOOK(...) header must end with ';'") - return contents[header_start:header_end], header_start, contents[index + 1 :] - - -def _parse_hook_header(*, header: str, header_line: int, file_path: Path) -> dict[str, object]: - parsed_header: dict[str, object] = parse_header_values( - header=header, - file_path=file_path, - statement_name="HOOK", - header_line=header_line, - error_class=SqlHookParseError, - ) - reject_unsupported_header_keys( - header_values=parsed_header, - supported_keys=SQL_HOOK_HEADER_KEYS, - statement="HOOK()", - header=header, - header_line=header_line, - file_path=file_path, - error_class=SqlHookParseError, - ) - description: object | None = parsed_header.get(_HOOK_DESCRIPTION_HEADER_KEY) - if _HOOK_DESCRIPTION_HEADER_KEY in parsed_header and ( - not isinstance(description, str) or not description.strip() - ): - raise SqlHookParseError(f"HOOK() description in '{file_path}' must be a non-empty string") - return parsed_header diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 5cb584b78e..f41b35b72b 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -8,7 +8,6 @@ LEGACY_LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.yml" TOML_FILE_SUFFIX: str = ".toml" SCHEMA_FILE_NAME: str = "schema.yml" -SEED_FILE_SUFFIX: str = ".csv" YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) RESERVED_MODEL_NAMES: frozenset[str] = frozenset({"_chain_"}) @@ -162,7 +161,6 @@ SOURCE_AGE_POLICY_CONFIG_KEY: str = "age_policy" MODELS_DIRECTORY_NAME: str = "models" -MODEL_SCHEMAS_DIRECTORY_NAME: str = "schemas" SEEDS_DIRECTORY_NAME: str = "seeds" CURRENT_DIRECTORY_PATH: str = "." SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" @@ -199,7 +197,6 @@ NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" NATIVE_UNLISTABLE_TAG: str = "unlistable" -NATIVE_DEFERRED_TAG: str = "defer" NATIVE_PARSED_TAG: str = "ok" NATIVE_SOURCE_YAML_KIND: str = "source" NATIVE_SCHEMA_YAML_KIND: str = "schema" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index daff13b0a5..e06a14f4de 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -15,7 +15,6 @@ type ProjectProvider = Provider type NativeLocation = tuple[str, int, int, int, int] type NativeListing = tuple[str, list[tuple[str, bool, bool]]] -type NativeDeclarationFact = tuple[str, str, str, str, str | None, str] type NativeFiles = list[tuple[str, tuple[object, ...]]] type NativeFileScope = tuple[str, str, str | None, str | None, str | None] diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index dd825586bd..c34afe7e93 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -50,13 +50,8 @@ class CompilerStage(StrEnum): class NativeFallbackSite(StrEnum): """A place where a shipped native stage still hands its work to the Python implementation.""" - DECLARATION_FILE = "declaration_files.file" - SCOPE_INDEX = "declaration_scopes.index" - SCOPE_GRANTS = "declaration_scopes.grants" - SCOPE_LOOKUP = "declaration_scopes.lookup" SCOPE_REBIND_LOOKUP = "declaration_scopes.rebind_lookup" SCOPE_RELATIONSHIP_CTES = "declaration_scopes.relationship_ctes" - DECLARATION_REFERENCES = "model_loop.declaration_references" DECLARATION_CONTEXT = "model_loop.declaration_context" CURSOR_INTRINSIC_VALIDATION = "model_loop.cursor_intrinsic_validation" LINT_EXPANSION = "model_loop.lint_expansion" diff --git a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py index 5d9d8b81e4..744faabeb5 100644 --- a/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py +++ b/src/sqlbuild/compiler/model_loop/main/_scan_native_declaration_references.py @@ -2,16 +2,21 @@ from __future__ import annotations +import sys +import unicodedata + import sqlbuild._native as _native from sqlbuild.compiler.model_loop.types import NativeDeclarationScan def scan_native_declaration_references( *, sqls: tuple[str, ...] -) -> tuple[NativeDeclarationScan | None, ...]: - """Return each string's references and the code of the error ending its walk, or None.""" +) -> tuple[NativeDeclarationScan, ...]: + """Return each string's references and the code of the error ending its walk.""" return tuple( - None if scan is None else (tuple(scan[0]), scan[1]) - for scan in _native.scan_sql_declaration_references(list(sqls)) + (tuple(scan[0]), scan[1]) + for scan in _native.scan_sql_declaration_references( + list(sqls), (sys.version_info[0], sys.version_info[1]), unicodedata.unidata_version + ) ) diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py index b7d7957705..f7b8bea4c3 100644 --- a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py +++ b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py @@ -8,6 +8,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows +from sqlbuild.compiler.scopes.exceptions import UnindexableScopeError from sqlbuild.compiler.scopes.models import ( DeclarationRecord, DeclarationVisibilityIndex, @@ -132,8 +133,8 @@ def __init__(self, *, native: _native.NativeScopeIndex, rows: NativeScopeRows) - completeness=_STATIC_COMPLETENESS, ) - def grant(self, facts: Sequence[RelationshipFact]) -> bool: - """Attach the relationship grants of `facts`; False when Python must resolve them.""" + def grant(self, facts: Sequence[RelationshipFact]) -> None: + """Attach the relationship grants of `facts`.""" rows: _GrantRows | None = self._native.grant( [ @@ -148,7 +149,7 @@ def grant(self, facts: Sequence[RelationshipFact]) -> bool: ] ) if rows is None: - return False + raise UnindexableScopeError identities: dict[tuple[str, str], ResourceIdentity] = {} self._grants = [ GrantRecord( @@ -165,7 +166,6 @@ def grant(self, facts: Sequence[RelationshipFact]) -> bool: ) for kind, name, declaration, through, grant_kind in rows ] - return True def index_with_relationships(self) -> ScopeIndex: """Return the index with its grants attached and relationships complete.""" @@ -176,12 +176,12 @@ def index_with_relationships(self) -> ScopeIndex: completeness=replace(self.index.completeness, relationships=True), ) - def lookup(self, *, index: ScopeIndex) -> ScopeLookup | None: - """Return Python's `build_lookup(index)` for `index_with_relationships()`, or None.""" + def lookup(self, *, index: ScopeIndex) -> ScopeLookup: + """Return Python's `build_lookup(index)` for `index_with_relationships()`.""" rows: _LookupRows | None = self._native.lookup() if rows is None: - return None + raise UnindexableScopeError resource_positions, declaration_positions, usage_positions, grant_positions = rows[:4] groups: tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups] = rows[4] global_positions, private_positions, local_positions, inherited_positions = rows[5] diff --git a/src/sqlbuild/compiler/scopes/exceptions.py b/src/sqlbuild/compiler/scopes/exceptions.py index e368b911e8..bc31940329 100644 --- a/src/sqlbuild/compiler/scopes/exceptions.py +++ b/src/sqlbuild/compiler/scopes/exceptions.py @@ -28,6 +28,17 @@ class InvalidQualifiedIdentityError(ScopeError): code: str = "S002" +class UnindexableScopeError(ScopeError): + """Raised when the native scope index cannot place a discovered path or resource kind.""" + + def __init__(self) -> None: + super().__init__( + "Native scope indexing could not place a discovered path or resource kind. Discovery " + "always yields normalized project-relative paths, so this is a SQLBuild bug; please " + "file an issue with the project layout" + ) + + class ScopeCacheDecodeError(ScopeError): """Raised when a persistent scope-index payload is malformed.""" diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py index bdc74e0f7c..fba02f1617 100644 --- a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py @@ -1,4 +1,4 @@ -"""Build the static scope index natively for the compiler engine switch.""" +"""Build the static scope index natively.""" from __future__ import annotations @@ -9,24 +9,20 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError +from sqlbuild.compiler.scopes.exceptions import UnindexableScopeError def open_native_scope_index( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] -) -> NativeScopeIndex | None: - """Return the natively built index, or None when the Python builder must run instead.""" +) -> NativeScopeIndex: + """Return the natively built index; discovery's normalized paths always index.""" - try: - rows: NativeScopeRows = NativeScopeRows( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - except (InvalidScopePathError, TypeError, ValueError): - return None - try: - native: _native.NativeScopeIndex | None = _native.build_native_scope_index( - rows.resources, rows.declarations - ) - except (TypeError, UnicodeError): - return None - return None if native is None else NativeScopeIndex(native=native, rows=rows) + rows: NativeScopeRows = NativeScopeRows( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + ) + native: _native.NativeScopeIndex | None = _native.build_native_scope_index( + rows.resources, rows.declarations + ) + if native is None: + raise UnindexableScopeError + return NativeScopeIndex(native=native, rows=rows) diff --git a/tests/goldens/compiler/failure/engine-error-non-ascii-digit-bare-number.json b/tests/goldens/compiler/failure/engine-error-non-ascii-digit-bare-number.json new file mode 100644 index 0000000000..9b393a86a3 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-non-ascii-digit-bare-number.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D013", + "message": "/enums/order_priority.sql has the bare number '٣', written with non-ASCII digits", + "help": "Quote it to keep it as text (\"٣\"), or write the number with ASCII digits 0-9" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 0898ca5204..ce3c94f921 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -2,7 +2,6 @@ import sys import unicodedata -from collections.abc import Callable from dataclasses import dataclass, field @@ -156,13 +155,13 @@ class NativeOsErrorTestCase: @dataclass(frozen=True) -class CollidingNamesTestCase: - """Files whose names are not UTF-8 but share a lossy spelling, each keeping its own identity.""" +class UndecodableDeclarationNameTestCase: + """A declaration file whose path holds a name that is not UTF-8, which discovery rejects.""" description: str - files: tuple[tuple[str, bytes], ...] - expected_enums: tuple[tuple[str, tuple[str, ...]], ...] - expected_models: tuple[str, ...] = ("models/orders.sql",) + relative_path: bytes + contents: bytes + expected_path: str @dataclass(frozen=True) @@ -189,19 +188,6 @@ class DeclarationFilesParityTestCase: expected_message_fragment: str = "" -@dataclass(frozen=True) -class NativeDeclarationFailureTestCase: - """A declaration file native discovery rejects while the Python parser is patched away.""" - - description: str - relative_path: str - contents: bytes - patched_parser: str - patched: Callable[..., object] - expected_failure_type: str - expected_error_fragment: str - - @dataclass(frozen=True) class NativeSessionTestCase: """Whether a discovery pass under one engine keeps its native declaration session.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index f9b725c006..33223ae925 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -8,7 +8,6 @@ import itertools import json import math -import os import random import struct import tomllib @@ -53,7 +52,6 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( - DiscoveredEnumFile, DiscoveredProjectInputs, DiscoveryFileFault, NamedSqlHookEntry, @@ -810,23 +808,6 @@ def _inputs_paths(inputs: DiscoveredProjectInputs) -> tuple[object, ...]: ) -def declared_enums_outcome(*, project_dir: Path) -> tuple[object, ...]: - """Return each discovered enum file with the enum names it declares, and the model paths.""" - - inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - return ( - tuple(_declared_enum_names(enum_file) for enum_file in inputs.enum_files), - tuple(model.relative_path.as_posix() for model in inputs.model_files), - ) - - -def _declared_enum_names(enum_file: DiscoveredEnumFile) -> tuple[str, tuple[str, ...]]: - return ( - enum_file.relative_path.as_posix(), - tuple(declaration.name for declaration in enum_file.declarations), - ) - - DECLARATION_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { "enum": discover_enum_files, "constant": discover_constant_files, @@ -1272,27 +1253,6 @@ def generated_declaration_outcomes( return tuple(compress(range(test_case.case_count), map(not_, agreements))), tags -def accept_any_declarations(**_: object) -> tuple[object, ...]: - """A patched Python parser that accepts every file.""" - - return () - - -def reject_any_contents(**_: object) -> tuple[object, ...]: - """A patched Python parser that rejects every file with an unrelated error.""" - - raise ValueError("different") - - -def write_undecodable_hook(*, project_dir: Path, contents: bytes) -> None: - """Write a project whose only SQL hook has a file name that is not valid UTF-8.""" - - write_project(project_dir=project_dir, files=()) - hooks: Path = project_dir / "hooks" / "sql" - hooks.mkdir(parents=True) - _ = Path(os.fsdecode(bytes(hooks) + b"/refresh_\xff.sql")).write_bytes(contents) - - def on_deep_stack(*, frames: int, call: Callable[[], object]) -> object: """Return `call()` invoked beneath `frames` extra Python frames.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 217af86f42..cbea4e7bcd 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -2,14 +2,12 @@ from __future__ import annotations -import sys from collections import Counter from pathlib import Path import pytest from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem import core from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR @@ -17,20 +15,16 @@ DeclarationFilesParityTestCase, DeepDeclarationHeaderTestCase, GeneratedDeclarationFileTestCase, - NativeDeclarationFailureTestCase, NativeSessionTestCase, TolerantDeclarationFilesTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - accept_any_declarations, declaration_files_outcome, generated_declaration_outcomes, native_declaration_tag, - reject_any_contents, stage_outcome, tolerant_declaration_outcome, write_project, - write_undecodable_hook, ) _PREVIEW: str = "native-preview" @@ -358,36 +352,6 @@ def test_given_deeply_nested_declaration_header_when_discovering_then_engines_re ) -@pytest.mark.skipif(sys.platform != "linux", reason="needs file names that are not UTF-8") -@pytest.mark.parametrize( - "test_case", - [ - DeclarationFilesParityTestCase( - description="a hook named with invalid UTF-8 is read by Python under both engines", - files=(("hooks/sql/refresh.sql", _HOOK),), - expected_failure_type="ResourceIdentityError", - expected_message_fragment="refresh_", - ) - ], - ids=lambda case: case.description, -) -def test_given_hook_named_with_invalid_utf8_when_discovering_then_engines_agree( - test_case: DeclarationFilesParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_undecodable_hook(project_dir=tmp_path, contents=test_case.files[0][1]) - python: tuple[object, ...] = stage_outcome( - project_dir=tmp_path, engine="python", monkeypatch=monkeypatch - ) - - native: tuple[object, ...] = stage_outcome( - project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch - ) - - assert native == python - assert native[1] == test_case.expected_failure_type - assert test_case.expected_message_fragment in str(native[2]) - - @pytest.mark.parametrize( "test_case", [ @@ -425,45 +389,6 @@ def test_given_broken_declaration_files_when_discovering_tolerantly_then_faults_ assert len(native[2]) == test_case.expected_declaration_faults -@pytest.mark.parametrize( - "test_case", - [ - NativeDeclarationFailureTestCase( - description="a native parse error stands even where Python would accept the file", - relative_path="enums/status.sql", - contents=b"ENUM (name order_status);\n", - patched_parser="parse_enum_declaration_file", - patched=accept_any_declarations, - expected_failure_type="DeclarationParseError", - expected_error_fragment="enums/status.sql", - ), - NativeDeclarationFailureTestCase( - description="a native parse error stands where Python would fail differently", - relative_path="audits/generic/is_true.sql", - contents=b"SELECT 1\n", - patched_parser="parse_sql_audit_file", - patched=reject_any_contents, - expected_failure_type="SqlAuditParseError", - expected_error_fragment="audits/generic/is_true.sql", - ), - ], - ids=lambda case: case.description, -) -def test_given_native_declaration_failure_when_discovering_then_python_is_not_rerun( - test_case: NativeDeclarationFailureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project(project_dir=tmp_path, files=((test_case.relative_path, test_case.contents),)) - monkeypatch.setattr(core, test_case.patched_parser, test_case.patched) - outcome: tuple[object, ...] = stage_outcome( - project_dir=tmp_path, engine=_PREVIEW, monkeypatch=monkeypatch - ) - - assert (outcome[1], test_case.expected_error_fragment in str(outcome[2])) == ( - test_case.expected_failure_type, - True, - ), outcome - - @pytest.mark.parametrize( "test_case", [ @@ -478,14 +403,14 @@ def test_given_native_declaration_failure_when_discovering_then_python_is_not_re expected_session=True, ), NativeSessionTestCase( - description="the Python engine reads declaration files in Python", + description="the Python engine keeps it too, since declaration files are native-only", engine="python", - expected_session=False, + expected_session=True, ), ], ids=lambda case: case.description, ) -def test_given_engine_when_discovering_then_native_session_is_kept_only_for_native_engines( +def test_given_engine_when_discovering_then_native_session_is_kept( test_case: NativeSessionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_project(project_dir=tmp_path, files=_EVERY_KIND) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py index e84753c024..adef309087 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py @@ -8,12 +8,13 @@ import pytest +from sqlbuild.compiler.discovery.exceptions import ProjectPathError +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( - CollidingNamesTestCase, + UndecodableDeclarationNameTestCase, UndecodablePathToleranceTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - declared_enums_outcome, description_inputs_outcome, selected_contract_outcome, tolerant_scope_fault_outcome, @@ -108,55 +109,45 @@ def test_given_undecodable_paths_when_discovering_tolerantly_then_each_read_path @pytest.mark.parametrize( "test_case", [ - CollidingNamesTestCase( - description="an enum file whose name is not UTF-8 is read by its real name", - files=((os.fsdecode(b"enums/st\xe9.sql"), _ORDER_STATUS),), - expected_enums=((os.fsdecode(b"enums/st\xe9.sql"), ("order_status",)),), + UndecodableDeclarationNameTestCase( + description="an enum file whose name is not UTF-8", + relative_path=b"enums/st\xe9.sql", + contents=_ORDER_STATUS, + expected_path="enums/st\\xe9.sql", ), - CollidingNamesTestCase( - description="enum files sharing a lossy name each keep their own declarations", - files=( - (os.fsdecode(b"enums/st\xe9.sql"), _ORDER_STATUS), - (os.fsdecode(b"enums/st\xe8.sql"), _TIER), - ), - expected_enums=( - (os.fsdecode(b"enums/st\xe8.sql"), ("tier",)), - (os.fsdecode(b"enums/st\xe9.sql"), ("order_status",)), - ), - ), - CollidingNamesTestCase( - description="a valid replacement-character name is not replaced by a raw sibling", - files=( - ("enums/st\ufffd.sql", _ORDER_STATUS), - (os.fsdecode(b"enums/st\xe8.sql"), _TIER), - ), - expected_enums=( - (os.fsdecode(b"enums/st\xe8.sql"), ("tier",)), - ("enums/st\ufffd.sql", ("order_status",)), - ), + UndecodableDeclarationNameTestCase( + description="a scoped enum directory whose name is not UTF-8", + relative_path=b"models/a\xe8/_enums/y.sql", + contents=_TIER, + expected_path="models/a\\xe8/_enums/y.sql", ), - CollidingNamesTestCase( - description="directories sharing a lossy name each keep their own files", - files=( - (os.fsdecode(b"models/a\xe9/_enums/x.sql"), _ORDER_STATUS), - (os.fsdecode(b"models/a\xe8/_enums/y.sql"), _TIER), - ), - expected_enums=( - (os.fsdecode(b"models/a\xe8/_enums/y.sql"), ("tier",)), - (os.fsdecode(b"models/a\xe9/_enums/x.sql"), ("order_status",)), - ), + UndecodableDeclarationNameTestCase( + description="a SQL hook whose name is not UTF-8", + relative_path=b"hooks/sql/refresh_\xff.sql", + contents=b"HOOK ();\nSELECT 1\n", + expected_path="hooks/sql/refresh_\\xff.sql", ), ], ids=lambda case: case.description, ) -def test_given_names_sharing_a_lossy_spelling_when_discovering_then_each_file_keeps_its_identity( - test_case: CollidingNamesTestCase, tmp_path: Path +def test_given_declaration_file_named_with_invalid_utf8_when_discovering_then_d016_names_it( + test_case: UndecodableDeclarationNameTestCase, tmp_path: Path ) -> None: - write_project(project_dir=tmp_path, files=(("models/orders.sql", _MODEL), *test_case.files)) + write_project( + project_dir=tmp_path, + files=( + ("models/orders.sql", _MODEL), + (os.fsdecode(test_case.relative_path), test_case.contents), + ), + ) - outcome: tuple[object, ...] = declared_enums_outcome(project_dir=tmp_path) + with pytest.raises(ProjectPathError) as raised: + discover_project_inputs(project_dir=tmp_path) - assert outcome == (test_case.expected_enums, test_case.expected_models), test_case.description + assert (raised.value.code, str(raised.value)) == ( + "D016", + f"Project path {tmp_path / test_case.expected_path}{_RENAME}", + ), test_case.description if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py index 299ea79987..9d48db4f9f 100644 --- a/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_loop/_test_types.py @@ -1,15 +1 @@ from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class DeclarationReferenceParityTestCase: - """Seeded SQL with `@enum`/`@const` references expanded natively and in Python.""" - - description: str - seed: int - count: int - expected_minimum_native_references: int - expected_maximum_deferred: int - expected_minimum_native_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py b/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py index a6b3feb1b7..bc508ca764 100644 --- a/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_loop/helpers.py @@ -3,24 +3,12 @@ from __future__ import annotations import random -from dataclasses import dataclass from pathlib import Path -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.render.declarations import ( - expand_declaration_references_result, - expand_scanned_declaration_references, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - DeclarationExpansionResult, DeclarationResolutionContext, ) from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration, EnumMember -from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( - scan_native_declaration_references, -) -from sqlbuild.compiler.model_loop.types import NativeDeclarationScan from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -36,7 +24,6 @@ VisibilityReason, ) from sqlbuild.sql_values.main.normalize import normalize_sql_value -from sqlbuild.sql_values.types import CollectionRendering _CONSUMER: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, "orders") _FILE_PATH: Path = Path("models/orders.sql") @@ -164,28 +151,12 @@ ) -@dataclass(frozen=True) -class ReferenceParity: - """Python's expansion (or error) and the native one (or error, or None) of one SQL string.""" - - sql: str - python: DeclarationExpansionResult | tuple[str, str, str | None] - native: DeclarationExpansionResult | tuple[str, str, str | None] | None - native_references: int - - def generated_reference_sql(*, rng: random.Random) -> str: """Return SQL mixing references, quoted text, comments and Unicode in random order.""" return _reference_sql(rng=rng, rare_parts=int(rng.random() < 0.15)) -def generated_failing_reference_sql(*, rng: random.Random) -> str: - """Return reference SQL that often holds several invalid references or unclosed text.""" - - return _reference_sql(rng=rng, rare_parts=rng.choice((0, 0, 0, 0, 1, 2, 3))) - - def _reference_sql(*, rng: random.Random, rare_parts: int) -> str: parts: list[str] = [ rng.choices((_reference_part, _text_part), weights=(4, 6))[0](rng) @@ -196,67 +167,9 @@ def _reference_sql(*, rng: random.Random, rare_parts: int) -> str: return "".join(parts) -def reference_parities(*, sqls: list[str]) -> list[ReferenceParity]: - """Expand every string in Python and through the native scan with Python rendering.""" - - scanned: tuple[NativeDeclarationScan | None, ...] = scan_native_declaration_references( - sqls=tuple(sqls) - ) - return [ - ReferenceParity( - sql=sql, - python=_python_outcome(sql=sql), - native=_native_outcome(sql=sql, references=references), - native_references=len((references or ((), None))[0]), - ) - for sql, references in zip(sqls, scanned, strict=True) - ] - - -def is_native(parity: ReferenceParity) -> bool: - """Whether the native path expanded the string instead of deferring to Python.""" - - return parity.native is not None - - def _reference_part(rng: random.Random) -> str: return rng.choice(_REFERENCES).format(s=rng.choice(_WHITESPACE)) def _text_part(rng: random.Random) -> str: return rng.choice(_TEXT) - - -def is_error(outcome: object) -> bool: - """Whether an expansion outcome is an error's code, message and help.""" - - return isinstance(outcome, tuple) - - -def _python_outcome(*, sql: str) -> DeclarationExpansionResult | tuple[str, str, str | None]: - try: - return expand_declaration_references_result( - sql=sql, - file_path=_FILE_PATH, - declarations=DECLARATIONS, - value_renderer=DuckDbAdapter(), - collection_rendering=CollectionRendering.VALUE_LIST, - ) - except CompileInputError as error: - return (error.code, str(error), error.help) - - -def _native_outcome( - *, sql: str, references: NativeDeclarationScan | None -) -> DeclarationExpansionResult | tuple[str, str, str | None] | None: - try: - return expand_scanned_declaration_references( - sql=sql, - references=references, - file_path=_FILE_PATH, - declarations=DECLARATIONS, - value_renderer=DuckDbAdapter(), - collection_rendering=CollectionRendering.VALUE_LIST, - ) - except CompileInputError as error: - return (error.code, str(error), error.help) diff --git a/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py b/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py deleted file mode 100644 index 409f409bb7..0000000000 --- a/tests/integration/src/sqlbuild/compiler/model_loop/test_native_declaration_references.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Natively scanned `@enum`/`@const` references expand, and fail, exactly as Python's do.""" - -from __future__ import annotations - -import random - -import pytest - -from tests.integration.src.sqlbuild.compiler.helpers import mismatches -from tests.integration.src.sqlbuild.compiler.model_loop._test_types import ( - DeclarationReferenceParityTestCase, -) -from tests.integration.src.sqlbuild.compiler.model_loop.helpers import ( - ReferenceParity, - generated_failing_reference_sql, - is_error, - is_native, - reference_parities, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationReferenceParityTestCase( - description="seeded references and errors among quotes, comments and Unicode", - seed=20261007, - count=3000, - expected_minimum_native_references=5000, - expected_maximum_deferred=150, - expected_minimum_native_errors=1200, - ), - ], - ids=lambda case: case.description, -) -def test_given_generated_sql_when_expanding_references_natively_then_python_outcome_matches( - test_case: DeclarationReferenceParityTestCase, -) -> None: - rng: random.Random = random.Random(test_case.seed) - sqls: list[str] = [generated_failing_reference_sql(rng=rng) for _ in range(test_case.count)] - - parities: list[ReferenceParity] = reference_parities(sqls=sqls) - - expanded: list[ReferenceParity] = list(filter(is_native, parities)) - - assert ( - mismatches( - inputs=[parity.sql for parity in expanded], - expected=[parity.python for parity in expanded], - actual=[parity.native for parity in expanded], - ), - sum(parity.native_references for parity in expanded) - >= test_case.expected_minimum_native_references, - len(parities) - len(expanded) <= test_case.expected_maximum_deferred, - sum(is_error(parity.native) for parity in expanded) - >= test_case.expected_minimum_native_errors, - ) == ([], True, True, True), test_case.description - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 971f779ef7..9cd83dec84 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -312,10 +312,8 @@ def scope_engine_outcomes( engine=CompilerEngine.NATIVE_PREVIEW.value, monkeypatch=monkeypatch, ) - built: bool = ( - open_native_scope_index(discovered_inputs=discovered, loaded_macros=macros) is not None - ) - return python, native, built + _ = open_native_scope_index(discovered_inputs=discovered, loaded_macros=macros) + return python, native, True def scope_command_indexes( @@ -341,7 +339,7 @@ def native_scope_attempts( def counted( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] - ) -> NativeScopeIndex | None: + ) -> NativeScopeIndex: attempts.append(None) return open_native_scope_index( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index e2c35f1d45..85375d41ea 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -216,7 +216,9 @@ def test_given_scoped_project_when_scope_command_builds_index_then_engines_match "test_case", [ ScopeEngineTierTestCase( - description="python oracle", engine="python", expected_native_attempts=0 + description="python engine, since the compile scope index is native-only", + engine="python", + expected_native_attempts=1, ), ScopeEngineTierTestCase( description="shipped native stages", engine="native", expected_native_attempts=1 @@ -227,7 +229,7 @@ def test_given_scoped_project_when_scope_command_builds_index_then_engines_match ], ids=lambda case: case.description, ) -def test_given_engine_tier_when_building_scope_then_native_scopes_run_only_in_native_engines( +def test_given_engine_tier_when_building_scope_then_the_native_index_is_built_once( test_case: ScopeEngineTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_project(project_dir=tmp_path, files=SCOPED_PROJECT) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index c863255e57..6052b8bc76 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -3,12 +3,14 @@ from typing import cast from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, + discover_model_schema_files, + discover_sql_function_files, + discover_sql_hook_files, ) -from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file -from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, @@ -105,12 +107,40 @@ def write_unreadable_files(*, project_dir: Path, relative_paths: tuple[str, ...] _ = file_path.write_bytes(b"\xff\xfe\xfa") +_DECLARATION_FILE_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { + "enums": discover_enum_files, + "constants": discover_constant_files, + "schemas": discover_model_schema_files, + "functions": discover_sql_function_files, + "hooks": discover_sql_hook_files, + "audits": discover_audit_files, +} + + +def discover_declaration_file(*, project_dir: Path, relative_path: str, contents: str) -> object: + """Write one declaration file into a project and return the file discovery reads from it.""" + + file_path: Path = project_dir / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + _ = file_path.write_text(contents, encoding="utf-8") + discovered: tuple[object, ...] = _DECLARATION_FILE_DISCOVERERS[relative_path.split("/")[0]]( + project_dir=project_dir + ) + return discovered[0] + + _STATEMENT_HEADER_PARSERS: dict[str, Callable[..., object]] = { "MODEL": parse_model_sql, - "AUDIT": parse_sql_audit_file, + "AUDIT": lambda *, contents, file_path: discover_declaration_file( + project_dir=file_path.parents[2], + relative_path=file_path.relative_to(file_path.parents[2]).as_posix(), + contents=contents, + ), "TEST": parse_sql_test_file, - "HOOK": lambda *, contents, file_path: parse_sql_hook_file( - contents=contents, file_path=file_path, relative_path=file_path + "HOOK": lambda *, contents, file_path: discover_declaration_file( + project_dir=file_path.parents[2], + relative_path=file_path.relative_to(file_path.parents[2]).as_posix(), + contents=contents, ), "SCENARIO": lambda *, contents, file_path: parse_sql_scenario_file( contents=contents, file_path=file_path, relative_path=file_path @@ -118,6 +148,21 @@ def write_unreadable_files(*, project_dir: Path, relative_paths: tuple[str, ...] } +_DECLARATION_STATEMENT_DIRECTORIES: dict[str, str] = { + "AUDIT": "audits/generic", + "HOOK": "hooks/sql", +} + + +def statement_header_file_path(*, statement: str, project_dir: Path) -> Path: + """Return where a statement file is parsed from: in the project for declaration kinds.""" + + directory: str | None = _DECLARATION_STATEMENT_DIRECTORIES.get(statement) + if directory is None: + return Path("orders.sql") + return project_dir / directory / "orders.sql" + + def parse_statement_header_file(*, statement: str, contents: str, file_path: Path) -> object: """Parse one authored statement file with the discovery parser for its header kind.""" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py index d6b6e26573..2a91cab738 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py @@ -3,15 +3,17 @@ from __future__ import annotations from pathlib import Path +from typing import cast import pytest from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function -from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError +from sqlbuild.compiler.discovery.models import DiscoveredSqlFunctionFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( FunctionHeaderKeysTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import discover_declaration_file @pytest.mark.parametrize( @@ -40,12 +42,16 @@ ids=lambda case: case.description, ) def test_given_supported_function_header_keys_when_parsing_then_accepts_header( - test_case: FunctionHeaderKeysTestCase, + test_case: FunctionHeaderKeysTestCase, tmp_path: Path ) -> None: - header_values: dict[str, object] - header_values, _ = parse_function_sql( - contents=test_case.contents, file_path=Path(test_case.file_name) - ) + header_values: dict[str, object] = cast( + DiscoveredSqlFunctionFile, + discover_declaration_file( + project_dir=tmp_path, + relative_path=f"functions/sql/{test_case.file_name}", + contents=test_case.contents, + ), + ).header_values assert tuple(sorted(header_values)) == test_case.expected_keys @@ -78,12 +84,19 @@ def test_given_supported_function_header_keys_when_parsing_then_accepts_header( ids=lambda case: case.description, ) def test_given_unknown_sql_function_header_key_when_parsing_then_rejects_key_with_location( - test_case: FunctionHeaderKeysTestCase, + test_case: FunctionHeaderKeysTestCase, tmp_path: Path ) -> None: + file_path: Path = tmp_path / "functions" / "sql" / test_case.file_name with pytest.raises(ModelSqlParseError) as raised: - parse_function_sql(contents=test_case.contents, file_path=Path(test_case.file_name)) + discover_declaration_file( + project_dir=tmp_path, + relative_path=f"functions/sql/{test_case.file_name}", + contents=test_case.contents, + ) - assert raised.value.message == test_case.expected_error + assert raised.value.message == test_case.expected_error.replace( + f"'{test_case.file_name}", f"'{file_path}" + ) assert raised.value.code == "D002" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py index 22b814489e..07084c8e4a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_audits.py @@ -6,14 +6,14 @@ import pytest from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file from sqlbuild.compiler.discovery.exceptions import SqlAuditParseError -from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock +from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock, DiscoveredAuditFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseMeasurementAuditTestCase, ParseSqlAuditFileErrorTestCase, ParseSqlAuditFileTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import discover_declaration_file @pytest.mark.parametrize( @@ -96,10 +96,16 @@ ) def test_given_sql_audit_file_variants_when_parsing_then_it_returns_expected_raw_blocks( test_case: ParseSqlAuditFileTestCase, + tmp_path: Path, ) -> None: - discovered_blocks: tuple[DiscoveredAuditBlock, ...] = parse_sql_audit_file( - contents=test_case.contents, file_path=Path("audits/orders.sql") - ) + discovered_blocks: tuple[DiscoveredAuditBlock, ...] = cast( + DiscoveredAuditFile, + discover_declaration_file( + project_dir=tmp_path, + relative_path="audits/generic/orders.sql", + contents=test_case.contents, + ), + ).blocks assert tuple(block.name for block in discovered_blocks) == test_case.expected_names assert tuple(block.sql_body for block in discovered_blocks) == test_case.expected_sql_bodies @@ -221,9 +227,14 @@ def test_given_sql_audit_file_variants_when_parsing_then_it_returns_expected_raw ) def test_given_invalid_sql_audit_file_contents_when_parsing_then_it_raises_clear_errors( test_case: ParseSqlAuditFileErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(SqlAuditParseError, match=test_case.expected_error_fragment): - parse_sql_audit_file(contents=test_case.contents, file_path=Path("audits/orders.sql")) + discover_declaration_file( + project_dir=tmp_path, + relative_path="audits/generic/orders.sql", + contents=test_case.contents, + ) @pytest.mark.parametrize( @@ -256,11 +267,16 @@ def test_given_invalid_sql_audit_file_contents_when_parsing_then_it_raises_clear ) def test_given_measurement_audit_with_evidence_when_parsing_then_contract_and_queries_are_separate( test_case: ParseMeasurementAuditTestCase, + tmp_path: Path, ) -> None: - blocks: tuple[DiscoveredAuditBlock, ...] = parse_sql_audit_file( - contents=test_case.contents, - file_path=Path("audits/rate.sql"), - ) + blocks: tuple[DiscoveredAuditBlock, ...] = cast( + DiscoveredAuditFile, + discover_declaration_file( + project_dir=tmp_path, + relative_path="audits/generic/rate.sql", + contents=test_case.contents, + ), + ).blocks block: DiscoveredAuditBlock = blocks[0] assert block.evaluation_mode == AuditEvaluationMode.MEASUREMENT @@ -305,6 +321,11 @@ def test_given_measurement_audit_with_evidence_when_parsing_then_contract_and_qu ) def test_given_invalid_evaluation_grammar_when_parsing_then_clear_error_is_raised( test_case: ParseSqlAuditFileErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(SqlAuditParseError, match=test_case.expected_error_fragment): - parse_sql_audit_file(contents=test_case.contents, file_path=Path("audits/rate.sql")) + discover_declaration_file( + project_dir=tmp_path, + relative_path="audits/generic/rate.sql", + contents=test_case.contents, + ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index e19694db84..510e54f6db 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -7,14 +7,14 @@ import pytest from sqlbuild.compiler.discovery._helpers.sql.declarations import ( - parse_constant_declaration_file, - parse_enum_declaration_file, parse_model_constant_declarations, - parse_model_schema_declaration_file, ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, + DiscoveredConstantFile, + DiscoveredEnumFile, + DiscoveredModelSchemaFile, EnumDeclaration, ModelSchemaDeclaration, ) @@ -28,6 +28,7 @@ ParseModelSchemaDeclarationTestCase, ParseTypedConstantsTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import discover_declaration_file @pytest.mark.parametrize( @@ -70,12 +71,14 @@ ) def test_given_enum_declarations_when_parsing_then_returns_typed_members( test_case: ParseDeclarationFileTestCase, + tmp_path: Path, ) -> None: - declarations: tuple[EnumDeclaration, ...] = parse_enum_declaration_file( - contents=test_case.contents, - file_path=Path("enums/domain.sql"), - relative_path=Path("enums/domain.sql"), - ) + declarations: tuple[EnumDeclaration, ...] = cast( + DiscoveredEnumFile, + discover_declaration_file( + project_dir=tmp_path, relative_path="enums/domain.sql", contents=test_case.contents + ), + ).declarations assert tuple(declaration.name for declaration in declarations) == test_case.expected_names assert ( @@ -106,12 +109,16 @@ def test_given_enum_declarations_when_parsing_then_returns_typed_members( ) def test_given_constant_declarations_when_parsing_then_returns_typed_values( test_case: ParseDeclarationFileTestCase, + tmp_path: Path, ) -> None: - declarations: tuple[ConstantDeclaration, ...] = parse_constant_declaration_file( - contents=test_case.contents, - file_path=Path("constants/thresholds.sql"), - relative_path=Path("constants/thresholds.sql"), - ) + declarations: tuple[ConstantDeclaration, ...] = cast( + DiscoveredConstantFile, + discover_declaration_file( + project_dir=tmp_path, + relative_path="constants/thresholds.sql", + contents=test_case.contents, + ), + ).declarations assert tuple(declaration.name for declaration in declarations) == test_case.expected_names assert ( @@ -148,12 +155,14 @@ def test_given_constant_declarations_when_parsing_then_returns_typed_values( ) def test_given_typed_public_constants_when_parsing_then_returns_normalized_values( test_case: ParseTypedConstantsTestCase, + tmp_path: Path, ) -> None: - declarations: tuple[ConstantDeclaration, ...] = parse_constant_declaration_file( - contents=test_case.contents, - file_path=Path("constants/domain.sql"), - relative_path=Path("constants/domain.sql"), - ) + declarations: tuple[ConstantDeclaration, ...] = cast( + DiscoveredConstantFile, + discover_declaration_file( + project_dir=tmp_path, relative_path="constants/domain.sql", contents=test_case.contents + ), + ).declarations assert ( tuple(declaration.value.kind.value for declaration in declarations) @@ -243,12 +252,11 @@ def test_given_local_shorthand_and_wrapper_constants_when_parsing_then_honors_op ) def test_given_invalid_typed_constant_when_parsing_then_raises_contextual_error( test_case: ParseDeclarationFileErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(DeclarationParseError, match=test_case.expected_error_fragment): - parse_constant_declaration_file( - contents=test_case.contents, - file_path=Path("constants/domain.sql"), - relative_path=Path("constants/domain.sql"), + discover_declaration_file( + project_dir=tmp_path, relative_path="constants/domain.sql", contents=test_case.contents ) @@ -295,12 +303,11 @@ def test_given_invalid_typed_constant_when_parsing_then_raises_contextual_error( ) def test_given_invalid_enum_declaration_when_parsing_then_raises_declaration_error( test_case: ParseDeclarationFileErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(DeclarationParseError, match=test_case.expected_error_fragment): - parse_enum_declaration_file( - contents=test_case.contents, - file_path=Path("enums/domain.sql"), - relative_path=Path("enums/domain.sql"), + discover_declaration_file( + project_dir=tmp_path, relative_path="enums/domain.sql", contents=test_case.contents ) @@ -319,15 +326,14 @@ def test_given_invalid_enum_declaration_when_parsing_then_raises_declaration_err ) def test_given_noncanonical_declaration_identity_when_parsing_then_d016_suggests_canonical_identity( test_case: DeclarationResourceIdentityErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises( ResourceIdentityError, match=test_case.expected_error_fragment, ) as error_info: - parse_enum_declaration_file( - contents=test_case.contents, - file_path=Path("enums/domain.sql"), - relative_path=Path("enums/domain.sql"), + discover_declaration_file( + project_dir=tmp_path, relative_path="enums/domain.sql", contents=test_case.contents ) assert error_info.value.code == "D016" @@ -366,12 +372,14 @@ def test_given_noncanonical_declaration_identity_when_parsing_then_d016_suggests ) def test_given_model_schema_declarations_when_parsing_then_returns_typed_columns( test_case: ParseModelSchemaDeclarationTestCase, + tmp_path: Path, ) -> None: - declarations: tuple[ModelSchemaDeclaration, ...] = parse_model_schema_declaration_file( - contents=test_case.contents, - file_path=Path("schemas/orders.sql"), - relative_path=Path("schemas/orders.sql"), - ) + declarations: tuple[ModelSchemaDeclaration, ...] = cast( + DiscoveredModelSchemaFile, + discover_declaration_file( + project_dir=tmp_path, relative_path="schemas/orders.sql", contents=test_case.contents + ), + ).declarations assert tuple(declaration.name for declaration in declarations) == test_case.expected_names assert declarations[0].description == test_case.expected_description @@ -405,12 +413,11 @@ def test_given_model_schema_declarations_when_parsing_then_returns_typed_columns ) def test_given_empty_model_schema_when_parsing_then_raises_declaration_error( test_case: ParseDeclarationFileErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(DeclarationParseError, match=test_case.expected_error_fragment): - parse_model_schema_declaration_file( - contents=test_case.contents, - file_path=Path("schemas/empty.sql"), - relative_path=Path("schemas/empty.sql"), + discover_declaration_file( + project_dir=tmp_path, relative_path="schemas/empty.sql", contents=test_case.contents ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py index 90fdb65d04..a90d183b86 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_hooks.py @@ -2,11 +2,11 @@ from collections.abc import Callable from pathlib import Path +from typing import cast import pytest from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_sql_hook_files -from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file from sqlbuild.compiler.discovery.exceptions import SqlHookParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( @@ -14,6 +14,7 @@ ParseSqlHookErrorTestCase, ParseSqlHookTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import discover_declaration_file @pytest.mark.parametrize( @@ -191,11 +192,15 @@ ) def test_given_hook_header_and_sql_when_parsing_then_returns_named_resource( test_case: ParseSqlHookTestCase, + tmp_path: Path, ) -> None: - hook: DiscoveredSqlHookFile = parse_sql_hook_file( - contents=test_case.contents, - file_path=Path("/project/hooks/sql/grant_access.sql"), - relative_path=Path("hooks/sql/grant_access.sql"), + hook: DiscoveredSqlHookFile = cast( + DiscoveredSqlHookFile, + discover_declaration_file( + project_dir=tmp_path, + relative_path="hooks/sql/grant_access.sql", + contents=test_case.contents, + ), ) assert hook.name == test_case.expected_name @@ -276,12 +281,11 @@ def test_given_nested_sql_hook_files_when_discovering_then_names_come_from_stems ) def test_given_invalid_sql_hook_when_parsing_then_raises( test_case: ParseSqlHookErrorTestCase, + tmp_path: Path, ) -> None: with pytest.raises(SqlHookParseError, match=test_case.expected_error_fragment): - parse_sql_hook_file( - contents=test_case.contents, - file_path=Path("hooks/sql/invalid.sql"), - relative_path=Path("hooks/sql/invalid.sql"), + discover_declaration_file( + project_dir=tmp_path, relative_path="hooks/sql/invalid.sql", contents=test_case.contents ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py index f4a0a486dc..c3e4a1b849 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_statement_header_keys.py @@ -11,7 +11,10 @@ from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( StatementHeaderKeysTestCase, ) -from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import parse_statement_header_file +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( + parse_statement_header_file, + statement_header_file_path, +) @pytest.mark.parametrize( @@ -99,14 +102,17 @@ ids=lambda case: case.description, ) def test_given_unsupported_header_key_when_parsing_then_rejects_with_location_and_help( - test_case: StatementHeaderKeysTestCase, + test_case: StatementHeaderKeysTestCase, tmp_path: Path ) -> None: + file_path: Path = statement_header_file_path( + statement=test_case.file_name, project_dir=tmp_path + ) with pytest.raises(DiscoveryError) as raised: parse_statement_header_file( - statement=test_case.file_name, contents=test_case.contents, file_path=Path("orders.sql") + statement=test_case.file_name, contents=test_case.contents, file_path=file_path ) - assert raised.value.message == test_case.expected_error + assert raised.value.message == test_case.expected_error.replace("'orders.sql", f"'{file_path}") assert raised.value.help == test_case.expected_help @@ -127,14 +133,17 @@ def test_given_unsupported_header_key_when_parsing_then_rejects_with_location_an ids=lambda case: case.description, ) def test_given_removed_model_key_when_parsing_then_explains_removal( - test_case: StatementHeaderKeysTestCase, + test_case: StatementHeaderKeysTestCase, tmp_path: Path ) -> None: + file_path: Path = statement_header_file_path( + statement=test_case.file_name, project_dir=tmp_path + ) with pytest.raises(DiscoveryError) as raised: parse_statement_header_file( - statement=test_case.file_name, contents=test_case.contents, file_path=Path("orders.sql") + statement=test_case.file_name, contents=test_case.contents, file_path=file_path ) - assert raised.value.message == test_case.expected_error + assert raised.value.message == test_case.expected_error.replace("'orders.sql", f"'{file_path}") assert raised.value.code == "D002" From 102c76ae3e0ef840cf8e741370c38e42ff58f32e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:43:19 +0100 Subject: [PATCH 04/40] feat(native): parse macro arguments natively with located errors and help; delete the Python argument parser --- .../_helpers/checks/python_values.rs | 10 +- .../_helpers/parsing/audit_files.rs | 34 +- .../_helpers/parsing/constant_files.rs | 15 +- .../_helpers/parsing/enum_files.rs | 12 +- .../_helpers/parsing/hook_files.rs | 22 +- .../_helpers/parsing/schema_files.rs | 11 +- .../main/parse_declaration_text.rs | 10 +- .../src/templates/_helpers/evaluation.rs | 2 +- .../src/templates/_helpers/tokens.rs | 2 +- .../templates/main/template_error_message.rs | 4 +- .../tests/test_expand_template_string.rs | 6 +- .../_helpers/model_config/config_templates.rs | 27 +- .../_helpers/render/macro_arguments.rs | 118 ++++++ .../src/bindings/_helpers/render/mod.rs | 1 + .../sqltext/declaration_references.rs | 2 +- .../src/bindings/main/register.rs | 1 + crates/sqlbuild-render/src/lib.rs | 3 +- .../macro_arguments/_helpers/evaluation.rs | 202 +++++++++ .../src/macro_arguments/_helpers/failures.rs | 107 +++++ .../src/macro_arguments/_helpers/lexer.rs | 190 +++++++++ .../src/macro_arguments/_helpers/literals.rs | 284 +++++++++++++ .../src/macro_arguments/_helpers/mod.rs | 5 + .../macro_arguments/_helpers/syntax_tree.rs | 385 ++++++++++++++++++ .../src/macro_arguments/main/mod.rs | 1 + .../main/parse_macro_arguments.rs | 41 ++ .../src/macro_arguments/mod.rs | 7 + .../src/macro_arguments/models.rs | 51 +++ .../src/macro_arguments/tests.rs | 6 + .../src/macro_arguments/tests/helpers.rs | 68 ++++ .../tests/test_parse_macro_arguments.rs | 202 +++++++++ .../src/macro_arguments/tests/test_types.rs | 7 + .../src/macro_arguments/types.rs | 9 + .../compiler/main/declaration_references.rs | 4 +- .../_helpers/corpus/failure_cases.py | 79 ++++ src/sqlbuild/_native.pyi | 1 + .../explicit_references/macro_arguments.py | 27 +- .../_helpers/render/macro_argument_values.py | 80 ++++ .../compile/_helpers/render/macros.py | 174 +------- src/sqlbuild/compiler/compile/constants.py | 1 - src/sqlbuild/compiler/compile/models.py | 10 + src/sqlbuild/compiler/frontier/types.py | 1 - .../_helpers/test_typed_macro_references.py | 6 +- 42 files changed, 1998 insertions(+), 230 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/main/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/mod.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/models.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/tests.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/types.rs create mode 100644 src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs index edc18ef323..a2f8922839 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs @@ -29,7 +29,10 @@ pub(crate) enum PythonType { } /// The projected type; a bare number written with non-ASCII digits is rejected. -pub(crate) fn python_type(value: &AuthoredValue, words: WordRules) -> Result { +pub(crate) fn python_type( + value: &AuthoredValue, + words: WordRules, +) -> Result { Ok(match value { AuthoredValue::Null => PythonType::None, AuthoredValue::Boolean(_) => PythonType::Bool, @@ -151,7 +154,10 @@ fn is_float(word: &str) -> bool { /// Whether Python's `^[+-]?\d+$` or float pattern, with Unicode `\d`, matches a non-ASCII word. fn is_unicode_number(word: &str, python: PythonText) -> bool { - let digits = |part: &str| part.chars().all(|character| is_python_decimal(python, character)); + let digits = |part: &str| { + part.chars() + .all(|character| is_python_decimal(python, character)) + }; let unsigned = unsigned(word); if !unsigned.is_empty() && digits(unsigned) { return true; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs index 0b45e63cde..e6225e3603 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs @@ -2,17 +2,16 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, - PythonType, failure, get, non_empty_str, python_str, python_type, + PythonType, WordRules, failure, get, non_empty_str, python_str, python_type, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::audit_text::{AuditText, chars, text}; use crate::declaration_files::models::{AuditBlock, AuditFile, DeclarationFileOptions}; use crate::models::FailureKind; -use sqlbuild_core::text::models::PythonText; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const AUDIT_KEYWORD: &str = "AUDIT"; @@ -115,7 +114,8 @@ impl AuditParse<'_> { .filter(|character| **character == '\n') .count(); let header_values: Vec<(String, AuthoredValue)> = self.header(&header, header_line)?; - let evaluation_mode: &'static str = evaluation_mode(&header_values, file_path, self.options.python)?; + let evaluation_mode: &'static str = + evaluation_mode(&header_values, file_path, self.options.python)?; let body: &[char] = &block_chars[body_start..]; let mut measure_sql: Option = None; let mut evidence_sql: Option = None; @@ -145,7 +145,14 @@ impl AuditParse<'_> { sql_body }; let name: Option = match get(&header_values, "name") { - Some(value) => python_str(value, WordRules { python: self.options.python, file_path })?.map(str::to_owned), + Some(value) => python_str( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? + .map(str::to_owned), None => None, }; Ok(AuditBlock { @@ -179,7 +186,14 @@ impl AuditParse<'_> { )?; for key in STRING_KEYS { if let Some(value) = get(&values, key) - && non_empty_str(value, WordRules { python: self.options.python, file_path })?.is_none() + && non_empty_str( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? + .is_none() { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a non-empty string" @@ -188,7 +202,13 @@ impl AuditParse<'_> { } for key in BOOLEAN_KEYS { if let Some(value) = get(&values, key) - && python_type(value, WordRules { python: self.options.python, file_path })? != PythonType::Bool + && python_type( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? != PythonType::Bool { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a boolean" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs index cebb8abc7f..b6a5ba6f24 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs @@ -1,8 +1,7 @@ //! Python's `parse_constant_declaration_file` up to value normalisation, which Python performs. use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, - failure, get, python_str, unknown_keys, + WordRules, failure, get, python_str, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; @@ -82,12 +81,18 @@ fn parse_constant( "{file_path} constant '{name}' is missing required value" ))); }; - let explicit_type: Option = - constant_option(explicit_type, file_path, &format!("constant '{name}' type"), python)?; + let explicit_type: Option = constant_option( + explicit_type, + file_path, + &format!("constant '{name}' type"), + python, + )?; let render_as: Option = constant_option( render_as, file_path, - &format!("constant '{name}' render_as"), python)?; + &format!("constant '{name}' render_as"), + python, + )?; if let Some(rendering) = &render_as && !COLLECTION_RENDERINGS.contains(&rendering.as_str()) { diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs index 5e9cf8d3d4..6364dc4be7 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs @@ -2,8 +2,7 @@ use crate::declaration_files::_helpers::checks::identities::validate_public_identity; use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, - PythonType, failure, get, is_identifier, python_str, python_type, unknown_keys, + PythonType, WordRules, failure, get, is_identifier, python_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; @@ -118,7 +117,9 @@ fn shorthand_members( ) -> Result, ParseStop> { let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); for item in items { - let Some(member) = python_str(item, WordRules { python, file_path })?.filter(|member| is_identifier(member)) else { + let Some(member) = python_str(item, WordRules { python, file_path })? + .filter(|member| is_identifier(member)) + else { return Err(declaration(format!( "{file_path} enum '{enum_name}' shorthand members must be identifiers" ))); @@ -146,7 +147,10 @@ fn explicit_members( "{file_path} enum '{enum_name}' member name must be a SQL identifier" ))); } - if !matches!(python_type(value, WordRules { python, file_path })?, PythonType::Str | PythonType::Int) { + if !matches!( + python_type(value, WordRules { python, file_path })?, + PythonType::Str | PythonType::Int + ) { return Err(declaration(format!( "{file_path} enum '{enum_name}' member '{member}' value must be a string or \ integer" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs index b52ec5bf9b..0a989bceb4 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs @@ -2,8 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, - failure, get, non_empty_str, python_str, + WordRules, failure, get, non_empty_str, python_str, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::models::{DeclarationFileOptions, HookFile}; @@ -42,12 +41,27 @@ pub(crate) fn parse_hook_file( )?; let description: Option = match get(&header_values, DESCRIPTION_KEY) { Some(value) => { - if non_empty_str(value, WordRules { python: options.python, file_path })?.is_none() { + if non_empty_str( + value, + WordRules { + python: options.python, + file_path, + }, + )? + .is_none() + { return Err(hook_failure(format!( "HOOK() description in '{file_path}' must be a non-empty string" ))); } - python_str(value, WordRules { python: options.python, file_path })?.map(str::to_owned) + python_str( + value, + WordRules { + python: options.python, + file_path, + }, + )? + .map(str::to_owned) } None => None, }; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs index e632f288f3..94feecee54 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs @@ -2,8 +2,7 @@ use crate::_helpers::locations::header_column_locations; use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, - PythonType, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, + PythonType, WordRules, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; @@ -71,11 +70,15 @@ fn parse_schema( let description: Option = optional_string( get(values, "description"), file_path, - &format!("schema '{name}' description"), python)?; + &format!("schema '{name}' description"), + python, + )?; let extends: Option = optional_string( get(values, "extends"), file_path, - &format!("schema '{name}' extends"), python)?; + &format!("schema '{name}' extends"), + python, + )?; if let Some(parent) = &extends && !is_identifier(parent) { diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs index 67c0936bfd..efac1f59ec 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs @@ -33,12 +33,10 @@ fn parse_text( CollectionKind::Enums => { parse_enum_file(file_path, contents, options.python).map(ParsedDeclarationText::Enum) } - CollectionKind::Constants => { - parse_constant_file(file_path, contents, options.python).map(ParsedDeclarationText::Constant) - } - CollectionKind::ModelSchemas => { - parse_schema_file(file_path, contents, options.python).map(ParsedDeclarationText::ModelSchema) - } + CollectionKind::Constants => parse_constant_file(file_path, contents, options.python) + .map(ParsedDeclarationText::Constant), + CollectionKind::ModelSchemas => parse_schema_file(file_path, contents, options.python) + .map(ParsedDeclarationText::ModelSchema), CollectionKind::SqlFunctions => parse_function_file(file_path, contents, options) .map(ParsedDeclarationText::SqlFunction), CollectionKind::SqlHooks => parse_hook_file(file_path, hook_name, contents, options) diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs index a297ddc3cf..305c7461fa 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs @@ -4,11 +4,11 @@ use crate::templates::errors::TemplateError; use crate::templates::main::template_error_message::{ is_missing_value_message, template_error_message, }; -use sqlbuild_core::text::main::python_strip::python_strip; use crate::templates::models::{ ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, }; use crate::templates::types::TemplateHost; +use sqlbuild_core::text::main::python_strip::python_strip; const TRUE_LITERAL: &str = "true"; const FALSE_LITERAL: &str = "false"; diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs index f3bef193f1..44be12fc4e 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs @@ -1,8 +1,8 @@ //! The Python template tokenizer. use crate::templates::errors::TemplateError; -use sqlbuild_core::text::main::is_python_space::is_python_space; use crate::templates::models::TemplateFailure; +use sqlbuild_core::text::main::is_python_space::is_python_space; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum TokenKind { diff --git a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs index 47c533b600..1c7578e44c 100644 --- a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs +++ b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs @@ -47,7 +47,9 @@ pub fn template_error_message(error: &TemplateError, label: &str) -> String { /// Whether Python's `coalesce` treats this error message as a missing value and skips it. pub fn is_missing_value_message(message: &str) -> bool { - MISSING_VALUE_PARTS.iter().any(|part| message.contains(part)) + MISSING_VALUE_PARTS + .iter() + .any(|part| message.contains(part)) || (message.contains(UNAVAILABLE_CONTEXT_PART) && message.contains(NO_VALUE_PART)) } diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs index 3e6997757e..ef8c06c86c 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs @@ -179,9 +179,9 @@ fn given_template_strings_when_expanding_then_values_and_reads_match_python() { TemplateTestCase { description: "coalesce stops at an error that is not a missing value", text: "${coalesce(other:thing, env)}", - expected: Err(TemplateFailure::Invalid(TemplateError::UnsupportedNamespace( - "other".to_owned(), - ))), + expected: Err(TemplateFailure::Invalid( + TemplateError::UnsupportedNamespace("other".to_owned()), + )), expected_reads: &[], }, TemplateTestCase { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 0f0a440a08..cc48cba2bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -5,8 +5,8 @@ use std::cell::RefCell; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ - PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, - PyStringMethods, PyTuple, PyTupleMethods, + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, + PyTuple, PyTupleMethods, }; use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::ConfigError; @@ -26,7 +26,8 @@ const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; const EFFECTIVE_VARS_LABEL: &str = "effective vars"; -const PROJECT_VAR_VALUES_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; +const PROJECT_VAR_VALUES_MODULE: &str = + "sqlbuild.compiler.authored_values.main._project_var_values"; const RENDER_PROJECT_VAR_TEXT: &str = "render_project_var_text"; /// Variables, the process environment and context values, all read through Python. @@ -75,9 +76,11 @@ impl<'py> PythonHost<'py> { TemplateFailure::Missing(error) | TemplateFailure::Invalid(error) => { Stop::Failure(error) } - TemplateFailure::Host => Stop::Python(self.raised.borrow_mut().take().unwrap_or_else( - || PyRuntimeError::new_err("template expansion stopped without an error"), - )), + TemplateFailure::Host => { + Stop::Python(self.raised.borrow_mut().take().unwrap_or_else(|| { + PyRuntimeError::new_err("template expansion stopped without an error") + })) + } } } @@ -96,7 +99,12 @@ impl<'py> PythonHost<'py> { effective: &EffectiveVars<'py>, name: &str, ) -> Result>, TemplateFailure> { - let Some(raw) = self.sources.0.get_item(name).map_err(|error| self.hold(error))? else { + let Some(raw) = self + .sources + .0 + .get_item(name) + .map_err(|error| self.hold(error))? + else { return Ok(None); }; if let Some(value) = effective @@ -176,7 +184,10 @@ impl<'py> TemplateHost for PythonHost<'py> { .borrow_mut() .push((ENVIRONMENT_READ, name.to_owned())); let environment: &Bound<'py, PyAny> = &self.sources.1; - if environment.contains(name).map_err(|error| self.hold(error))? { + if environment + .contains(name) + .map_err(|error| self.hold(error))? + { environment .get_item(name) .map(Some) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs new file mode 100644 index 0000000000..b16754c22b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs @@ -0,0 +1,118 @@ +//! Macro call arguments parsed natively into a value plan Python builds objects from. + +use pyo3::prelude::{ + Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyList, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments as parse; +use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; +use sqlbuild_render::macro_arguments::types::ArgumentHost; + +/// `unicodedata` answers character names and identifier normalization as CPython does. +struct UnicodeHost<'py> { + unicodedata: Bound<'py, PyModule>, +} + +impl ArgumentHost for UnicodeHost<'_> { + fn character_named(&self, name: &str) -> Option { + self.unicodedata + .call_method1("lookup", (name,)) + .and_then(|found| found.extract::()) + .ok() + } + + fn identifier(&self, text: &str) -> Option { + let py: Python<'_> = self.unicodedata.py(); + let candidate = text.into_pyobject(py).ok()?; + let valid: bool = candidate + .call_method0("isidentifier") + .and_then(|valid| valid.extract()) + .ok()?; + if !valid { + return None; + } + self.unicodedata + .call_method1("normalize", ("NFKC", candidate)) + .and_then(|normalized| normalized.extract()) + .ok() + } +} + +/// `("ok", positional, keywords, typed references)` or `("error", detail, help, line, column)`. +#[pyfunction] +fn parse_macro_arguments( + py: Python<'_>, + text: &str, + nested: Vec<(usize, usize)>, +) -> PyResult> { + let host = UnicodeHost { + unicodedata: py.import("unicodedata")?, + }; + match parse(&host, text, &nested) { + Ok(arguments) => arguments_row(py, &arguments), + Err(error) => Ok( + ("error", error.detail, error.help, error.line, error.column) + .into_pyobject(py)? + .into_any() + .unbind(), + ), + } +} + +fn arguments_row(py: Python<'_>, arguments: &MacroArguments) -> PyResult> { + let positional = values_list(py, &arguments.positional)?; + let keywords = PyList::empty(py); + for (name, value) in &arguments.keywords { + keywords.append((name, value_row(py, value)?))?; + } + Ok(( + "ok", + positional, + keywords, + arguments.typed_references.clone(), + ) + .into_pyobject(py)? + .into_any() + .unbind()) +} + +fn values_list<'py>(py: Python<'py>, values: &[ArgumentValue]) -> PyResult> { + let list = PyList::empty(py); + for value in values { + list.append(value_row(py, value)?)?; + } + Ok(list) +} + +/// One value as a tagged tuple: the tag, then its fields. +fn value_row<'py>(py: Python<'py>, value: &ArgumentValue) -> PyResult> { + let row: Bound<'py, PyTuple> = match value { + ArgumentValue::Str(text) => ("s", text).into_pyobject(py)?, + ArgumentValue::Int { radix, digits } => ("i", radix, digits).into_pyobject(py)?, + ArgumentValue::Float(text) => ("f", text).into_pyobject(py)?, + ArgumentValue::Bool(flag) => ("b", flag).into_pyobject(py)?, + ArgumentValue::None => ("n",).into_pyobject(py)?, + ArgumentValue::NestedCall(call) => ("c", call).into_pyobject(py)?, + ArgumentValue::TypedReference { function, name } => { + ("r", function, name).into_pyobject(py)? + } + ArgumentValue::List(items) => ("l", values_list(py, items)?).into_pyobject(py)?, + ArgumentValue::Tuple(items) => ("t", values_list(py, items)?).into_pyobject(py)?, + ArgumentValue::Dict(pairs) => { + let rows = PyList::empty(py); + for (key, item) in pairs { + rows.append((value_row(py, key)?, value_row(py, item)?))?; + } + ("d", rows).into_pyobject(py)? + } + ArgumentValue::Negative(inner) => ("-", value_row(py, inner)?).into_pyobject(py)?, + ArgumentValue::Positive(inner) => ("+", value_row(py, inner)?).into_pyobject(py)?, + }; + Ok(row.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_macro_arguments, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 2a04cacf7b..9fe00bacbe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod macro_arguments; pub(crate) mod macro_call_memo; pub(crate) mod macro_call_rows; pub(crate) mod macro_calls; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index 6313a42b6c..384070befe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -2,8 +2,8 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use sqlbuild_core::text::main::python_text::python_text; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 9e1dce2fcb..f4cd61114b 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -31,6 +31,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; + render::macro_arguments::register(module)?; render::macro_calls::register(module)?; cache::registration::register(module)?; attachments::registration::register(module)?; diff --git a/crates/sqlbuild-render/src/lib.rs b/crates/sqlbuild-render/src/lib.rs index d2567e5538..3c534852b1 100644 --- a/crates/sqlbuild-render/src/lib.rs +++ b/crates/sqlbuild-render/src/lib.rs @@ -1,5 +1,6 @@ -//! Native rendering: macro call sites, splicing of macro results and the in-compile call memo. +//! Native rendering: macro call sites and arguments, splicing of results and the call memo. #![forbid(unsafe_code)] +pub mod macro_arguments; pub mod macro_calls; diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs new file mode 100644 index 0000000000..bb2ecdd966 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/evaluation.rs @@ -0,0 +1,202 @@ +//! Evaluate parsed nodes in Python's order into argument values, and walk typed references. + +use std::collections::VecDeque; + +use crate::macro_arguments::_helpers::failures::{ + Failure, bytes_literal, complex_literal, dict_unpacking, ellipsis, formatted_string, + keyword_expansion, typed_reference, unary, unsupported, +}; +use crate::macro_arguments::_helpers::lexer::NumberToken; +use crate::macro_arguments::_helpers::syntax_tree::{CallArguments, Expr, Keyword, Node}; +use crate::macro_arguments::models::{ArgumentValue, MacroArguments}; + +const TYPED_REFERENCE_FUNCTIONS: [&str; 3] = ["__ref", "__source", "__seed"]; + +/// Positional values, then keyword values, failing at the first node Python would reject. +pub(crate) fn evaluate_arguments(arguments: &CallArguments) -> Result { + let positional: Vec = arguments + .args + .iter() + .map(evaluate) + .collect::>()?; + let mut keywords: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.keywords.len()); + for keyword in &arguments.keywords { + let Some(name) = &keyword.name else { + return Err(keyword_expansion(keyword.value.position)); + }; + keywords.push((name.clone(), evaluate(&keyword.value)?)); + } + Ok(MacroArguments { + positional, + keywords, + typed_references: walked_references(arguments), + }) +} + +fn evaluate(node: &Node) -> Result { + let position: usize = node.position; + match &node.expr { + Expr::Str { bytes: true, .. } => Err(bytes_literal(position)), + Expr::Str { + formatted: true, .. + } => Err(formatted_string(position)), + Expr::Str { value, .. } => Ok(ArgumentValue::Str(value.clone())), + Expr::Number(NumberToken::Int { radix, digits }) => Ok(ArgumentValue::Int { + radix: *radix, + digits: digits.clone(), + }), + Expr::Number(NumberToken::Float(text)) => Ok(ArgumentValue::Float(text.clone())), + Expr::Number(NumberToken::Complex) => Err(complex_literal(position)), + Expr::Ellipsis => Err(ellipsis(position)), + Expr::Name(name) => match name.as_str() { + "True" => Ok(ArgumentValue::Bool(true)), + "False" => Ok(ArgumentValue::Bool(false)), + "None" => Ok(ArgumentValue::None), + _ => Err(unsupported(position)), + }, + Expr::Nested(call) => Ok(ArgumentValue::NestedCall(*call)), + Expr::Call { + function, + args, + keywords, + } => typed_reference_value(function, args, keywords, position), + Expr::List(items) => Ok(ArgumentValue::List(evaluate_all(items)?)), + Expr::Tuple(items) => Ok(ArgumentValue::Tuple(evaluate_all(items)?)), + Expr::Dict(pairs) => { + let mut values: Vec<(ArgumentValue, ArgumentValue)> = Vec::with_capacity(pairs.len()); + for (key, value) in pairs { + let Some(key) = key else { + return Err(dict_unpacking(value.position)); + }; + values.push((evaluate(key)?, evaluate(value)?)); + } + Ok(ArgumentValue::Dict(values)) + } + Expr::Set(_) | Expr::Starred(_) => Err(unsupported(position)), + Expr::Unary { negative, operand } => { + let value: ArgumentValue = evaluate(operand)?; + if !is_number(&value) { + return Err(unary(position)); + } + Ok(if *negative { + ArgumentValue::Negative(Box::new(value)) + } else { + ArgumentValue::Positive(Box::new(value)) + }) + } + } +} + +fn evaluate_all(items: &[Node]) -> Result, Failure> { + items.iter().map(evaluate).collect() +} + +/// Python's `isinstance(value, int | float)`, deferring a nested call's value to Python. +fn is_number(value: &ArgumentValue) -> bool { + match value { + ArgumentValue::Int { .. } + | ArgumentValue::Float(_) + | ArgumentValue::Bool(_) + | ArgumentValue::NestedCall(_) => true, + ArgumentValue::Negative(inner) | ArgumentValue::Positive(inner) => is_number(inner), + _ => false, + } +} + +fn typed_reference_value( + function: &Node, + args: &[Node], + keywords: &[Keyword], + position: usize, +) -> Result { + let Expr::Name(name) = &function.expr else { + return Err(unsupported(position)); + }; + if !TYPED_REFERENCE_FUNCTIONS.contains(&name.as_str()) { + return Err(unsupported(position)); + } + match (args, keywords) { + ( + [ + Node { + expr: + Expr::Str { + value, + bytes: false, + formatted: false, + }, + .. + }, + ], + [], + ) if !value.is_empty() => Ok(ArgumentValue::TypedReference { + function: name.clone(), + name: value.clone(), + }), + _ => Err(typed_reference(name, position)), + } +} + +enum Walked<'tree> { + Node(&'tree Node), + Keyword(&'tree Keyword), +} + +/// Every typed reference call in `ast.walk` order: breadth first, fields in `_fields` order. +fn walked_references(arguments: &CallArguments) -> Vec<(String, String)> { + let mut pending: VecDeque> = arguments + .args + .iter() + .map(Walked::Node) + .chain(arguments.keywords.iter().map(Walked::Keyword)) + .collect(); + let mut references: Vec<(String, String)> = Vec::new(); + while let Some(walked) = pending.pop_front() { + let node: &Node = match walked { + Walked::Keyword(keyword) => { + pending.push_back(Walked::Node(&keyword.value)); + continue; + } + Walked::Node(node) => node, + }; + match &node.expr { + Expr::Call { + function, + args, + keywords, + } => { + if let ( + Expr::Name(name), + [ + Node { + expr: Expr::Str { value, .. }, + .. + }, + ], + ) = (&function.expr, args.as_slice()) + { + references.push((name.clone(), value.clone())); + } + pending.push_back(Walked::Node(function)); + pending.extend(args.iter().map(Walked::Node)); + pending.extend(keywords.iter().map(Walked::Keyword)); + } + Expr::List(items) | Expr::Tuple(items) | Expr::Set(items) => { + pending.extend(items.iter().map(Walked::Node)); + } + Expr::Dict(pairs) => { + pending.extend( + pairs + .iter() + .filter_map(|(key, _)| key.as_ref().map(Walked::Node)), + ); + pending.extend(pairs.iter().map(|(_, value)| Walked::Node(value))); + } + Expr::Starred(inner) | Expr::Unary { operand: inner, .. } => { + pending.push_back(Walked::Node(inner)); + } + _ => {} + } + } + references +} diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs new file mode 100644 index 0000000000..01f1cad354 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs @@ -0,0 +1,107 @@ +//! The errors argument parsing reports, each with its help. + +const LITERALS_HELP: &str = "Macro arguments are Python literals (strings, numbers, True, False, \ + None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() \ + references; compute anything else inside the macro"; +const BYTES_HELP: &str = + "Pass text as a string without the b prefix, for example 'orders' instead of b'orders'"; +const COMPLEX_HELP: &str = + "Pass int or float numbers; give a complex value's real and imaginary parts as two arguments"; +const ELLIPSIS_HELP: &str = "Pass None, or a string the macro understands, instead of '...'"; +const FORMATTED_HELP: &str = "Pass the parts as separate arguments and format them in the macro"; +const REFERENCE_HELP: &str = "Write one quoted resource name, for example __ref(\"orders\")"; +const UNPACKING_HELP: &str = "Write each argument or key out explicitly"; +const KEYWORD_HELP: &str = "Pass each keyword argument once, after every positional argument"; +const SURROGATE_HELP: &str = "Lone surrogates are not valid Unicode text and cannot be sent to a \ + warehouse; write the character itself or the escape of a valid code point"; + +/// One argument error at a code point position of the argument text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Failure { + pub(crate) detail: String, + pub(crate) help: &'static str, + pub(crate) position: usize, +} + +fn failure(detail: &str, help: &'static str, position: usize) -> Failure { + Failure { + detail: detail.to_owned(), + help, + position, + } +} + +/// Text Python cannot tokenize or parse. +pub(crate) fn syntax(detail: &str, position: usize) -> Failure { + failure( + &format!("could not be parsed: {detail}"), + LITERALS_HELP, + position, + ) +} + +/// Valid Python that is not a supported argument value. +pub(crate) fn unsupported(position: usize) -> Failure { + failure( + "must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() \ + references", + LITERALS_HELP, + position, + ) +} + +pub(crate) fn bytes_literal(position: usize) -> Failure { + failure("use a bytes literal", BYTES_HELP, position) +} + +pub(crate) fn complex_literal(position: usize) -> Failure { + failure("use a complex number literal", COMPLEX_HELP, position) +} + +pub(crate) fn ellipsis(position: usize) -> Failure { + failure("use '...' (Ellipsis)", ELLIPSIS_HELP, position) +} + +pub(crate) fn formatted_string(position: usize) -> Failure { + failure("use an f-string", FORMATTED_HELP, position) +} + +pub(crate) fn typed_reference(function: &str, position: usize) -> Failure { + failure( + &format!("must give {function}() exactly one quoted resource name"), + REFERENCE_HELP, + position, + ) +} + +pub(crate) fn unary(position: usize) -> Failure { + failure( + "use unary + or - on a value that is not a number", + LITERALS_HELP, + position, + ) +} + +pub(crate) fn keyword_expansion(position: usize) -> Failure { + failure( + "must not use **kwargs expansion syntax", + UNPACKING_HELP, + position, + ) +} + +pub(crate) fn dict_unpacking(position: usize) -> Failure { + failure("must not use dict unpacking", UNPACKING_HELP, position) +} + +pub(crate) fn keyword_order(detail: &str, position: usize) -> Failure { + failure(detail, KEYWORD_HELP, position) +} + +pub(crate) fn surrogate(escape: &str, position: usize) -> Failure { + failure( + &format!("contain the lone surrogate escape '{escape}'"), + SURROGATE_HELP, + position, + ) +} diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs new file mode 100644 index 0000000000..7842d856da --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs @@ -0,0 +1,190 @@ +//! Python's tokenizer for the literal subset macro arguments use, over code points. + +use crate::macro_arguments::_helpers::failures::{Failure, syntax}; +use crate::macro_arguments::_helpers::literals::{number, string_body}; +use crate::macro_arguments::types::ArgumentHost; + +const STRING_PREFIXES: [&str; 8] = ["r", "u", "b", "f", "br", "rb", "fr", "rf"]; +const OPERATORS: [&str; 15] = [ + "**", "...", "(", ")", "[", "]", "{", "}", ",", ":", "=", "*", "-", "+", ".", +]; + +/// One literal string piece; adjacent pieces concatenate. +#[derive(Clone, Debug)] +pub(crate) struct StringPiece { + pub(crate) value: String, + pub(crate) bytes: bool, + pub(crate) formatted: bool, +} + +/// A number token as Python reads it. +#[derive(Clone, Debug)] +pub(crate) enum NumberToken { + Int { radix: u32, digits: String }, + Float(String), + Complex, +} + +#[derive(Clone, Debug)] +pub(crate) enum TokenKind { + Str(StringPiece), + Number(NumberToken), + Name(String), + Nested(usize), + Operator(&'static str), + /// Any other operator, which no supported argument uses. + Other, + End, +} + +#[derive(Clone, Debug)] +pub(crate) struct Token { + pub(crate) kind: TokenKind, + pub(crate) position: usize, +} + +/// Tokenize `text`; `nested` are the `[start, end)` code point spans of nested macro calls. +pub(crate) fn tokenize( + host: &dyn ArgumentHost, + text: &[char], + nested: &[(usize, usize)], +) -> Result, Failure> { + let mut tokens: Vec = Vec::new(); + let mut index: usize = 0; + while index < text.len() { + if let Some(call) = nested.iter().position(|(start, _)| *start == index) { + tokens.push(Token { + kind: TokenKind::Nested(call), + position: index, + }); + index = nested[call].1; + continue; + } + let character: char = text[index]; + match character { + ' ' | '\t' | '\x0c' | '\n' | '\r' => index += 1, + '#' => { + while index < text.len() && !matches!(text[index], '\n' | '\r') { + index += 1; + } + } + '\\' => index = line_continuation(text, index)?, + '\'' | '"' => { + let (piece, end) = string_body(host, text, index, "")?; + tokens.push(Token { + kind: TokenKind::Str(piece), + position: index, + }); + index = end; + } + '0'..='9' => index = push_number(text, index, &mut tokens)?, + '.' if text.get(index + 1).is_some_and(char::is_ascii_digit) => { + index = push_number(text, index, &mut tokens)?; + } + _ if is_name_start(character) => index = push_name(host, text, index, &mut tokens)?, + _ => index = push_operator(text, index, &mut tokens)?, + } + } + tokens.push(Token { + kind: TokenKind::End, + position: text.len(), + }); + Ok(tokens) +} + +fn line_continuation(text: &[char], index: usize) -> Result { + match (text.get(index + 1), text.get(index + 2)) { + (Some('\n'), _) => Ok(index + 2), + (Some('\r'), Some('\n')) => Ok(index + 3), + (Some('\r'), _) => Ok(index + 2), + _ => Err(syntax("a '\\' outside a string must end the line", index)), + } +} + +fn push_number(text: &[char], index: usize, tokens: &mut Vec) -> Result { + let (token, end) = number(text, index)?; + tokens.push(Token { + kind: TokenKind::Number(token), + position: index, + }); + Ok(end) +} + +fn push_name( + host: &dyn ArgumentHost, + text: &[char], + index: usize, + tokens: &mut Vec, +) -> Result { + let mut end: usize = index; + while end < text.len() && is_name_continue(text[end]) { + end += 1; + } + let word: String = text[index..end].iter().collect(); + let lowered: String = word.to_ascii_lowercase(); + if matches!(text.get(end), Some('\'' | '"')) && STRING_PREFIXES.contains(&lowered.as_str()) { + let (piece, string_end) = string_body(host, text, end, &lowered)?; + tokens.push(Token { + kind: TokenKind::Str(piece), + position: index, + }); + return Ok(string_end); + } + let name: String = if word.is_ascii() { + word + } else { + host.identifier(&word).ok_or_else(|| { + let invalid: char = text[index..end] + .iter() + .copied() + .find(|character| { + !character.is_ascii() && host.identifier(&format!("a{character}")).is_none() + }) + .unwrap_or(text[index]); + syntax( + &format!( + "the character '{invalid}' (U+{:04X}) is not valid in a name", + u32::from(invalid) + ), + index, + ) + })? + }; + tokens.push(Token { + kind: TokenKind::Name(name), + position: index, + }); + Ok(end) +} + +fn push_operator(text: &[char], index: usize, tokens: &mut Vec) -> Result { + if text[index] == '\0' { + return Err(syntax("a NUL character is not allowed", index)); + } + let rest: String = text[index..text.len().min(index + 3)].iter().collect(); + let kind: TokenKind = OPERATORS + .iter() + .find(|operator| rest.starts_with(**operator)) + .map_or_else( + || TokenKind::Other, + |operator| TokenKind::Operator(operator), + ); + let width: usize = match &kind { + TokenKind::Operator(operator) => operator.len(), + _ => 1, + }; + tokens.push(Token { + kind, + position: index, + }); + Ok(index + width) +} + +/// Python's tokenizer reads ASCII letters, `_` and any non-ASCII character into a name. +pub(crate) fn is_name_start(character: char) -> bool { + character.is_ascii_alphabetic() || character == '_' || !character.is_ascii() +} + +fn is_name_continue(character: char) -> bool { + is_name_start(character) || character.is_ascii_digit() +} diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs new file mode 100644 index 0000000000..2a5e8c421d --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs @@ -0,0 +1,284 @@ +//! Python string and number literals: escapes, prefixes, underscores and radixes. + +use crate::macro_arguments::_helpers::failures::{Failure, surrogate, syntax}; +use crate::macro_arguments::_helpers::lexer::{NumberToken, StringPiece, is_name_start}; +use crate::macro_arguments::types::ArgumentHost; + +/// The string starting at the quote `start` under `prefix`, and the index after it. +pub(crate) fn string_body( + host: &dyn ArgumentHost, + text: &[char], + start: usize, + prefix: &str, +) -> Result<(StringPiece, usize), Failure> { + let quote: char = text[start]; + let triple: bool = text.get(start + 1) == Some("e) && text.get(start + 2) == Some("e); + let width: usize = if triple { 3 } else { 1 }; + let raw: bool = prefix.contains('r'); + let mut value: String = String::new(); + let mut index: usize = start + width; + loop { + let Some(&character) = text.get(index) else { + return Err(syntax("a string is not closed", start)); + }; + if character == quote && (!triple || closes_triple(text, index, quote)) { + index += width; + break; + } + if !triple && matches!(character, '\n' | '\r') { + return Err(syntax("a string is not closed before the line ends", start)); + } + if character != '\\' { + value.push(character); + index += 1; + continue; + } + let Some(&escaped) = text.get(index + 1) else { + return Err(syntax("a string is not closed", start)); + }; + if raw { + value.push('\\'); + value.push(escaped); + index += 2; + continue; + } + index = escape(host, text, index, &mut value)?; + } + Ok(( + StringPiece { + value, + bytes: prefix.contains('b'), + formatted: prefix.contains('f'), + }, + index, + )) +} + +fn closes_triple(text: &[char], index: usize, quote: char) -> bool { + text.get(index + 1) == Some("e) && text.get(index + 2) == Some("e) +} + +/// Decode the escape at the backslash `index` into `value`; return the index after it. +fn escape( + host: &dyn ArgumentHost, + text: &[char], + index: usize, + value: &mut String, +) -> Result { + let escaped: char = text[index + 1]; + let simple: Option = match escaped { + '\\' => Some('\\'), + '\'' => Some('\''), + '"' => Some('"'), + 'a' => Some('\x07'), + 'b' => Some('\x08'), + 'f' => Some('\x0c'), + 'n' => Some('\n'), + 'r' => Some('\r'), + 't' => Some('\t'), + 'v' => Some('\x0b'), + _ => None, + }; + if let Some(character) = simple { + value.push(character); + return Ok(index + 2); + } + match escaped { + '\n' => Ok(index + 2), + '\r' => Ok(if text.get(index + 2) == Some(&'\n') { + index + 3 + } else { + index + 2 + }), + '0'..='7' => { + let digits: usize = text[index + 1..] + .iter() + .take(3) + .take_while(|character| matches!(character, '0'..='7')) + .count(); + let code: u32 = radix_value(&text[index + 1..index + 1 + digits], 8); + value.push(char::from_u32(code).unwrap_or_default()); + Ok(index + 1 + digits) + } + 'x' => hex_escape(text, index, 2, value), + 'u' => hex_escape(text, index, 4, value), + 'U' => hex_escape(text, index, 8, value), + 'N' => named_escape(host, text, index, value), + _ => { + value.push('\\'); + Ok(index + 1) + } + } +} + +fn hex_escape( + text: &[char], + index: usize, + width: usize, + value: &mut String, +) -> Result { + let digits: &[char] = &text[index + 2..text.len().min(index + 2 + width)]; + if digits.len() < width || !digits.iter().all(char::is_ascii_hexdigit) { + let escape: String = text[index..index + 2].iter().collect(); + return Err(syntax( + &format!("a {escape} escape needs exactly {width} hex digits"), + index, + )); + } + let code: u32 = radix_value(digits, 16); + let end: usize = index + 2 + width; + if (0xD800..=0xDFFF).contains(&code) { + let escape: String = text[index..end].iter().collect(); + return Err(surrogate(&escape, index)); + } + let character: char = char::from_u32(code).ok_or_else(|| { + let escape: String = text[index..end].iter().collect(); + syntax(&format!("{escape} is not a Unicode code point"), index) + })?; + value.push(character); + Ok(end) +} + +fn named_escape( + host: &dyn ArgumentHost, + text: &[char], + index: usize, + value: &mut String, +) -> Result { + let close: Option = (text.get(index + 2) == Some(&'{')) + .then(|| { + text[index + 3..] + .iter() + .position(|character| *character == '}') + }) + .flatten(); + let Some(close) = close else { + return Err(syntax("a \\N escape needs a {name}", index)); + }; + let name: String = text[index + 3..index + 3 + close].iter().collect(); + let character: char = host + .character_named(&name) + .ok_or_else(|| syntax(&format!("\\N{{{name}}} names no Unicode character"), index))?; + value.push(character); + Ok(index + 4 + close) +} + +fn radix_value(digits: &[char], radix: u32) -> u32 { + digits.iter().fold(0, |total, digit| { + total * radix + digit.to_digit(radix).unwrap_or_default() + }) +} + +/// The number starting at `start` and the index after it. +pub(crate) fn number(text: &[char], start: usize) -> Result<(NumberToken, usize), Failure> { + let prefixed: Option = (text[start] == '0') + .then(|| match text.get(start + 1) { + Some('x' | 'X') => Some(16), + Some('o' | 'O') => Some(8), + Some('b' | 'B') => Some(2), + _ => None, + }) + .flatten(); + if let Some(radix) = prefixed { + let mut index: usize = start + 2; + if text.get(index) == Some(&'_') { + index += 1; + } + let (digits, end) = digit_part(text, index, radix, start)?; + if digits.is_empty() { + return Err(syntax( + "a number has no digits after its base prefix", + start, + )); + } + let end: usize = reject_trailing_name(text, end, start)?; + return Ok((NumberToken::Int { radix, digits }, end)); + } + let (whole, mut index) = digit_part(text, start, 10, start)?; + let mut float: bool = false; + let mut literal: String = whole.clone(); + if text.get(index) == Some(&'.') { + float = true; + literal.push('.'); + let (fraction, end) = digit_part(text, index + 1, 10, start)?; + literal.push_str(&fraction); + index = end; + } + if matches!(text.get(index), Some('e' | 'E')) { + let mut exponent_start: usize = index + 1; + let mut exponent: String = String::from("e"); + if let Some(sign @ ('+' | '-')) = text.get(exponent_start) { + exponent.push(*sign); + exponent_start += 1; + } + let (digits, end) = digit_part(text, exponent_start, 10, start)?; + if digits.is_empty() { + return Err(syntax("a number's exponent has no digits", start)); + } + float = true; + literal.push_str(&exponent); + literal.push_str(&digits); + index = end; + } + if matches!(text.get(index), Some('j' | 'J')) { + let end: usize = reject_trailing_name(text, index + 1, start)?; + return Ok((NumberToken::Complex, end)); + } + let end: usize = reject_trailing_name(text, index, start)?; + if float { + return Ok((NumberToken::Float(literal), end)); + } + if whole.len() > 1 && whole.starts_with('0') && whole.chars().any(|digit| digit != '0') { + return Err(syntax( + "leading zeros in a decimal integer are not allowed; use 0o for octal", + start, + )); + } + Ok(( + NumberToken::Int { + radix: 10, + digits: whole, + }, + end, + )) +} + +/// Digits in `radix` with single underscores between them, without the underscores. +fn digit_part( + text: &[char], + start: usize, + radix: u32, + number_start: usize, +) -> Result<(String, usize), Failure> { + let mut digits: String = String::new(); + let mut index: usize = start; + while let Some(&character) = text.get(index) { + if character.is_digit(radix) { + digits.push(character); + index += 1; + } else if character == '_' + && !digits.is_empty() + && text.get(index + 1).is_some_and(|next| next.is_digit(radix)) + { + index += 1; + } else if character == '_' || (character.is_ascii_digit() && radix < 10) { + return Err(syntax( + "a number has a misplaced '_' or digit", + number_start, + )); + } else { + break; + } + } + Ok((digits, index)) +} + +fn reject_trailing_name(text: &[char], end: usize, start: usize) -> Result { + if text + .get(end) + .is_some_and(|next| is_name_start(*next) || next.is_ascii_digit()) + { + return Err(syntax("a number runs into a name", start)); + } + Ok(end) +} diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/mod.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/mod.rs new file mode 100644 index 0000000000..2e06ba4d6a --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod evaluation; +pub(crate) mod failures; +pub(crate) mod lexer; +pub(crate) mod literals; +pub(crate) mod syntax_tree; diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs new file mode 100644 index 0000000000..70d9f8bb02 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs @@ -0,0 +1,385 @@ +//! Parse argument tokens into Python's `ast` node shapes for the literal subset. + +use crate::macro_arguments::_helpers::failures::{Failure, keyword_order, syntax, unsupported}; +use crate::macro_arguments::_helpers::lexer::{NumberToken, Token, TokenKind}; + +const PYTHON_KEYWORDS: [&str; 35] = [ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", + "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", + "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", + "with", "yield", +]; + +/// One expression node and the code point position it starts at. +#[derive(Clone, Debug)] +pub(crate) struct Node { + pub(crate) expr: Expr, + pub(crate) position: usize, +} + +#[derive(Clone, Debug)] +pub(crate) enum Expr { + Str { + value: String, + bytes: bool, + formatted: bool, + }, + Number(NumberToken), + Ellipsis, + Name(String), + Nested(usize), + Call { + function: Box, + args: Vec, + keywords: Vec, + }, + List(Vec), + Tuple(Vec), + Dict(Vec<(Option, Node)>), + Set(Vec), + Starred(Box), + Unary { + negative: bool, + operand: Box, + }, +} + +/// One keyword argument; `name` is `None` for `**` expansion. +#[derive(Clone, Debug)] +pub(crate) struct Keyword { + pub(crate) name: Option, + pub(crate) value: Node, +} + +/// The arguments of one call, as `ast.Call.args` and `ast.Call.keywords`. +pub(crate) struct CallArguments { + pub(crate) args: Vec, + pub(crate) keywords: Vec, +} + +struct Parser<'tokens> { + tokens: &'tokens [Token], + index: usize, +} + +/// Parse every token as the arguments of one call. +pub(crate) fn parse_call_arguments(tokens: &[Token]) -> Result { + let mut parser = Parser { tokens, index: 0 }; + let arguments: CallArguments = parser.arguments()?; + let end: &Token = parser.peek(); + if matches!(end.kind, TokenKind::End) { + Ok(arguments) + } else { + Err(parser.unexpected()) + } +} + +impl Parser<'_> { + fn peek(&self) -> &Token { + &self.tokens[self.index] + } + + fn peek_at(&self, offset: usize) -> Option<&Token> { + self.tokens.get(self.index + offset) + } + + fn at_operator(&self, operator: &str) -> bool { + matches!(self.peek().kind, TokenKind::Operator(found) if found == operator) + } + + fn eat(&mut self, operator: &str) -> bool { + let found: bool = self.at_operator(operator); + if found { + self.index += 1; + } + found + } + + /// The error for the next token where no supported argument can continue. + fn unexpected(&self) -> Failure { + let token: &Token = self.peek(); + match &token.kind { + TokenKind::End => syntax( + "the arguments end where a value is expected", + token.position, + ), + TokenKind::Operator(")" | "]" | "}") => syntax( + "a closing bracket does not match an opening one", + token.position, + ), + _ => unsupported(token.position), + } + } + + /// Arguments up to a `)` or the end, as Python's call grammar orders them. + fn arguments(&mut self) -> Result { + let mut args: Vec = Vec::new(); + let mut keywords: Vec = Vec::new(); + let mut expanded: bool = false; + while !matches!(self.peek().kind, TokenKind::End) && !self.at_operator(")") { + let position: usize = self.peek().position; + if self.eat("**") { + expanded = true; + keywords.push(Keyword { + name: None, + value: self.expression()?, + }); + } else if let Some(name) = self.keyword_name()? { + if keywords + .iter() + .any(|keyword| keyword.name.as_deref() == Some(&name)) + { + return Err(keyword_order( + &format!("could not be parsed: keyword argument '{name}' is repeated"), + position, + )); + } + keywords.push(Keyword { + name: Some(name), + value: self.expression()?, + }); + } else { + if !keywords.is_empty() { + let detail: &str = if expanded { + "could not be parsed: a positional argument follows ** expansion" + } else { + "could not be parsed: a positional argument follows a keyword argument" + }; + return Err(keyword_order(detail, position)); + } + args.push(self.starred_or_expression()?); + } + if !self.eat(",") { + break; + } + } + Ok(CallArguments { args, keywords }) + } + + /// `name=` starting a keyword argument, consumed, or `None` without consuming. + fn keyword_name(&mut self) -> Result, Failure> { + let TokenKind::Name(name) = &self.peek().kind else { + return Ok(None); + }; + let assigns: bool = matches!( + self.peek_at(1).map(|token| &token.kind), + Some(TokenKind::Operator("=")) + ) && !matches!( + self.peek_at(2).map(|token| &token.kind), + Some(TokenKind::Operator("=")) + ); + if !assigns { + return Ok(None); + } + let name: String = name.clone(); + let position: usize = self.peek().position; + if PYTHON_KEYWORDS.contains(&name.as_str()) { + return Err(syntax( + &format!("'{name}' is a Python keyword, not an argument name"), + position, + )); + } + self.index += 2; + Ok(Some(name)) + } + + fn starred_or_expression(&mut self) -> Result { + let position: usize = self.peek().position; + if self.eat("*") { + return Ok(Node { + expr: Expr::Starred(Box::new(self.expression()?)), + position, + }); + } + self.expression() + } + + fn expression(&mut self) -> Result { + let position: usize = self.peek().position; + if self.at_operator("-") || self.at_operator("+") { + let negative: bool = self.at_operator("-"); + self.index += 1; + return Ok(Node { + expr: Expr::Unary { + negative, + operand: Box::new(self.expression()?), + }, + position, + }); + } + let mut node: Node = self.atom()?; + while self.at_operator("(") { + self.index += 1; + let arguments: CallArguments = self.arguments()?; + self.close(")", position)?; + node = Node { + expr: Expr::Call { + function: Box::new(node), + args: arguments.args, + keywords: arguments.keywords, + }, + position, + }; + } + Ok(node) + } + + fn close(&mut self, operator: &str, open: usize) -> Result<(), Failure> { + if self.eat(operator) { + return Ok(()); + } + if matches!(self.peek().kind, TokenKind::End) { + return Err(syntax( + &format!("a bracket is not closed with '{operator}'"), + open, + )); + } + Err(self.unexpected()) + } + + fn atom(&mut self) -> Result { + let token: Token = self.peek().clone(); + let expr: Expr = match token.kind { + TokenKind::Str(_) => return Ok(self.strings()), + TokenKind::Number(number) => Expr::Number(number), + TokenKind::Name(name) => Expr::Name(name), + TokenKind::Nested(call) => Expr::Nested(call), + TokenKind::Operator("...") => Expr::Ellipsis, + TokenKind::Operator("(") => return self.parenthesized(token.position), + TokenKind::Operator("[") => { + self.index += 1; + let items: Vec = self.items("]", token.position)?; + Expr::List(items) + } + TokenKind::Operator("{") => return self.braced(token.position), + TokenKind::Operator(_) => { + return Err(syntax("a value is missing here", token.position)); + } + _ => return Err(self.unexpected()), + }; + if !matches!(expr, Expr::List(_)) { + self.index += 1; + } + Ok(Node { + expr, + position: token.position, + }) + } + + /// Adjacent string pieces concatenate into one constant, as Python's parser joins them. + fn strings(&mut self) -> Node { + let position: usize = self.peek().position; + let mut value: String = String::new(); + let mut bytes: bool = false; + let mut formatted: bool = false; + while let TokenKind::Str(piece) = &self.peek().kind { + value.push_str(&piece.value); + bytes |= piece.bytes; + formatted |= piece.formatted; + self.index += 1; + } + Node { + expr: Expr::Str { + value, + bytes, + formatted, + }, + position, + } + } + + fn parenthesized(&mut self, position: usize) -> Result { + self.index += 1; + if self.eat(")") { + return Ok(Node { + expr: Expr::Tuple(Vec::new()), + position, + }); + } + let first: Node = self.starred_or_expression()?; + if !self.eat(",") { + self.close(")", position)?; + return Ok(first); + } + let mut items: Vec = vec![first]; + items.extend(self.items(")", position)?); + Ok(Node { + expr: Expr::Tuple(items), + position, + }) + } + + /// Comma-separated items up to `close`, which is consumed; a trailing comma is allowed. + fn items(&mut self, close: &str, open: usize) -> Result, Failure> { + let mut items: Vec = Vec::new(); + while !self.at_operator(close) && !matches!(self.peek().kind, TokenKind::End) { + items.push(self.starred_or_expression()?); + if !self.eat(",") { + break; + } + } + self.close(close, open)?; + Ok(items) + } + + fn braced(&mut self, position: usize) -> Result { + self.index += 1; + if self.eat("}") { + return Ok(Node { + expr: Expr::Dict(Vec::new()), + position, + }); + } + if !self.at_operator("**") { + let first: Node = self.starred_or_expression()?; + if !self.eat(":") { + let mut items: Vec = vec![first]; + if self.eat(",") { + items.extend(self.items("}", position)?); + } else { + self.close("}", position)?; + } + return Ok(Node { + expr: Expr::Set(items), + position, + }); + } + let value: Node = self.expression()?; + let mut pairs: Vec<(Option, Node)> = vec![(Some(first), value)]; + if self.eat(",") { + pairs.extend(self.pairs(position)?); + } else { + self.close("}", position)?; + } + return Ok(Node { + expr: Expr::Dict(pairs), + position, + }); + } + let pairs: Vec<(Option, Node)> = self.pairs(position)?; + Ok(Node { + expr: Expr::Dict(pairs), + position, + }) + } + + fn pairs(&mut self, open: usize) -> Result, Node)>, Failure> { + let mut pairs: Vec<(Option, Node)> = Vec::new(); + while !self.at_operator("}") && !matches!(self.peek().kind, TokenKind::End) { + if self.eat("**") { + pairs.push((None, self.expression()?)); + } else { + let key: Node = self.expression()?; + if !self.eat(":") { + return Err(self.unexpected()); + } + pairs.push((Some(key), self.expression()?)); + } + if !self.eat(",") { + break; + } + } + self.close("}", open)?; + Ok(pairs) + } +} diff --git a/crates/sqlbuild-render/src/macro_arguments/main/mod.rs b/crates/sqlbuild-render/src/macro_arguments/main/mod.rs new file mode 100644 index 0000000000..72a3a5e619 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/main/mod.rs @@ -0,0 +1 @@ +pub mod parse_macro_arguments; diff --git a/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs new file mode 100644 index 0000000000..8ad1f29c7a --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs @@ -0,0 +1,41 @@ +//! Parse one macro call's argument text as Python's literal grammar, with native errors. + +use crate::macro_arguments::_helpers::evaluation::evaluate_arguments; +use crate::macro_arguments::_helpers::failures::Failure; +use crate::macro_arguments::_helpers::lexer::tokenize; +use crate::macro_arguments::_helpers::syntax_tree::parse_call_arguments; +use crate::macro_arguments::models::{ArgumentError, MacroArguments}; +use crate::macro_arguments::types::ArgumentHost; + +/// The value plan of `text`, whose nested macro calls fill the `[start, end)` code point spans. +pub fn parse_macro_arguments( + host: &dyn ArgumentHost, + text: &str, + nested: &[(usize, usize)], +) -> Result { + let characters: Vec = text.chars().collect(); + tokenize(host, &characters, nested) + .and_then(|tokens| parse_call_arguments(&tokens)) + .and_then(|arguments| evaluate_arguments(&arguments)) + .map_err(|failure| located(&characters, failure)) +} + +fn located(characters: &[char], failure: Failure) -> ArgumentError { + let mut line: usize = 1; + let mut line_start: usize = 0; + let mut index: usize = 0; + while index < failure.position.min(characters.len()) { + let character: char = characters[index]; + index += 1; + if character == '\n' || (character == '\r' && characters.get(index) != Some(&'\n')) { + line += 1; + line_start = index; + } + } + ArgumentError { + detail: failure.detail, + help: failure.help.to_owned(), + line, + column: failure.position - line_start + 1, + } +} diff --git a/crates/sqlbuild-render/src/macro_arguments/mod.rs b/crates/sqlbuild-render/src/macro_arguments/mod.rs new file mode 100644 index 0000000000..91fb358a0f --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-render/src/macro_arguments/models.rs b/crates/sqlbuild-render/src/macro_arguments/models.rs new file mode 100644 index 0000000000..3a86ab5715 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/models.rs @@ -0,0 +1,51 @@ +//! The value plan of one macro call's arguments, and why parsing them stopped. + +/// One argument value; Python builds the object, calling nested macros by their index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ArgumentValue { + Str(String), + /// An integer as its digits in `radix`, without underscores, for Python's unbounded `int`. + Int { + radix: u32, + digits: String, + }, + /// A float as Python's `float()` reads it, without underscores. + Float(String), + Bool(bool), + None, + /// The value of the nested macro call at this index of the call's nested spans. + NestedCall(usize), + /// A typed reference: the function (`__ref`, `__source` or `__seed`) and the resource name. + TypedReference { + function: String, + name: String, + }, + List(Vec), + Tuple(Vec), + /// Key and value pairs in source order; a later duplicate key replaces an earlier value. + Dict(Vec<(ArgumentValue, ArgumentValue)>), + /// Unary minus over a number, or over a nested call whose number Python checks. + Negative(Box), + /// Unary plus, which leaves its number unchanged. + Positive(Box), +} + +/// A call's parsed arguments and the typed references written in them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroArguments { + pub positional: Vec, + pub keywords: Vec<(String, ArgumentValue)>, + /// `(function, name)` of every typed reference, in Python's `ast.walk` order. + pub typed_references: Vec<(String, String)>, +} + +/// Why the arguments cannot be used, where, and how to fix them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArgumentError { + /// What is wrong, completing "Macro arguments in '' ...". + pub detail: String, + pub help: String, + /// 1-based line and column, in characters, within the argument text. + pub line: usize, + pub column: usize, +} diff --git a/crates/sqlbuild-render/src/macro_arguments/tests.rs b/crates/sqlbuild-render/src/macro_arguments/tests.rs new file mode 100644 index 0000000000..e6145b4ebf --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_parse_macro_arguments.rs"] +mod parse_macro_arguments; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs b/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs new file mode 100644 index 0000000000..93887a7f10 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs @@ -0,0 +1,68 @@ +use crate::macro_arguments::main::parse_macro_arguments::parse_macro_arguments; +use crate::macro_arguments::models::ArgumentValue; +use crate::macro_arguments::types::ArgumentHost; + +/// A host knowing one character name and NFKC-folding the `fi` ligature, rejecting `€`. +struct TestHost; + +impl ArgumentHost for TestHost { + fn character_named(&self, name: &str) -> Option { + (name == "LATIN SMALL LETTER E WITH ACUTE").then_some('é') + } + + fn identifier(&self, text: &str) -> Option { + (!text.contains('€')).then(|| text.replace('fi', "fi")) + } +} + +/// Spell the plan as `positional | keywords | references`, or the error and its location. +pub(crate) fn spelled_arguments(text: &str, nested: &[(usize, usize)]) -> String { + match parse_macro_arguments(&TestHost, text, nested) { + Ok(arguments) => format!( + "{} | {} | {}", + spelled_list(&arguments.positional), + arguments + .keywords + .iter() + .map(|(name, value)| format!("{name}={}", spelled(value))) + .collect::>() + .join(", "), + arguments + .typed_references + .iter() + .map(|(function, name)| format!("{function}:{name}")) + .collect::>() + .join(", "), + ), + Err(error) => format!("error: {} @{}:{}", error.detail, error.line, error.column), + } +} + +fn spelled_list(values: &[ArgumentValue]) -> String { + values.iter().map(spelled).collect::>().join(", ") +} + +fn spelled(value: &ArgumentValue) -> String { + match value { + ArgumentValue::Str(text) => format!("{text:?}"), + ArgumentValue::Int { radix: 10, digits } => digits.clone(), + ArgumentValue::Int { radix, digits } => format!("{digits}/{radix}"), + ArgumentValue::Float(text) => format!("f{text}"), + ArgumentValue::Bool(flag) => if *flag { "True" } else { "False" }.to_owned(), + ArgumentValue::None => "None".to_owned(), + ArgumentValue::NestedCall(call) => format!("@{call}"), + ArgumentValue::TypedReference { function, name } => format!("{function}({name})"), + ArgumentValue::List(items) => format!("[{}]", spelled_list(items)), + ArgumentValue::Tuple(items) => format!("({})", spelled_list(items)), + ArgumentValue::Dict(pairs) => format!( + "{{{}}}", + pairs + .iter() + .map(|(key, value)| format!("{}: {}", spelled(key), spelled(value))) + .collect::>() + .join(", ") + ), + ArgumentValue::Negative(inner) => format!("-{}", spelled(inner)), + ArgumentValue::Positive(inner) => format!("+{}", spelled(inner)), + } +} diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs new file mode 100644 index 0000000000..f4e36f5d9a --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs @@ -0,0 +1,202 @@ +use crate::macro_arguments::tests::helpers::spelled_arguments; +use crate::macro_arguments::tests::test_types::ParseMacroArgumentsTestCase; + +#[test] +fn given_argument_text_when_parsing_then_values_match_python_literals() { + let test_cases = [ + ParseMacroArgumentsTestCase { + description: "scalars, containers and keywords", + text: "'a', 1, 2.5, True, None, [1, (2,)], {'k': -3}, (), x=+1.0, y=[]", + nested: &[], + expected: "\"a\", 1, f2.5, True, None, [1, (2)], {\"k\": -3}, () | x=+f1.0, y=[] | ", + }, + ParseMacroArgumentsTestCase { + description: "adjacent, raw, escaped and triple-quoted strings", + text: "'it''s', \"a\" 'b', r'\\d', '\\x41\\u00e9\\N{LATIN SMALL LETTER E WITH ACUTE}\\101\\q', '''x\\ny'''", + nested: &[], + expected: "\"its\", \"ab\", \"\\\\d\", \"AééA\\\\q\", \"x\\ny\" | | ", + }, + ParseMacroArgumentsTestCase { + description: "integer radixes, underscores and float forms", + text: "0x_ff, 0o17, 0b1_0, 1_000, 00, .5, 5., 1e-3, 1_0.0_1E+2", + nested: &[], + expected: "ff/16, 17/8, 10/2, 1000, 00, f.5, f5., f1e-3, f10.01e+2 | | ", + }, + ParseMacroArgumentsTestCase { + description: "typed references in ast.walk order", + text: "[__ref('a')], __source(\"b\",), k=__seed('c')", + nested: &[], + expected: "[__ref(a)], __source(b) | k=__seed(c) | __source:b, __ref:a, __seed:c", + }, + ParseMacroArgumentsTestCase { + description: "nested calls fill their spans, even under unary minus", + text: "@m(1), -@n()", + nested: &[(0, 5), (8, 12)], + expected: "@0, -@1 | | ", + }, + ParseMacroArgumentsTestCase { + description: "a keyword name is NFKC-normalized", + text: "field=1", + nested: &[], + expected: " | field=1 | ", + }, + ParseMacroArgumentsTestCase { + description: "comments and line continuations", + text: "1, # note\n 2, \\\n 3", + nested: &[], + expected: "1, 2, 3 | | ", + }, + ParseMacroArgumentsTestCase { + description: "unary signs over booleans and other signs", + text: "-True, +-1", + nested: &[], + expected: "-True, +-1 | | ", + }, + ParseMacroArgumentsTestCase { + description: "a bytes literal is rejected", + text: "b'x'", + nested: &[], + expected: "error: use a bytes literal @1:1", + }, + ParseMacroArgumentsTestCase { + description: "a complex literal is rejected", + text: "1, 2j", + nested: &[], + expected: "error: use a complex number literal @1:4", + }, + ParseMacroArgumentsTestCase { + description: "an Ellipsis is rejected", + text: "...", + nested: &[], + expected: "error: use '...' (Ellipsis) @1:1", + }, + ParseMacroArgumentsTestCase { + description: "an f-string is rejected", + text: "f'{x}'", + nested: &[], + expected: "error: use an f-string @1:1", + }, + ParseMacroArgumentsTestCase { + description: "bytes joined to a string are rejected", + text: "'a' b'c'", + nested: &[], + expected: "error: use a bytes literal @1:1", + }, + ParseMacroArgumentsTestCase { + description: "an empty typed reference name", + text: "__ref('')", + nested: &[], + expected: "error: must give __ref() exactly one quoted resource name @1:1", + }, + ParseMacroArgumentsTestCase { + description: "a bare name", + text: "x", + nested: &[], + expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:1", + }, + ParseMacroArgumentsTestCase { + description: "a binary operation", + text: "1 + 2", + nested: &[], + expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:3", + }, + ParseMacroArgumentsTestCase { + description: "a set", + text: "{1, 2}", + nested: &[], + expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:1", + }, + ParseMacroArgumentsTestCase { + description: "keyword expansion", + text: "**k", + nested: &[], + expected: "error: must not use **kwargs expansion syntax @1:3", + }, + ParseMacroArgumentsTestCase { + description: "dict unpacking", + text: "{**d}", + nested: &[], + expected: "error: must not use dict unpacking @1:4", + }, + ParseMacroArgumentsTestCase { + description: "unary minus on text", + text: "-'a'", + nested: &[], + expected: "error: use unary + or - on a value that is not a number @1:1", + }, + ParseMacroArgumentsTestCase { + description: "a positional argument after a keyword", + text: "x=1, 2", + nested: &[], + expected: "error: could not be parsed: a positional argument follows a keyword argument @1:6", + }, + ParseMacroArgumentsTestCase { + description: "a repeated keyword", + text: "x=1, x=2", + nested: &[], + expected: "error: could not be parsed: keyword argument 'x' is repeated @1:6", + }, + ParseMacroArgumentsTestCase { + description: "a doubled comma", + text: "'amount',,", + nested: &[], + expected: "error: could not be parsed: a value is missing here @1:10", + }, + ParseMacroArgumentsTestCase { + description: "an unclosed string", + text: "'abc", + nested: &[], + expected: "error: could not be parsed: a string is not closed @1:1", + }, + ParseMacroArgumentsTestCase { + description: "an unclosed bracket on a later line", + text: "[1,\n 2", + nested: &[], + expected: "error: could not be parsed: a bracket is not closed with ']' @1:1", + }, + ParseMacroArgumentsTestCase { + description: "an error on a later line", + text: "1,\n x", + nested: &[], + expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @2:3", + }, + ParseMacroArgumentsTestCase { + description: "a lone surrogate escape", + text: "'\\ud800'", + nested: &[], + expected: "error: contain the lone surrogate escape '\\ud800' @1:2", + }, + ParseMacroArgumentsTestCase { + description: "leading zeros in a decimal integer", + text: "007", + nested: &[], + expected: "error: could not be parsed: leading zeros in a decimal integer are not allowed; use 0o for octal @1:1", + }, + ParseMacroArgumentsTestCase { + description: "a character Python rejects in a name", + text: "1, é€", + nested: &[], + expected: "error: could not be parsed: the character '€' (U+20AC) is not valid in a name @1:4", + }, + ParseMacroArgumentsTestCase { + description: "a keyword used as an argument name", + text: "True=1", + nested: &[], + expected: "error: could not be parsed: 'True' is a Python keyword, not an argument name @1:1", + }, + ParseMacroArgumentsTestCase { + description: "an unknown character name", + text: "'\\N{NOPE}'", + nested: &[], + expected: "error: could not be parsed: \\N{NOPE} names no Unicode character @1:2", + }, + ]; + for test_case in test_cases { + assert_eq!( + spelled_arguments(test_case.text, test_case.nested), + test_case.expected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs new file mode 100644 index 0000000000..c5235931e5 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct ParseMacroArgumentsTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) nested: &'static [(usize, usize)], + /// The spelled plan, or `error: @line:column`. + pub(crate) expected: &'static str, +} diff --git a/crates/sqlbuild-render/src/macro_arguments/types.rs b/crates/sqlbuild-render/src/macro_arguments/types.rs new file mode 100644 index 0000000000..f5dc60fb24 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/types.rs @@ -0,0 +1,9 @@ +//! What argument parsing asks of the host Python for Unicode text it does not decide itself. + +/// Unicode lookups the host Python answers, so names match CPython's own tables. +pub trait ArgumentHost { + /// The character `\N{name}` names, or `None` when Python knows no such name. + fn character_named(&self, name: &str) -> Option; + /// The NFKC form of a non-ASCII identifier, or `None` when Python rejects it as one. + fn identifier(&self, text: &str) -> Option; +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs index 6880c5aa0c..5e5fd1ec90 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs @@ -12,5 +12,7 @@ pub fn scan_declaration_references( python: PythonText, sqls: &[String], ) -> Vec { - sqls.par_iter().map(|sql| scan_references(python, sql)).collect() + sqls.par_iter() + .map(|sql| scan_references(python, sql)) + .collect() } diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index e5f5ac19e0..b68503f980 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -596,6 +596,13 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) +_ENGINE_ERROR_MACROS: str = "macros/cents.py" +_ENGINE_ERROR_CENTS: str = ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + ' return f"({expression} * 100)"\n' +) + + def engine_error_cases() -> tuple[FailureCase, ...]: """User-facing errors every engine must report with the exact text; see the README.""" @@ -642,6 +649,78 @@ def engine_error_cases() -> tuple[FailureCase, ...]: 'Quote it to keep it as text ("\u0663"), or write the number with ASCII digits 0-9' ), ), + failure_case( + name="engine-error-macro-argument-syntax", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, @cents('amount',, 2) AS cents\n" + 'FROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments in '/{FAILURE_MART_PATH}' " + "could not be parsed: a value is missing here at line 1, column 10 of the " + "'@cents' arguments" + ), + expected_help=( + "Macro arguments are Python literals (strings, numbers, True, False, None, lists, " + "tuples and dicts), nested macro calls, and __ref(), __source() or __seed() " + "references; compute anything else inside the macro" + ), + ), + failure_case( + name="engine-error-macro-argument-bytes", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, @cents(b'amount') AS cents\\nFROM __ref(\"stg_orders\")\n" + ), + }, + expected_message=( + f"Macro arguments in '/{FAILURE_MART_PATH}' " + "use a bytes literal at line 1, column 1 of the '@cents' arguments" + ), + expected_help=( + "Pass text as a string without the b prefix, for example 'orders' instead of " + "b'orders'" + ), + ), + failure_case( + name="engine-error-macro-argument-complex", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + 'SELECT customer_id, @cents(2j) AS cents\\nFROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments in '/{FAILURE_MART_PATH}' " + "use a complex number literal at line 1, column 1 of the '@cents' arguments" + ), + expected_help=( + "Pass int or float numbers; give a complex value's real and imaginary parts as two " + "arguments" + ), + ), + failure_case( + name="engine-error-macro-argument-ellipsis", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + 'SELECT customer_id, @cents(...) AS cents\\nFROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments in '/{FAILURE_MART_PATH}' " + "use '...' (Ellipsis) at line 1, column 1 of the '@cents' arguments" + ), + expected_help=("Pass None, or a string the macro understands, instead of '...'"), + ), failure_case( name="engine-error-week-date-cursor-start", expected_code="P001", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8f235cefdc..a89084ae72 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -299,6 +299,7 @@ def render_attached_generic_audit( arguments: tuple[dict[str, object], dict[str, object]], policies: dict[str, object], ) -> tuple[str | None, str, str | None, str, str, str | None] | None: ... +def parse_macro_arguments(text: str, nested: list[tuple[int, int]]) -> tuple[object, ...]: ... def scan_sql_declaration_references( sqls: list[str], python_version: tuple[int, int], unicode_version: str ) -> list[tuple[list[tuple[int, str, str | None, int, int]], int | None]]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index e16700e9b0..ad05d2f336 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -2,7 +2,6 @@ from __future__ import annotations -import ast import re from pathlib import Path @@ -265,25 +264,7 @@ def _generated_references(sql: str) -> tuple[SqlResourceRef, ...]: return tuple(generated) -def evaluate_typed_reference(*, node: ast.Call, file_path: Path) -> SqlResourceRef: - function_name: str | None = node.func.id if isinstance(node.func, ast.Name) else None - kind: SqlResourceRefKind | None = ( - _TYPED_REFERENCE_KINDS.get(function_name) if function_name is not None else None - ) - if kind is None: - raise CompileInputError( - f"Macro arguments in '{file_path}' must use only Python literals, nested macro " - "calls, and __ref(), __source(), or __seed() references" - ) - if ( - node.keywords - or len(node.args) != 1 - or not isinstance(node.args[0], ast.Constant) - or not isinstance(node.args[0].value, str) - or not node.args[0].value - ): - raise CompileInputError( - f"Macro argument {function_name}() in '{file_path}' must contain exactly one " - "quoted resource name" - ) - return SqlResourceRef(kind=kind, name=node.args[0].value) +def typed_reference_value(*, function: str, name: str) -> SqlResourceRef: + """Return the reference a `__ref`, `__source` or `__seed` macro argument names.""" + + return SqlResourceRef(kind=_TYPED_REFERENCE_KINDS[function], name=name) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py new file mode 100644 index 0000000000..a5c85a649c --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py @@ -0,0 +1,80 @@ +"""Build macro call arguments from the value plan native parsing returns for their text.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( + typed_reference_value, +) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ParsedMacroArguments + +type _ValueRow = tuple[object, ...] + + +def parse_macro_call_arguments( + *, + args_source: str, + nested: tuple[tuple[int, int], ...], + nested_values: Sequence[object], + macro_name: str, + file_path: Path, +) -> ParsedMacroArguments: + """Parse `args_source` natively; nested calls at `nested` take their `nested_values`.""" + + parsed: tuple[object, ...] = _native.parse_macro_arguments(args_source, list(nested)) + label: str = f"the '@{macro_name}' arguments" + if parsed[0] == "error": + _, detail, help_text, line, column = parsed + raise CompileInputError( + f"Macro arguments in '{file_path}' {detail} at line {line}, column {column} of {label}", + help=cast(str, help_text), + ) + _, positional, keywords, references = parsed + context: tuple[Sequence[object], str] = (nested_values, f"'{file_path}' in {label}") + return ParsedMacroArguments( + args=tuple(_built(row, context) for row in cast(list[_ValueRow], positional)), + kwargs={ + name: _built(row, context) for name, row in cast(list[tuple[str, _ValueRow]], keywords) + }, + typed_references=tuple( + typed_reference_value(function=function, name=name) + for function, name in cast(list[tuple[str, str]], references) + ), + ) + + +def _built(row: _ValueRow, context: tuple[Sequence[object], str]) -> object: + tag: object = row[0] + if tag in ("s", "b"): + return row[1] + if tag == "i": + return int(cast(str, row[2]), cast(int, row[1])) + if tag == "f": + return float(cast(str, row[1])) + if tag == "c": + return context[0][cast(int, row[1])] + if tag == "r": + return typed_reference_value(function=cast(str, row[1]), name=cast(str, row[2])) + if tag == "l": + return [_built(item, context) for item in cast(list[_ValueRow], row[1])] + if tag == "t": + return tuple(_built(item, context) for item in cast(list[_ValueRow], row[1])) + if tag == "d": + return { + _built(key, context): _built(value, context) + for key, value in cast(list[tuple[_ValueRow, _ValueRow]], row[1]) + } + if tag in ("-", "+"): + number: object = _built(cast(_ValueRow, row[1]), context) + if not isinstance(number, int | float): + raise CompileInputError( + f"Macro arguments in {context[1]} use unary + or - on a value that is not a number", + help="Apply the sign inside the macro, or pass a number literal", + ) + return -number if tag == "-" else number + return None diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index eb8ef33c10..9024e614ec 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -18,16 +18,17 @@ from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( call_site_sql_references, - evaluate_typed_reference, reject_macro_generated_references, report_macro_reference_call_syntax, ) +from sqlbuild.compiler.compile._helpers.render.macro_argument_values import ( + parse_macro_call_arguments, +) from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import ( DECLARATION_REFERENCE_NAMES, MACRO_CONTEXT_PARAMETER_NAME, MACRO_TOKEN, - PYTHON_LITERAL_NAMES, SQL_OPEN_PAREN_TOKEN, SQL_QUOTE_TOKENS, ) @@ -40,6 +41,7 @@ MacroContext, MacroExpansionResult, MacroExpansionState, + ParsedMacroArguments, StaticMacroExport, StaticMacroFault, StaticMacroInventory, @@ -1458,6 +1460,7 @@ def _evaluate_macro_call( _ = state.facts.open_call_site() try: args, kwargs = _parse_macro_arguments( + macro_name=macro_name, args_source=args_source, file_path=file_path, state=state, @@ -1772,6 +1775,7 @@ def _call_loaded_macro( def _parse_macro_arguments( *, + macro_name: str, args_source: str, file_path: Path, state: MacroExpansionState, @@ -1780,167 +1784,31 @@ def _parse_macro_arguments( ) -> tuple[tuple[object, ...], dict[str, object]]: if not args_source.strip(): return (), {} - if native_stage_enabled(NativeStage.MACRO_CALLS): - report_native_fallback(site=NativeFallbackSite.MACRO_ARGUMENTS) - rewritten_args_source: str - placeholder_values: dict[str, object] - rewritten_args_source, placeholder_values = _rewrite_nested_macro_calls( - args_source=args_source, - file_path=file_path, - state=state, - declarations=declarations, - stack=stack, - ) - try: - expression: ast.Expression = ast.parse(f"_macro_call({rewritten_args_source})", mode="eval") - except SyntaxError as error: - raise CompileInputError( - f"Macro arguments in '{file_path}' could not be parsed: {error}" - ) from error - if not isinstance(expression.body, ast.Call): - raise CompileInputError(f"Macro arguments in '{file_path}' could not be parsed") - call_expression: ast.Call = expression.body - args: tuple[object, ...] = tuple( - _evaluate_literal_ast_node( - node=argument, - placeholder_values=placeholder_values, - file_path=file_path, - ) - for argument in call_expression.args - ) - kwargs: dict[str, object] = {} - keyword: ast.keyword - for keyword in call_expression.keywords: - if keyword.arg is None: - raise CompileInputError( - f"Macro arguments in '{file_path}' must not use **kwargs expansion syntax" - ) - kwargs[keyword.arg] = _evaluate_literal_ast_node( - node=keyword.value, - placeholder_values=placeholder_values, - file_path=file_path, - ) - state.facts.record_call_site_refs( - tuple( - evaluate_typed_reference(node=node, file_path=file_path) - for node in ast.walk(call_expression) - if isinstance(node, ast.Call) and node is not call_expression - ) - ) - return args, kwargs - - -def _rewrite_nested_macro_calls( - *, - args_source: str, - file_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - stack: tuple[DeclarationIdentity, ...], -) -> tuple[str, dict[str, object]]: - rewritten_parts: list[str] = [] - placeholder_values: dict[str, object] = {} + nested: list[tuple[int, int]] = [] + nested_values: list[object] = [] cursor: int = 0 - replacement_index: int = 0 - while cursor < len(args_source): - macro_start_index: int | None = _find_next_macro_start(sql=args_source, start_index=cursor) - if macro_start_index is None: - rewritten_parts.append(args_source[cursor:]) - break - rewritten_parts.append(args_source[cursor:macro_start_index]) - nested_result: object - next_index: int - nested_result, next_index = _evaluate_macro_call( + while (start := _find_next_macro_start(sql=args_source, start_index=cursor)) is not None: + nested_value: object + nested_value, cursor = _evaluate_macro_call( sql=args_source, - call_start_index=macro_start_index, + call_start_index=start, file_path=file_path, state=state, declarations=declarations, stack=stack, top_level=False, ) - placeholder: str = f"__sqlbuild_macro_arg_{replacement_index}" - replacement_index += 1 - placeholder_values[placeholder] = nested_result - rewritten_parts.append(placeholder) - cursor = next_index - return "".join(rewritten_parts), placeholder_values - - -def _evaluate_literal_ast_node( - *, - node: ast.AST, - placeholder_values: dict[str, object], - file_path: Path, -) -> object: - if isinstance(node, ast.Constant): - return node.value - if isinstance(node, ast.Call): - return evaluate_typed_reference(node=node, file_path=file_path) - if isinstance(node, ast.Name): - if node.id in placeholder_values: - return placeholder_values[node.id] - if node.id in PYTHON_LITERAL_NAMES: - return ast.literal_eval(node) - if isinstance(node, ast.List): - return [ - _evaluate_literal_ast_node( - node=element, - placeholder_values=placeholder_values, - file_path=file_path, - ) - for element in node.elts - ] - if isinstance(node, ast.Tuple): - return tuple( - _evaluate_literal_ast_node( - node=element, - placeholder_values=placeholder_values, - file_path=file_path, - ) - for element in node.elts - ) - if isinstance(node, ast.Dict): - return { - _evaluate_dict_key_ast_node( - key_node=key, - placeholder_values=placeholder_values, - file_path=file_path, - ): _evaluate_literal_ast_node( - node=value, - placeholder_values=placeholder_values, - file_path=file_path, - ) - for key, value in zip(node.keys, node.values, strict=True) - } - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): - operand: object = _evaluate_literal_ast_node( - node=node.operand, - placeholder_values=placeholder_values, - file_path=file_path, - ) - if not isinstance(operand, int | float): - raise CompileInputError(f"Macro arguments in '{file_path}' use unsupported unary value") - return -operand if isinstance(node.op, ast.USub) else operand - raise CompileInputError( - f"Macro arguments in '{file_path}' must use only Python literals, nested macro calls, " - "and __ref(), __source(), or __seed() references" - ) - - -def _evaluate_dict_key_ast_node( - *, - key_node: ast.AST | None, - placeholder_values: dict[str, object], - file_path: Path, -) -> object: - if key_node is None: - raise CompileInputError(f"Macro arguments in '{file_path}' must not use dict unpacking") - return _evaluate_literal_ast_node( - node=key_node, - placeholder_values=placeholder_values, + nested.append((start, cursor)) + nested_values.append(nested_value) + parsed: ParsedMacroArguments = parse_macro_call_arguments( + args_source=args_source, + nested=tuple(nested), + nested_values=nested_values, + macro_name=macro_name, file_path=file_path, ) + state.facts.record_call_site_refs(parsed.typed_references) + return parsed.args, parsed.kwargs def _find_next_macro_start(*, sql: str, start_index: int) -> int | None: diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 63c94a9167..2fa1b1bc3c 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -110,7 +110,6 @@ MACRO_TOKEN: str = "@" DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" -PYTHON_LITERAL_NAMES: frozenset[str] = frozenset({"True", "False", "None"}) SQL_INTERPOLATION_TOKEN: str = "@@" TEMPLATE_OPEN_TOKEN: str = "${" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 069c4ce8d7..f4ccdefbec 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -67,6 +67,7 @@ VisibilityRecord, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax +from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -337,6 +338,15 @@ class DeclarationScopeBuild: sql_test_scans: SqlTestScanCache +@dataclass(frozen=True) +class ParsedMacroArguments: + """One macro call's argument objects and the typed references written in them, in order.""" + + args: tuple[object, ...] + kwargs: dict[str, object] + typed_references: tuple[SqlResourceRef, ...] + + @dataclass(frozen=True) class MacroContext: """Expose target settings and caller-visible declarations to a Python SQL macro.""" diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index c34afe7e93..047090a4bf 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -59,7 +59,6 @@ class NativeFallbackSite(StrEnum): MACRO_CALL_RESOLUTION = "macro_calls.resolution" MACRO_CALL_SPLICE = "macro_calls.splice" MACRO_CALL_MEMO = "macro_calls.memo" - MACRO_ARGUMENTS = "macro_calls.arguments" MACRO_UNBRIDGED_EXPANSION = "macro_calls.unbridged_expansion" MACRO_BRIDGE_UNAVAILABLE = "macro_calls.bridge_unavailable" REFERENCE_SCAN = "reference_extraction.scan" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index d4134b031f..72cf455472 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -279,17 +279,17 @@ def test_given_enforcement_disabled_when_macro_emits_reference_then_expands_unch MalformedTypedMacroReferenceTestCase( description="non-string name", sql="@base(__ref(1))", - expected_error_fragment="must contain exactly one quoted resource name", + expected_error_fragment="must give __ref() exactly one quoted resource name", ), MalformedTypedMacroReferenceTestCase( description="two names", sql='@base(__ref("a", "b"))', - expected_error_fragment="must contain exactly one quoted resource name", + expected_error_fragment="must give __ref() exactly one quoted resource name", ), MalformedTypedMacroReferenceTestCase( description="keyword name", sql='@base(__ref(name="a"))', - expected_error_fragment="must contain exactly one quoted resource name", + expected_error_fragment="must give __ref() exactly one quoted resource name", ), MalformedTypedMacroReferenceTestCase( description="unsupported reference function", From f629b36808cfca411ed72a3f36656cb66b0b1807 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:01:18 +0100 Subject: [PATCH 05/40] feat(native): expand SQL-test macro mocks through the macro bridge and reuse the native scope lookup --- .../compile/_helpers/assembly/project.py | 19 +++++++----- .../_helpers/macro_bridge/expansion.py | 31 +++++++++++++++++++ .../compile/_helpers/render/declarations.py | 5 ++- .../compile/_helpers/render/macros.py | 5 ++- .../compile/main/_assemble_project.py | 14 ++++++++- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 998ad6b660..8d13e10215 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -152,6 +152,7 @@ CompileSqlFunctionInput, CompileSqlScenarioInput, CompileSqlTestInput, + DeclarationScopeResolver, DynamicColumnContractProof, InferredColumn, MacroContext, @@ -1883,6 +1884,16 @@ def _build_test_model_query_overrides( vars=inputs.effective_vars, ) macro_context: MacroContext = replace(model_macro_context, _enforce_explicit_references=False) + declaration_resolver: DeclarationScopeResolver = build_declaration_scope_resolver( + discovered_inputs=inputs.discovered_inputs, + scope_index=inputs.scope_index, + loaded_macros=inputs.loaded_macros, + lookup=( + inputs.declaration_scope.resolver.lookup + if inputs.declaration_scope is not None + else None + ), + ) overrides: dict[str, str] = {} model_input: CompileModelInput for model_input in model_inputs: @@ -1895,13 +1906,7 @@ def _build_test_model_query_overrides( loaded_macros=inputs.loaded_macros, macro_overrides=test_input.payload.macro_mocks, macro_context=macro_context, - declaration_resolver=( - build_declaration_scope_resolver( - discovered_inputs=inputs.discovered_inputs, - scope_index=inputs.scope_index, - loaded_macros=inputs.loaded_macros, - ) - ), + declaration_resolver=declaration_resolver, ), config_values=model_input.config.values, model_name=model_name, diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py index dd2456cccd..82b7cdbbc9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -102,6 +102,8 @@ def _bridged_call_output( # noqa: PLR0913 call_class: MacroCallClass, ) -> str: facts: MacroExpansionFacts = state.facts + if state.macro_overrides.keys() & set(site.tree_names): + return _mocked_call_output(sql=sql, consumer_path=consumer_path, state=state, site=site) call_text: str = sql[site.start : site.end] prior_relations: tuple[SqlResourceRef, ...] | None = ( tuple(facts.relations) if site.typed_reference_text else None @@ -161,6 +163,35 @@ def _bridged_call_output( # noqa: PLR0913 return facts.render_relation_placeholders(macro_result) +def _mocked_call_output( + *, sql: str, consumer_path: Path, state: MacroExpansionState, site: MacroCallSite +) -> str: + """Run a call whose tree a test mocks; its output differs from the shared memo's, so skip it.""" + + macro_result: object + next_index: int + macro_result, next_index = _evaluate_macro_call( + sql=sql, + call_start_index=site.start, + file_path=consumer_path, + state=state, + declarations=_expansion_declarations(state=state, consumer_path=consumer_path), + stack=(), + top_level=True, + ) + if not isinstance(macro_result, str): + raise CompileInputError( + f"Macro '@{site.name}' in '{consumer_path}' must return a SQL string when used " + "directly in SQL" + ) + if next_index != site.end: + raise NativeStageMismatchError( + f"Native macro call scan of '{consumer_path}' ended at {site.end}, Python at " + f"{next_index}" + ) + return state.facts.render_relation_placeholders(macro_result) + + def _replay_macro_call_event( *, event: MacroCallEvent, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index 3fdbfd73b3..499e8835b4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -33,6 +33,7 @@ ModelSchemaDeclaration, ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.model_loop.constants import ( @@ -168,7 +169,9 @@ def build_declaration_scope_resolver( if loaded_macros is not None: for macro in loaded_macros.values(): declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro - if lookup is None and native_stage_enabled(NativeStage.DECLARATION_SCOPES): + if lookup is not None: + report_native_answer(stage=NativeStage.DECLARATION_SCOPES, kind="resolver_lookups") + elif native_stage_enabled(NativeStage.DECLARATION_SCOPES): report_native_fallback(site=NativeFallbackSite.SCOPE_REBIND_LOOKUP) scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup return DeclarationScopeResolver( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index 9024e614ec..f54161d931 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -1213,11 +1213,10 @@ def expand_sql_macros_result( facts=facts, consumer=consumer, ) - bridge: MacroBridge | None = active_macro_bridge() if not state.macro_overrides else None + bridge: MacroBridge | None = active_macro_bridge() if bridge is None and MACRO_TOKEN in sql and native_stage_enabled(NativeStage.MACRO_CALLS): report_native_fallback( - site=NativeFallbackSite.MACRO_UNBRIDGED_EXPANSION, - kind="macro_mocks" if state.macro_overrides else "outside_compile_inputs", + site=NativeFallbackSite.MACRO_UNBRIDGED_EXPANSION, kind="outside_compile_inputs" ) expanded_sql, spans = ( _expand_sql_macros( diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py index 9cf506e6e5..09b05d4715 100644 --- a/src/sqlbuild/compiler/compile/main/_assemble_project.py +++ b/src/sqlbuild/compiler/compile/main/_assemble_project.py @@ -2,7 +2,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import replace +from functools import partial from pathlib import Path from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -23,7 +25,11 @@ CompilerDiagnostic, PythonSqlReferenceReport, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge @with_collected_compile_diagnostics @@ -38,7 +44,8 @@ def assemble_project( ) -> CompiledProject: """Convert compile inputs into the planner-ready project view.""" - project: CompiledProject = assemble_compiled_project( + assemble: Callable[[], CompiledProject] = partial( + assemble_compiled_project, inputs=inputs, inference_profile=inference_profile, skip_column_inference=skip_column_inference, @@ -46,6 +53,11 @@ def assemble_project( analysis_cache_dir=analysis_cache_dir, analysis_model_names=analysis_model_names, ) + project: CompiledProject = ( + run_with_macro_bridge(stage=assemble) + if native_stage_enabled(NativeStage.MACRO_CALLS) and active_macro_bridge() is None + else assemble() + ) python_sql: PythonSqlReferenceReport = ( python_sql_reference_diagnostics( project=project, discovered_inputs=inputs.discovered_inputs From 302fbbadb60ce98868a32c0361b54b36b943179c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:05:07 +0100 Subject: [PATCH 06/40] feat(native): reject lone surrogates where env vars, vars and macro output enter; delete the fallbacks they forced --- .../_helpers/corpus/failure_cases.py | 37 ++++++++++++++++++ scripts/compiler_differential/constants.py | 6 ++- .../_helpers/attachment/model_config.py | 4 +- .../compiler/compile/_helpers/refs/native.py | 5 +-- .../compile/_helpers/render/macros.py | 11 ++++++ .../compile/_helpers/render/sql_vars.py | 4 +- .../compile/_helpers/render/templating.py | 28 ++++++++++++- .../compile/classes/unicode_environment.py | 35 +++++++++++++++++ src/sqlbuild/compiler/frontier/types.py | 2 - .../macro_bridge/_helpers/splicing.py | 23 ----------- .../macro_bridge/classes/macro_bridge.py | 39 +++++++------------ .../main/_expand_native_config_templates.py | 5 +-- .../cli/commands/main/compile/_test_types.py | 3 +- .../compile/test_compile_macro_call_store.py | 13 +++++-- .../src/sqlbuild/compiler/compile/helpers.py | 3 +- 15 files changed, 146 insertions(+), 72 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/classes/unicode_environment.py delete mode 100644 src/sqlbuild/compiler/macro_bridge/_helpers/splicing.py diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index b68503f980..2f0612cc46 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -33,6 +33,7 @@ FAILURE_CONFIG_PATH, FAILURE_MART_PATH, FAILURE_SOURCES_PATH, + FAILURE_UNDECODABLE_ENV_VAR, ) from scripts.compiler_differential.models import FailureCase @@ -721,6 +722,42 @@ def engine_error_cases() -> tuple[FailureCase, ...]: ), expected_help=("Pass None, or a string the macro understands, instead of '...'"), ), + failure_case( + name="engine-error-undecodable-environment-variable", + expected_code="P001", + files=mart_body_files( + f"SELECT customer_id, '@@ENV:{FAILURE_UNDECODABLE_ENV_VAR}' AS region\n" + 'FROM __ref("stg_orders")\n' + ), + expected_message=( + f"Environment variable '{FAILURE_UNDECODABLE_ENV_VAR}' is not valid UTF-8 text" + ), + expected_help=( + "Its value is b'\\xff'; set it to UTF-8 text, since SQLBuild cannot send " + "undecodable bytes to a warehouse" + ), + ), + failure_case( + name="engine-error-macro-output-lone-surrogate", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + " return expression + chr(0xDCFF)\n" + ), + **mart_body_files( + "SELECT customer_id, @cents('amount') AS cents\nFROM __ref(\"stg_orders\")\n" + ), + }, + expected_message=( + f"Macro '@cents' in '/{FAILURE_MART_PATH}' returned text with the lone " + "surrogate '\\udcff', which is not valid Unicode" + ), + expected_help=( + "Return valid Unicode text from the macro; lone surrogates usually come from " + "bytes decoded with errors='surrogateescape'" + ), + ), failure_case( name="engine-error-week-date-cursor-start", expected_code="P001", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 1ee5525a4d..a2b7b2c605 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -330,7 +330,11 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ) GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" GENERATOR_MISSING_ENV_VAR: str = "SQB_DIFFERENTIAL_UNSET" -GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} +FAILURE_UNDECODABLE_ENV_VAR: str = "SQB_FAILURE_UNDECODABLE_REGION" +GENERATOR_ENVIRONMENT: dict[str, str] = { + GENERATOR_CHANNEL_ENV_VAR: "web-orders", + FAILURE_UNDECODABLE_ENV_VAR: "\udcff", +} GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" GENERATOR_CRLF: str = "\r\n" GENERATOR_BOM: str = "\ufeff" diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index a80106dbd7..3e4d49941d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from dataclasses import replace from pathlib import Path from typing import cast @@ -36,6 +35,7 @@ contains_template_data, record_template_reads, ) +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_AUDIT_OVERRIDE_KEYS, @@ -882,7 +882,7 @@ def native_model_config_session( project_config.path_defaults, (SqlHookEntry, NamedSqlHookEntry, PythonHookEntry), ), - (inputs.effective_vars, os.environ), + (inputs.effective_vars, UnicodeEnvironment()), (inputs.effective_target_name, inputs.run_id), None if target is None else (target.database, target.schema), ), diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/native.py b/src/sqlbuild/compiler/compile/_helpers/refs/native.py index 3e1f24cbb9..febe9223ac 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/native.py @@ -32,10 +32,7 @@ def extract_native_sql_references( ) -> SqlReferenceScan | SqlReferenceScanFailure | None: """Return the references and rejected calls, Python's located error, or None for Python.""" - try: - extraction: _NativeExtraction = _scanner(syntax).extract(sql) - except UnicodeError: - return None + extraction: _NativeExtraction = _scanner(syntax).extract(sql) if extraction is None: return None scanned, failure = extraction diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index f54161d931..ecb3188b69 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -1743,6 +1743,17 @@ def _record_macro_declaration_usage( def _validate_final_macro_sql(*, macro_name: str, file_path: Path, macro_result: str) -> None: + try: + _ = macro_result.encode("utf-8") + except UnicodeEncodeError as error: + raise CompileInputError( + f"Macro '@{macro_name}' in '{file_path}' returned text with the lone surrogate " + f"{macro_result[error.start]!r}, which is not valid Unicode", + help=( + "Return valid Unicode text from the macro; lone surrogates usually come from " + "bytes decoded with errors='surrogateescape'" + ), + ) from None generated_calls: tuple[str, ...] = _find_sqlbuild_call_names(macro_result) if not generated_calls: return diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 41ebdfee10..0ecdfe8b25 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import sys import unicodedata from collections.abc import Mapping @@ -18,6 +17,7 @@ expand_sql_macros_with_spans, ) from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -179,7 +179,7 @@ def interpolate_sql_batch( rows = _native.interpolate_sql_batch( [(sql, str(file_path)) for sql, file_path in sqls], - (effective_vars, os.environ, context_values, _render_variable), + (effective_vars, UnicodeEnvironment(), context_values, _render_variable), (sys.version_info[0], sys.version_info[1]), unicodedata.unidata_version, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index f75b39ba2b..1f6a0d660c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -2,11 +2,12 @@ from __future__ import annotations -import os from typing import cast import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, TEMPLATE_OPEN_TOKEN +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( expand_native_config_templates, @@ -22,8 +23,10 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: """Resolve merged effective vars with recursive `${name}` expansion.""" + for name, value in raw_values.items(): + _reject_lone_surrogates(name=name, value=value, root=value) outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( - raw_values, os.environ + raw_values, UnicodeEnvironment() ) record_template_reads(tuple(outcome[1])) if isinstance(outcome[0], _native.NativeConfigError): @@ -31,6 +34,27 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: return cast(dict[str, object], outcome[0]) +def _reject_lone_surrogates(*, name: str, value: object, root: object) -> None: + if isinstance(value, dict): + for item in (*value.keys(), *value.values()): + _reject_lone_surrogates(name=name, value=item, root=root) + elif isinstance(value, list | tuple): + for item in value: + _reject_lone_surrogates(name=name, value=item, root=root) + elif isinstance(value, str): + try: + _ = value.encode("utf-8") + except UnicodeEncodeError as error: + raise CompileInputError( + f"Variable '{name}' holds the lone surrogate {value[error.start]!r}, which is not " + "valid Unicode text", + help=( + f"Set '{name}' to valid Unicode text in --vars or the [vars] table; its " + f"current value is {root!r}" + ), + ) from None + + def contains_template_data(value: object) -> bool: """Return whether a supported nested value contains a template token.""" diff --git a/src/sqlbuild/compiler/compile/classes/unicode_environment.py b/src/sqlbuild/compiler/compile/classes/unicode_environment.py new file mode 100644 index 0000000000..f43db63af5 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/unicode_environment.py @@ -0,0 +1,35 @@ +"""The process environment as compile text reads it, rejecting values that are not UTF-8.""" + +from __future__ import annotations + +import os +from collections.abc import Iterator, Mapping + +from sqlbuild.compiler.compile.exceptions import CompileInputError + + +class UnicodeEnvironment(Mapping[str, str]): + """`os.environ`, raising a clear error for a value holding undecodable bytes.""" + + def __getitem__(self, name: str) -> str: + value: str = os.environ[name] + try: + _ = value.encode("utf-8") + except UnicodeEncodeError: + raise CompileInputError( + f"Environment variable '{name}' is not valid UTF-8 text", + help=( + f"Its value is {value.encode('utf-8', 'surrogateescape')!r}; set it to UTF-8 " + "text, since SQLBuild cannot send undecodable bytes to a warehouse" + ), + ) from None + return value + + def __contains__(self, name: object) -> bool: + return name in os.environ + + def __iter__(self) -> Iterator[str]: + return iter(os.environ) + + def __len__(self) -> int: + return len(os.environ) diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 047090a4bf..8ae3eadf74 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -57,8 +57,6 @@ class NativeFallbackSite(StrEnum): LINT_EXPANSION = "model_loop.lint_expansion" MACRO_CALL_SCAN = "macro_calls.scan" MACRO_CALL_RESOLUTION = "macro_calls.resolution" - MACRO_CALL_SPLICE = "macro_calls.splice" - MACRO_CALL_MEMO = "macro_calls.memo" MACRO_UNBRIDGED_EXPANSION = "macro_calls.unbridged_expansion" MACRO_BRIDGE_UNAVAILABLE = "macro_calls.bridge_unavailable" REFERENCE_SCAN = "reference_extraction.scan" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/splicing.py b/src/sqlbuild/compiler/macro_bridge/_helpers/splicing.py deleted file mode 100644 index f160f2f1e8..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/splicing.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Splice macro outputs in Python for text the native splice cannot read.""" - -from __future__ import annotations - - -def splice_text( - *, sql: str, bounds: list[tuple[int, int]], outputs: list[str] -) -> tuple[str, list[tuple[int, int, int, int]]]: - """Replace each `(start, end)` span with its output, as `splice_macro_calls` does.""" - - parts: list[str] = [] - spans: list[tuple[int, int, int, int]] = [] - cursor: int = 0 - length: int = 0 - for (start, end), output in zip(bounds, outputs, strict=True): - parts.append(sql[cursor:start]) - length += start - cursor - parts.append(output) - spans.append((start, end, length, length + len(output))) - length += len(output) - cursor = end - parts.append(sql[cursor:]) - return "".join(parts), spans diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index 2ee2d9cf15..0cf496ca68 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -16,7 +16,6 @@ ) from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback from sqlbuild.compiler.frontier.types import NativeFallbackSite -from sqlbuild.compiler.macro_bridge._helpers.splicing import splice_text from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( digest_module_files, module_digests_metadata, @@ -136,13 +135,9 @@ def _observe_modules(self) -> bool: def scan(self, sql: str) -> tuple[MacroCallSite, ...] | None: """Return the top-level call sites of `sql`, or None when Python must expand it.""" - try: - rows: list[tuple[int, int, str, list[str], bool]] | None = ( - _native.scan_macro_call_sites(sql, self._python_version, self._unicode_version) - ) - except UnicodeEncodeError: - report_native_fallback(site=NativeFallbackSite.MACRO_CALL_SCAN, kind="unencodable") - return None + rows: list[tuple[int, int, str, list[str], bool]] | None = _native.scan_macro_call_sites( + sql, self._python_version, self._unicode_version + ) if rows is None: report_native_fallback(site=NativeFallbackSite.MACRO_CALL_SCAN) return None @@ -251,19 +246,15 @@ def record( if self._store_trusted and len(sys.modules) != len(self._observed_modules): _ = self._observe_modules() - try: - self._memo.record( - class_id, - call_text, - ( - sql, - [(relation.kind.value, relation.name) for relation in relations], - list(events), - ), - ) - except UnicodeEncodeError: - report_native_fallback(site=NativeFallbackSite.MACRO_CALL_MEMO, kind="unencodable") - return + self._memo.record( + class_id, + call_text, + ( + sql, + [(relation.kind.value, relation.name) for relation in relations], + list(events), + ), + ) def splice( self, *, sql: str, sites: tuple[MacroCallSite, ...], outputs: list[str] @@ -273,11 +264,7 @@ def splice( rendered: str spans: list[tuple[int, int, int, int]] bounds: list[tuple[int, int]] = [(site.start, site.end) for site in sites] - try: - rendered, spans = _native.splice_macro_calls(sql, bounds, outputs) - except UnicodeEncodeError: - report_native_fallback(site=NativeFallbackSite.MACRO_CALL_SPLICE, kind="unencodable") - rendered, spans = splice_text(sql=sql, bounds=bounds, outputs=outputs) + rendered, spans = _native.splice_macro_calls(sql, bounds, outputs) return rendered, tuple( ExpansionSpan( source_start=source_start, diff --git a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py index 405a2e575f..c955e8dd0a 100644 --- a/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py +++ b/src/sqlbuild/compiler/model_config/main/_expand_native_config_templates.py @@ -2,9 +2,8 @@ from __future__ import annotations -import os - import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.model_config.models import ( NativeTemplateExpansion, NativeTemplateRejection, @@ -24,7 +23,7 @@ def expand_native_config_templates( outcome: tuple[object, list[tuple[str, str]]] = _native.expand_config_templates( value, - (variables, os.environ, context_values), + (variables, UnicodeEnvironment(), context_values), ( flags.allow_context, flags.preserve_context_tokens, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 977aafbf7b..ab4ec1a811 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -955,12 +955,13 @@ class NativeMacroErrorTestCase: @dataclass(frozen=True) class UnkeyableMacroCallTestCase: - """Compile arguments whose values no store key can hold; every engine must match Python.""" + """Compile arguments holding text that is not valid Unicode; every engine rejects them.""" description: str project_files: dict[str, str] compile_args: tuple[str, ...] expected_returncodes: tuple[int, ...] + expected_report_fragment: str @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 544ed4fc50..87863d58a5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -515,15 +515,19 @@ def test_given_no_proc_filesystem_when_compiling_repeatedly_then_the_store_still "test_case", [ UnkeyableMacroCallTestCase( - description="context macro with a lone surrogate var", + description="a lone surrogate var is rejected before any macro runs", project_files=_TAGGED_STAGING, compile_args=("--vars", '{"regions": ["\\udcff"]}'), - expected_returncodes=(0, 0, 0), + expected_returncodes=(1, 1, 1), + expected_report_fragment=( + "Variable 'regions' holds the lone surrogate '\\udcff', which is not valid " + "Unicode text" + ), ), ], ids=lambda case: case.description, ) -def test_given_var_no_store_key_can_hold_when_compiling_then_every_engine_matches_python( +def test_given_unencodable_var_when_compiling_then_every_engine_rejects_it( test_case: UnkeyableMacroCallTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "orders" @@ -544,6 +548,7 @@ def test_given_var_no_store_key_can_hold_when_compiling_then_every_engine_matche tuple(run.returncode for run in runs), {report_without_engine(run) for run in runs} == {report_without_engine(runs[0])}, [run.compiled for run in runs] == [runs[0].compiled] * len(runs), - ) == (test_case.expected_returncodes, True, True), tuple( + all(test_case.expected_report_fragment in run.report + run.stderr for run in runs), + ) == (test_case.expected_returncodes, True, True, True), tuple( run.report + run.stderr for run in runs ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index eeb2dbc841..8378aafded 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -51,8 +51,7 @@ "@n(@mm(3), '@m(4)')", "@tag(__ref('orders'))", "@tag(__seed('rates'))", "@tag(__ref('customers'))", "@cols(__ref('orders'))", "@gen()", "@nope(1)", "@m(x)", "@ab c(1)", "@m('é')", "'@m(1)'", "-- @m(1)\n", "/* @n() */", "$$@m()$$", "`@m()`", "@@v", - "@", " ", "\n", "\u3000", "é", "'", "(", ")", "SELECT ", ", ", "\"q\"", "'\udcff'", - "@m('\udcff')", "@sur()", + "@", " ", "\n", "\u3000", "é", "'", "(", ")", "SELECT ", ", ", "\"q\"", "@sur()", ) # fmt: skip _DIFFERENTIAL_FILE: Path = Path("models/orders_summary.sql") From 4cce632b1312246dce8e149d1b8a3c2173276688 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:36:10 +0100 Subject: [PATCH 07/40] test(native): count template, effective-var, macro-argument and scope-lookup answers; regenerate baselines --- .../native_fallbacks.toml | 338 ++++++++---------- .../_helpers/render/macro_argument_values.py | 3 + .../compile/_helpers/render/templating.py | 4 + .../engine-error-macro-argument-bytes.json | 19 + .../engine-error-macro-argument-complex.json | 19 + .../engine-error-macro-argument-ellipsis.json | 19 + .../engine-error-macro-argument-syntax.json | 19 + ...ine-error-macro-output-lone-surrogate.json | 19 + ...rror-undecodable-environment-variable.json | 19 + .../failure/macro-arguments-do-not-parse.json | 3 +- .../failure/macro-non-literal-argument.json | 3 +- 11 files changed, 266 insertions(+), 199 deletions(-) create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-complex.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-output-lone-surrogate.json create mode 100644 tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json diff --git a/scripts/compiler_differential/native_fallbacks.toml b/scripts/compiler_differential/native_fallbacks.toml index 6d596e22db..656fd241c5 100644 --- a/scripts/compiler_differential/native_fallbacks.toml +++ b/scripts/compiler_differential/native_fallbacks.toml @@ -25,7 +25,7 @@ engine = "native" stage = "attachments" site = "attachments.native" kind = "cursor_intrinsic_rejections" -counts = { example = 160, failure = 177, fixture = 156, seed = 961 } +counts = { example = 160, failure = 170, fixture = 156, seed = 961 } [[entry]] engine = "native" @@ -39,7 +39,7 @@ engine = "native" stage = "attachments" site = "attachments.native" kind = "omitted_select_scans" -counts = { example = 24, failure = 22, fixture = 32, seed = 193 } +counts = { example = 24, failure = 21, fixture = 32, seed = 193 } [[entry]] engine = "native" @@ -67,14 +67,14 @@ engine = "native" stage = "declaration_files" site = "declaration_files.native" kind = "failed_files" -counts = { failure = 9 } +counts = { failure = 10 } [[entry]] engine = "native" stage = "declaration_files" site = "declaration_files.native" kind = "parsed_files" -counts = { example = 32, failure = 69, fixture = 20, seed = 799 } +counts = { example = 32, failure = 74, fixture = 20, seed = 799 } [[entry]] engine = "native" @@ -87,15 +87,15 @@ counts = { example = 8, failure = 3, fixture = 4, seed = 43 } engine = "native" stage = "declaration_scopes" site = "declaration_scopes.native" -kind = "scope_indexes" -counts = { example = 12, failure = 229, fixture = 12, seed = 44 } +kind = "resolver_lookups" +counts = { example = 12, failure = 231, fixture = 12, seed = 73 } [[entry]] engine = "native" stage = "declaration_scopes" -site = "declaration_scopes.rebind_lookup" -kind = "deferred" -counts = { failure = 1, seed = 802 } +site = "declaration_scopes.native" +kind = "scope_indexes" +counts = { example = 12, failure = 230, fixture = 12, seed = 44 } [[entry]] engine = "native" @@ -109,7 +109,7 @@ engine = "native" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 294, fixture = 12, seed = 50 } +counts = { example = 12, failure = 296, fixture = 12, seed = 50 } [[entry]] engine = "native" @@ -121,16 +121,16 @@ counts = { example = 18, fixture = 15, seed = 192 } [[entry]] engine = "native" stage = "macro_calls" -site = "macro_calls.arguments" -kind = "deferred" -counts = { example = 6, failure = 14, fixture = 5, seed = 484 } +site = "macro_calls.native" +kind = "bridged_calls" +counts = { example = 32, failure = 9, fixture = 28, seed = 748 } [[entry]] engine = "native" stage = "macro_calls" site = "macro_calls.native" -kind = "bridged_calls" -counts = { example = 32, failure = 9, fixture = 28, seed = 466 } +kind = "parsed_arguments" +counts = { example = 6, failure = 13, fixture = 5, seed = 369 } [[entry]] engine = "native" @@ -139,13 +139,6 @@ site = "macro_calls.resolution" kind = "deferred" counts = { failure = 4, seed = 3 } -[[entry]] -engine = "native" -stage = "macro_calls" -site = "macro_calls.unbridged_expansion" -kind = "macro_mocks" -counts = { seed = 238 } - [[entry]] engine = "native" stage = "model_config" @@ -165,56 +158,28 @@ engine = "native" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 310, fixture = 64, seed = 914 } +counts = { example = 56, failure = 307, fixture = 64, seed = 890 } [[entry]] engine = "native" stage = "model_config" site = "model_config.native" -kind = "validations" -counts = { example = 56, failure = 383, fixture = 60, seed = 1115 } - -[[entry]] -engine = "native" -stage = "model_config" -site = "model_config.python_templates" -kind = "api" -counts = { example = 16, failure = 2, seed = 36 } - -[[entry]] -engine = "native" -stage = "model_config" -site = "model_config.python_templates" -kind = "config" -counts = { example = 12, failure = 119, fixture = 8, seed = 36 } - -[[entry]] -engine = "native" -stage = "model_config" -site = "model_config.python_templates" -kind = "connection" -counts = { example = 18, failure = 121, fixture = 9, seed = 63 } - -[[entry]] -engine = "native" -stage = "model_config" -site = "model_config.python_templates" kind = "effective_vars" -counts = { example = 29, failure = 244, fixture = 15, seed = 121 } +counts = { example = 29, failure = 245, fixture = 15, seed = 121 } [[entry]] engine = "native" stage = "model_config" -site = "model_config.python_templates" -kind = "seed_target" -counts = { example = 8, seed = 30 } +site = "model_config.native" +kind = "template_expansions" +counts = { example = 230, failure = 1588, fixture = 333, seed = 1524 } [[entry]] engine = "native" stage = "model_config" -site = "model_config.python_templates" -kind = "source_target" -counts = { example = 8, seed = 23 } +site = "model_config.native" +kind = "validations" +counts = { example = 56, failure = 372, fixture = 60, seed = 1091 } [[entry]] engine = "native" @@ -228,7 +193,7 @@ engine = "native" stage = "model_loop" site = "model_loop.cursor_intrinsic_validation" kind = "deferred" -counts = { example = 56, failure = 413, fixture = 64, seed = 1124 } +counts = { example = 56, failure = 402, fixture = 64, seed = 1100 } [[entry]] engine = "native" @@ -237,68 +202,26 @@ site = "model_loop.declaration_context" kind = "no_resource" counts = { example = 24, failure = 9, fixture = 20, seed = 245 } -[[entry]] -engine = "native" -stage = "model_loop" -site = "model_loop.declaration_references" -kind = "after_variables" -counts = { seed = 3 } - [[entry]] engine = "native" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 424, fixture = 64, seed = 1124 } +counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } [[entry]] engine = "native" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 466, fixture = 72, seed = 1181 } - -[[entry]] -engine = "native" -stage = "model_loop" -site = "model_loop.sql_variables" -kind = "deferred" -counts = { failure = 3, seed = 195 } - -[[entry]] -engine = "native" -stage = "model_loop" -site = "model_loop.sql_variables" -kind = "error" -counts = { failure = 2 } +counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } [[entry]] engine = "native" stage = "reference_extraction" site = "reference_extraction.native" kind = "reference_scans" -counts = { example = 250, failure = 479, fixture = 249, seed = 1532 } - -[[entry]] -engine = "native-preview" -stage = "analysis" -site = "project_assembly" -kind = "syntax_error" -counts = { failure = 3 } - -[[entry]] -engine = "native-preview" -stage = "analysis" -site = "project_assembly" -kind = "unsupported_dialect" -counts = { failure = 1 } - -[[entry]] -engine = "native-preview" -stage = "analysis" -site = "sql_test_assembly" -kind = "macro_mocks" -counts = { seed = 29 } +counts = { example = 250, failure = 463, fixture = 249, seed = 1523 } [[entry]] engine = "native-preview" @@ -319,7 +242,7 @@ engine = "native-preview" stage = "attachments" site = "attachments.native" kind = "cursor_intrinsic_rejections" -counts = { example = 160, failure = 177, fixture = 156, seed = 961 } +counts = { example = 160, failure = 170, fixture = 156, seed = 961 } [[entry]] engine = "native-preview" @@ -333,7 +256,7 @@ engine = "native-preview" stage = "attachments" site = "attachments.native" kind = "omitted_select_scans" -counts = { example = 24, failure = 22, fixture = 32, seed = 193 } +counts = { example = 24, failure = 21, fixture = 32, seed = 193 } [[entry]] engine = "native-preview" @@ -361,28 +284,28 @@ engine = "native-preview" stage = "contracts" site = "contracts.native" kind = "contract_validations" -counts = { example = 9, failure = 113, fixture = 6, seed = 25 } +counts = { example = 9, failure = 109, fixture = 6, seed = 25 } [[entry]] engine = "native-preview" stage = "contracts" site = "contracts.native" kind = "promotion_checks" -counts = { example = 12, failure = 113, fixture = 8, seed = 33 } +counts = { example = 12, failure = 109, fixture = 8, seed = 33 } [[entry]] engine = "native-preview" stage = "declaration_files" site = "declaration_files.native" kind = "failed_files" -counts = { failure = 9 } +counts = { failure = 10 } [[entry]] engine = "native-preview" stage = "declaration_files" site = "declaration_files.native" kind = "parsed_files" -counts = { example = 32, failure = 69, fixture = 20, seed = 799 } +counts = { example = 32, failure = 74, fixture = 20, seed = 799 } [[entry]] engine = "native-preview" @@ -395,15 +318,15 @@ counts = { example = 8, failure = 3, fixture = 4, seed = 43 } engine = "native-preview" stage = "declaration_scopes" site = "declaration_scopes.native" -kind = "scope_indexes" -counts = { example = 12, failure = 229, fixture = 12, seed = 44 } +kind = "resolver_lookups" +counts = { example = 12, failure = 231, fixture = 12, seed = 73 } [[entry]] engine = "native-preview" stage = "declaration_scopes" -site = "declaration_scopes.rebind_lookup" -kind = "deferred" -counts = { failure = 1, seed = 802 } +site = "declaration_scopes.native" +kind = "scope_indexes" +counts = { example = 12, failure = 230, fixture = 12, seed = 44 } [[entry]] engine = "native-preview" @@ -417,14 +340,14 @@ engine = "native-preview" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 294, fixture = 12, seed = 50 } +counts = { example = 12, failure = 296, fixture = 12, seed = 50 } [[entry]] engine = "native-preview" stage = "lineage_facts" site = "lineage_facts.native" kind = "lineage_graphs" -counts = { example = 9, failure = 113, fixture = 6, seed = 22 } +counts = { example = 9, failure = 109, fixture = 6, seed = 22 } [[entry]] engine = "native-preview" @@ -436,16 +359,16 @@ counts = { example = 18, fixture = 15, seed = 192 } [[entry]] engine = "native-preview" stage = "macro_calls" -site = "macro_calls.arguments" -kind = "deferred" -counts = { example = 6, failure = 14, fixture = 5, seed = 484 } +site = "macro_calls.native" +kind = "bridged_calls" +counts = { example = 32, failure = 9, fixture = 28, seed = 748 } [[entry]] engine = "native-preview" stage = "macro_calls" site = "macro_calls.native" -kind = "bridged_calls" -counts = { example = 32, failure = 9, fixture = 28, seed = 466 } +kind = "parsed_arguments" +counts = { example = 6, failure = 13, fixture = 5, seed = 369 } [[entry]] engine = "native-preview" @@ -454,13 +377,6 @@ site = "macro_calls.resolution" kind = "deferred" counts = { failure = 4, seed = 3 } -[[entry]] -engine = "native-preview" -stage = "macro_calls" -site = "macro_calls.unbridged_expansion" -kind = "macro_mocks" -counts = { seed = 238 } - [[entry]] engine = "native-preview" stage = "model_analysis" @@ -487,14 +403,14 @@ engine = "native-preview" stage = "model_analysis" site = "model_analysis.native" kind = "expression_shapes" -counts = { example = 4, failure = 119, fixture = 4, seed = 100 } +counts = { example = 4, failure = 114, fixture = 4, seed = 100 } [[entry]] engine = "native-preview" stage = "model_analysis" site = "model_analysis.native" kind = "model_analyses" -counts = { example = 56, failure = 232, fixture = 52, seed = 698 } +counts = { example = 56, failure = 221, fixture = 52, seed = 686 } [[entry]] engine = "native-preview" @@ -515,42 +431,28 @@ engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 310, fixture = 64, seed = 914 } +counts = { example = 56, failure = 307, fixture = 64, seed = 890 } [[entry]] engine = "native-preview" stage = "model_config" site = "model_config.native" -kind = "validations" -counts = { example = 56, failure = 383, fixture = 60, seed = 1115 } - -[[entry]] -engine = "native-preview" -stage = "model_config" -site = "model_config.python_templates" -kind = "api" -counts = { example = 16, failure = 2, seed = 36 } - -[[entry]] -engine = "native-preview" -stage = "model_config" -site = "model_config.python_templates" -kind = "config" -counts = { example = 12, failure = 119, fixture = 8, seed = 36 } +kind = "effective_vars" +counts = { example = 29, failure = 245, fixture = 15, seed = 121 } [[entry]] engine = "native-preview" stage = "model_config" -site = "model_config.python_templates" -kind = "connection" -counts = { example = 18, failure = 121, fixture = 9, seed = 63 } +site = "model_config.native" +kind = "template_expansions" +counts = { example = 230, failure = 1588, fixture = 333, seed = 1524 } [[entry]] engine = "native-preview" stage = "model_config" -site = "model_config.python_templates" -kind = "effective_vars" -counts = { example = 29, failure = 244, fixture = 15, seed = 121 } +site = "model_config.native" +kind = "validations" +counts = { example = 56, failure = 372, fixture = 60, seed = 1091 } [[entry]] engine = "native-preview" @@ -564,7 +466,7 @@ engine = "native-preview" stage = "model_loop" site = "model_loop.cursor_intrinsic_validation" kind = "deferred" -counts = { example = 56, failure = 413, fixture = 64, seed = 1124 } +counts = { example = 56, failure = 402, fixture = 64, seed = 1100 } [[entry]] engine = "native-preview" @@ -573,93 +475,135 @@ site = "model_loop.declaration_context" kind = "no_resource" counts = { example = 24, failure = 9, fixture = 20, seed = 245 } -[[entry]] -engine = "native-preview" -stage = "model_loop" -site = "model_loop.declaration_references" -kind = "after_variables" -counts = { seed = 3 } - [[entry]] engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 424, fixture = 64, seed = 1124 } +counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } [[entry]] engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 466, fixture = 72, seed = 1181 } - -[[entry]] -engine = "native-preview" -stage = "model_loop" -site = "model_loop.sql_variables" -kind = "deferred" -counts = { failure = 3, seed = 195 } - -[[entry]] -engine = "native-preview" -stage = "model_loop" -site = "model_loop.sql_variables" -kind = "error" -counts = { failure = 2 } +counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } [[entry]] engine = "native-preview" stage = "project_assembly" site = "project_assembly.assembly" kind = "deferred" -counts = { failure = 4 } - -[[entry]] -engine = "native-preview" -stage = "project_assembly" -site = "project_assembly.native" -kind = "project_assemblies" -counts = { example = 12, failure = 115, fixture = 8, seed = 36 } +counts = { example = 12, failure = 114, fixture = 8, seed = 36 } [[entry]] engine = "native-preview" stage = "reference_extraction" site = "reference_extraction.native" kind = "reference_scans" -counts = { example = 250, failure = 477, fixture = 249, seed = 1532 } +counts = { example = 250, failure = 463, fixture = 249, seed = 1523 } [[entry]] engine = "native-preview" stage = "semantic_checks" site = "semantic_checks.native" kind = "semantic_completions" -counts = { example = 12, failure = 115, fixture = 8, seed = 36 } +counts = { example = 12, failure = 111, fixture = 8, seed = 36 } [[entry]] engine = "native-preview" stage = "sql_test_glue" site = "sql_test_glue.assembly" kind = "deferred" -counts = { seed = 29 } +counts = { example = 12, failure = 111, fixture = 8, seed = 36 } [[entry]] engine = "native-preview" stage = "sql_test_glue" site = "sql_test_glue.native" -kind = "sql_test_assemblies" -counts = { example = 16, failure = 7, fixture = 20, seed = 143 } +kind = "sql_test_plans" +counts = { example = 1, failure = 4, fixture = 1, seed = 9 } [[entry]] -engine = "native-preview" -stage = "sql_test_glue" -site = "sql_test_glue.native" -kind = "sql_test_plans" -counts = { example = 1, failure = 5, fixture = 1, seed = 9 } +engine = "python" +stage = "declaration_files" +site = "declaration_files.native" +kind = "failed_files" +counts = { failure = 10 } + +[[entry]] +engine = "python" +stage = "declaration_files" +site = "declaration_files.native" +kind = "parsed_files" +counts = { example = 32, failure = 74, fixture = 20, seed = 799 } + +[[entry]] +engine = "python" +stage = "declaration_files" +site = "declaration_files.native" +kind = "seed_files" +counts = { example = 8, failure = 3, fixture = 4, seed = 43 } + +[[entry]] +engine = "python" +stage = "declaration_scopes" +site = "declaration_scopes.native" +kind = "resolver_lookups" +counts = { example = 12, failure = 231, fixture = 12, seed = 73 } + +[[entry]] +engine = "python" +stage = "declaration_scopes" +site = "declaration_scopes.native" +kind = "scope_indexes" +counts = { example = 12, failure = 230, fixture = 12, seed = 44 } + +[[entry]] +engine = "python" +stage = "declaration_scopes" +site = "declaration_scopes.relationship_ctes" +kind = "expected_names" +counts = { example = 1, failure = 3, fixture = 3, seed = 26 } [[entry]] engine = "python" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 294, fixture = 12, seed = 50 } +counts = { example = 12, failure = 296, fixture = 12, seed = 50 } + +[[entry]] +engine = "python" +stage = "macro_calls" +site = "macro_calls.native" +kind = "parsed_arguments" +counts = { example = 32, failure = 13, fixture = 28, seed = 840 } + +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.native" +kind = "effective_vars" +counts = { example = 29, failure = 245, fixture = 15, seed = 121 } + +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.native" +kind = "template_expansions" +counts = { example = 286, failure = 1597, fixture = 333, seed = 2503 } + +[[entry]] +engine = "python" +stage = "model_loop" +site = "model_loop.native" +kind = "declaration_expansions" +counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } + +[[entry]] +engine = "python" +stage = "model_loop" +site = "model_loop.native" +kind = "variable_substitutions" +counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py index a5c85a649c..f7a9b563d3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py @@ -12,6 +12,8 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ParsedMacroArguments +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage type _ValueRow = tuple[object, ...] @@ -35,6 +37,7 @@ def parse_macro_call_arguments( help=cast(str, help_text), ) _, positional, keywords, references = parsed + report_native_answer(stage=NativeStage.MACRO_CALLS, kind="parsed_arguments") context: tuple[Sequence[object], str] = (nested_values, f"'{file_path}' in {label}") return ParsedMacroArguments( args=tuple(_built(row, context) for row in cast(list[_ValueRow], positional)), diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 1f6a0d660c..3911006f07 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -8,6 +8,8 @@ from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, TEMPLATE_OPEN_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( expand_native_config_templates, @@ -29,6 +31,7 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: raw_values, UnicodeEnvironment() ) record_template_reads(tuple(outcome[1])) + report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="effective_vars") if isinstance(outcome[0], _native.NativeConfigError): raise native_config_error(error=outcome[0], bridge_independent=True) return cast(dict[str, object], outcome[0]) @@ -91,6 +94,7 @@ def expand_template_data( context_label=context_label, ) record_template_reads(outcome.reads) + report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="template_expansions") if isinstance(outcome, NativeTemplateRejection): raise native_config_error(error=outcome.error, bridge_independent=True) return outcome.value diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json b/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json new file mode 100644 index 0000000000..72b9d3d540 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments in '/models/marts/customer_totals.sql' use a bytes literal at line 1, column 1 of the '@cents' arguments", + "help": "Pass text as a string without the b prefix, for example 'orders' instead of b'orders'" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json b/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json new file mode 100644 index 0000000000..b7e1c02328 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments in '/models/marts/customer_totals.sql' use a complex number literal at line 1, column 1 of the '@cents' arguments", + "help": "Pass int or float numbers; give a complex value's real and imaginary parts as two arguments" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json b/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json new file mode 100644 index 0000000000..ca37c846b6 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments in '/models/marts/customer_totals.sql' use '...' (Ellipsis) at line 1, column 1 of the '@cents' arguments", + "help": "Pass None, or a string the macro understands, instead of '...'" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json b/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json new file mode 100644 index 0000000000..05bbb6a514 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments in '/models/marts/customer_totals.sql' could not be parsed: a value is missing here at line 1, column 10 of the '@cents' arguments", + "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-output-lone-surrogate.json b/tests/goldens/compiler/failure/engine-error-macro-output-lone-surrogate.json new file mode 100644 index 0000000000..92fc0bc397 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-output-lone-surrogate.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro '@cents' in '/models/marts/customer_totals.sql' returned text with the lone surrogate '\\udcff', which is not valid Unicode", + "help": "Return valid Unicode text from the macro; lone surrogates usually come from bytes decoded with errors='surrogateescape'" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json b/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json new file mode 100644 index 0000000000..17af644d67 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Environment variable 'SQB_FAILURE_UNDECODABLE_REGION' is not valid UTF-8 text", + "help": "Its value is b'\\xff'; set it to UTF-8 text, since SQLBuild cannot send undecodable bytes to a warehouse" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json b/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json index 76bd9e282b..4f1b942ed5 100644 --- a/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json +++ b/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json @@ -8,7 +8,8 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/staging/stg_orders.sql' could not be parsed: invalid syntax (, line 1)" + "message": "Macro arguments in '/models/staging/stg_orders.sql' could not be parsed: a value is missing here at line 1, column 10 of the '@cents' arguments", + "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" } ] } diff --git a/tests/goldens/compiler/failure/macro-non-literal-argument.json b/tests/goldens/compiler/failure/macro-non-literal-argument.json index c2bf929aa5..1be2bcd2d5 100644 --- a/tests/goldens/compiler/failure/macro-non-literal-argument.json +++ b/tests/goldens/compiler/failure/macro-non-literal-argument.json @@ -8,7 +8,8 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/staging/stg_orders.sql' must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references" + "message": "Macro arguments in '/models/staging/stg_orders.sql' must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references at line 1, column 1 of the '@cents' arguments", + "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" } ] } From e5d4eecfc2342877e9299e42a913640e834aebe4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:37:01 +0100 Subject: [PATCH 08/40] docs(macros): note the rejected macro argument literals and located errors --- website/src/content/docs/docs/concepts/macros.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 09e3391d77..2b4e2d5744 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -69,6 +69,9 @@ Macro calls accept Python literal values: - `__ref("model")`, `__source("source")`, and `__seed("seed")` references, alone or inside lists and dictionaries +Bytes (`b"text"`), complex numbers (`2j`), f-strings, and `...` are rejected. An argument error +names its line and column within the call's arguments. + Positional and keyword arguments are supported: ```sql From 131503b94a3654911f05f0641cf6bfc5404c9225 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:46:46 +0100 Subject: [PATCH 09/40] test(native): stage-switch sabotage covers only stages that still switch to Python --- .../compiler_differential/test_native_fallback_gate.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py index 45e8ceccec..27073a525e 100644 --- a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py +++ b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py @@ -19,14 +19,11 @@ _WAFFLE_SHOP_NATIVE_STAGES: tuple[str, ...] = ( "attachments", "contracts", - "declaration_files", - "declaration_scopes", "lineage_facts", "macro_call_store", "macro_calls", "model_analysis", "model_config", - "model_loop", "reference_extraction", "semantic_checks", ) @@ -84,14 +81,14 @@ ), NativeFallbackGateTestCase( description="shipped_stages_switched_to_python_fail", - perturbation=stage_disable_sabotage(("reference_extraction", "declaration_files")), + perturbation=stage_disable_sabotage(("reference_extraction", "model_config")), appended_entries="", expected_exit_code=1, expected_lines=( "Native fallback allow-list: native-preview reference_extraction " f"reference_extraction.native reference_scans (project): {_ANSWER_VANISHED}", - "Native fallback allow-list: native-preview declaration_files " - f"declaration_files.native parsed_files (project): {_ANSWER_VANISHED}", + "Native fallback allow-list: native-preview model_config " + f"model_config.native config_builds (project): {_ANSWER_VANISHED}", "Intended? Run `make compiler-baselines`", ), ), From a3e7ef30c6348b203649dcba21e486328f0413aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:59:57 +0100 Subject: [PATCH 10/40] refactor(native): satisfy the structure rules for the expansion ports --- .../src/text/_helpers/code_point_ranges.rs | 10 ++ crates/sqlbuild-core/src/text/_helpers/mod.rs | 1 + .../src/text/main/is_python_alnum.rs | 10 +- .../src/text/main/is_python_alpha.rs | 2 +- .../src/text/main/is_python_decimal.rs | 2 +- .../src/templates/_helpers/evaluation.rs | 5 +- .../src/templates/_helpers/missing_values.rs | 17 ++ .../src/templates/_helpers/mod.rs | 2 + .../src/templates/_helpers/substitution.rs | 28 ++++ .../templates/main/expand_template_string.rs | 31 +--- .../templates/main/expand_template_text.rs | 15 ++ .../src/templates/main/mod.rs | 1 + .../templates/main/template_error_message.rs | 16 -- .../tests/test_expand_template_string.rs | 6 +- .../_helpers/model_config/config_templates.rs | 8 +- .../_helpers/render/macro_arguments.rs | 57 +++++-- .../src/bindings/_helpers/render/mod.rs | 1 + .../bindings/_helpers/render/registration.rs | 11 ++ .../bindings/_helpers/sqltext/static_sql.rs | 34 +++- .../src/bindings/main/register.rs | 3 +- .../src/macro_arguments/_helpers/lexer.rs | 157 ++++++++---------- .../src/macro_arguments/_helpers/literals.rs | 81 +++++---- .../src/macro_arguments/constants.rs | 6 + .../src/macro_arguments/errors.rs | 12 ++ .../main/parse_macro_arguments.rs | 3 +- .../src/macro_arguments/mod.rs | 2 + .../src/macro_arguments/models.rs | 13 +- .../src/macro_arguments/tests/helpers.rs | 67 +------- .../tests/test_parse_macro_arguments.rs | 64 +++---- .../src/macro_arguments/tests/test_types.rs | 19 ++- .../declaration_references/scanner.rs | 25 +-- .../src/compiler/tests/helpers.rs | 57 +------ .../compiler/tests/test_sql_interpolation.rs | 74 +++++---- .../src/compiler/tests/test_types.rs | 58 ++++++- .../_helpers/corpus/failure_cases.py | 13 +- src/sqlbuild/_native.pyi | 2 +- .../_helpers/render/macro_argument_values.py | 82 +++++---- .../compile/_helpers/render/sql_vars.py | 30 ++-- src/sqlbuild/compiler/compile/constants.py | 1 + .../compiler/discovery/_helpers/helpers.py | 13 +- 40 files changed, 541 insertions(+), 498 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/missing_values.rs create mode 100644 crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs create mode 100644 crates/sqlbuild-model-config/src/templates/main/expand_template_text.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/constants.rs create mode 100644 crates/sqlbuild-render/src/macro_arguments/errors.rs diff --git a/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs new file mode 100644 index 0000000000..2a0a763184 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs @@ -0,0 +1,10 @@ +//! Membership in the generated, ordered code point range tables. + +/// Whether one character falls inside the inclusive, ordered code point `ranges`. +pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { + let code_point: u32 = u32::from(character); + let index: usize = ranges.partition_point(|(_, end)| *end < code_point); + ranges + .get(index) + .is_some_and(|(start, _)| *start <= code_point) +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 0a2f6ee237..ede18696f6 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod alnum_tables; pub(crate) mod cleandoc; +pub(crate) mod code_point_ranges; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs index de2b45f741..d7024a577d 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs @@ -1,5 +1,6 @@ //! Python's `str.isalnum()` for one character, from a table generated by Python. +use crate::text::_helpers::code_point_ranges::in_ranges; use crate::text::models::PythonText; /// Whether Python's `str.isalnum()` is true for one character. @@ -9,12 +10,3 @@ pub fn is_python_alnum(python: PythonText, character: char) -> bool { } in_ranges(python.alnum_ranges, character) } - -/// Whether one character falls inside the inclusive, ordered code point `ranges`. -pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { - let code_point: u32 = u32::from(character); - let index: usize = ranges.partition_point(|(_, end)| *end < code_point); - ranges - .get(index) - .is_some_and(|(start, _)| *start <= code_point) -} diff --git a/crates/sqlbuild-core/src/text/main/is_python_alpha.rs b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs index 36f3bcf40c..aca761d0ed 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_alpha.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs @@ -1,6 +1,6 @@ //! Python's `str.isalpha()` for one character, from a table generated by Python. -use crate::text::main::is_python_alnum::in_ranges; +use crate::text::_helpers::code_point_ranges::in_ranges; use crate::text::models::PythonText; /// Whether Python's `str.isalpha()` is true for one character. diff --git a/crates/sqlbuild-core/src/text/main/is_python_decimal.rs b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs index e24c260d67..c98d7658b3 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_decimal.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs @@ -1,6 +1,6 @@ //! Python's `str.isdecimal()`, the regex `\d` of `str` patterns, from a generated table. -use crate::text::main::is_python_alnum::in_ranges; +use crate::text::_helpers::code_point_ranges::in_ranges; use crate::text::models::PythonText; /// Whether Python's `str.isdecimal()` is true for one character. diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs index 305c7461fa..f29e86262e 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs @@ -1,9 +1,8 @@ //! The Python template evaluator: references, `if`, `eq`, `ne` and `coalesce`. +use crate::templates::_helpers::missing_values::is_missing_value_message; use crate::templates::errors::TemplateError; -use crate::templates::main::template_error_message::{ - is_missing_value_message, template_error_message, -}; +use crate::templates::main::template_error_message::template_error_message; use crate::templates::models::{ ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, }; diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/missing_values.rs b/crates/sqlbuild-model-config/src/templates/_helpers/missing_values.rs new file mode 100644 index 0000000000..0c56afe012 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/missing_values.rs @@ -0,0 +1,17 @@ +//! The template errors Python's `coalesce` treats as a missing value. + +const MISSING_VALUE_PARTS: [&str; 3] = [ + "references missing ENV variable", + "references unknown variable", + "references unknown CTX key", +]; +const UNAVAILABLE_CONTEXT_PART: &str = "references CTX key"; +const NO_VALUE_PART: &str = "no value is available"; + +/// Whether Python's `coalesce` treats this error message as a missing value and skips it. +pub(crate) fn is_missing_value_message(message: &str) -> bool { + MISSING_VALUE_PARTS + .iter() + .any(|part| message.contains(part)) + || (message.contains(UNAVAILABLE_CONTEXT_PART) && message.contains(NO_VALUE_PART)) +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs index 458598d7c2..71a3096584 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod evaluation; pub(crate) mod expressions; +pub(crate) mod missing_values; pub(crate) mod pattern; +pub(crate) mod substitution; pub(crate) mod tokens; diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs b/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs new file mode 100644 index 0000000000..c562ed4191 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs @@ -0,0 +1,28 @@ +//! Substitute each `${...}` template in a string by its rendered value. + +use crate::templates::_helpers::evaluation::evaluate; +use crate::templates::_helpers::expressions::parse_expression; +use crate::templates::models::{TemplateFailure, TemplateOptions}; +use crate::templates::types::TemplateHost; + +/// `text` with every template at `spans` replaced by its value rendered as text. +pub(crate) fn substituted( + host: &H, + text: &str, + spans: &[(usize, usize)], + options: TemplateOptions, +) -> Result { + let mut expanded = String::with_capacity(text.len()); + let mut copied = 0; + for &(start, end) in spans { + expanded.push_str(&text[copied..start]); + let body = &text[start + 2..end - 1]; + let expression = parse_expression(body)?; + let value = evaluate(host, &expression, options)?; + let label = format!("{} variable '{body}'", host.label()); + expanded.push_str(&host.render(&value, &label)?); + copied = end; + } + expanded.push_str(&text[copied..]); + Ok(expanded) +} diff --git a/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs index b88496de79..a9afdb307d 100644 --- a/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/main/expand_template_string.rs @@ -3,6 +3,7 @@ use crate::templates::_helpers::evaluation::evaluate; use crate::templates::_helpers::expressions::parse_expression; use crate::templates::_helpers::pattern::template_spans; +use crate::templates::_helpers::substitution::substituted; use crate::templates::models::{StringExpansion, TemplateFailure, TemplateOptions}; use crate::templates::types::TemplateHost; @@ -25,33 +26,3 @@ pub fn expand_template_string( } substituted(host, text, &spans, options).map(StringExpansion::Text) } - -/// Substitute every template in `text` as text, as `expand_template_string` does in Python. -pub fn expand_template_text( - host: &H, - text: &str, - options: TemplateOptions, -) -> Result { - substituted(host, text, &template_spans(text), options) -} - -fn substituted( - host: &H, - text: &str, - spans: &[(usize, usize)], - options: TemplateOptions, -) -> Result { - let mut expanded = String::with_capacity(text.len()); - let mut copied = 0; - for &(start, end) in spans { - expanded.push_str(&text[copied..start]); - let body = &text[start + 2..end - 1]; - let expression = parse_expression(body)?; - let value = evaluate(host, &expression, options)?; - let label = format!("{} variable '{body}'", host.label()); - expanded.push_str(&host.render(&value, &label)?); - copied = end; - } - expanded.push_str(&text[copied..]); - Ok(expanded) -} diff --git a/crates/sqlbuild-model-config/src/templates/main/expand_template_text.rs b/crates/sqlbuild-model-config/src/templates/main/expand_template_text.rs new file mode 100644 index 0000000000..cdbebbcc32 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/main/expand_template_text.rs @@ -0,0 +1,15 @@ +//! Expand every `${...}` template in one string as text. + +use crate::templates::_helpers::pattern::template_spans; +use crate::templates::_helpers::substitution::substituted; +use crate::templates::models::{TemplateFailure, TemplateOptions}; +use crate::templates::types::TemplateHost; + +/// Substitute every template in `text` as text, as `expand_template_string` does in Python. +pub fn expand_template_text( + host: &H, + text: &str, + options: TemplateOptions, +) -> Result { + substituted(host, text, &template_spans(text), options) +} diff --git a/crates/sqlbuild-model-config/src/templates/main/mod.rs b/crates/sqlbuild-model-config/src/templates/main/mod.rs index 7ea112bac5..2d0b299792 100644 --- a/crates/sqlbuild-model-config/src/templates/main/mod.rs +++ b/crates/sqlbuild-model-config/src/templates/main/mod.rs @@ -1,2 +1,3 @@ pub mod expand_template_string; +pub mod expand_template_text; pub mod template_error_message; diff --git a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs index 1c7578e44c..936f13a7b5 100644 --- a/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs +++ b/crates/sqlbuild-model-config/src/templates/main/template_error_message.rs @@ -44,19 +44,3 @@ pub fn template_error_message(error: &TemplateError, label: &str) -> String { TemplateError::Message(message) => message.clone(), } } - -/// Whether Python's `coalesce` treats this error message as a missing value and skips it. -pub fn is_missing_value_message(message: &str) -> bool { - MISSING_VALUE_PARTS - .iter() - .any(|part| message.contains(part)) - || (message.contains(UNAVAILABLE_CONTEXT_PART) && message.contains(NO_VALUE_PART)) -} - -const MISSING_VALUE_PARTS: [&str; 3] = [ - "references missing ENV variable", - "references unknown variable", - "references unknown CTX key", -]; -const UNAVAILABLE_CONTEXT_PART: &str = "references CTX key"; -const NO_VALUE_PART: &str = "no value is available"; diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs index ef8c06c86c..5a098a498e 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs @@ -165,9 +165,9 @@ fn given_template_strings_when_expanding_then_values_and_reads_match_python() { expected_reads: &[], }, TemplateTestCase { - description: "a mapping is truthy by its text and comparable", - text: "${if(eq(opaque, \"{'a': 1}\"), 'same', 'different')}", - expected: value(Value::Text("same".to_owned())), + description: "a mapping is truthy by its text", + text: "${if(opaque, 'truthy', 'falsy')}", + expected: value(Value::Text("truthy".to_owned())), expected_reads: &[], }, TemplateTestCase { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index cc48cba2bd..54a388c461 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -11,9 +11,8 @@ use pyo3::types::{ use pyo3::{FromPyObject, IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::templates::errors::TemplateError; -use sqlbuild_model_config::templates::main::expand_template_string::{ - expand_template_string, expand_template_text, -}; +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::main::expand_template_text::expand_template_text; use sqlbuild_model_config::templates::main::template_error_message::template_error_message; use sqlbuild_model_config::templates::models::{ ContextValue, Scalar, StringExpansion, TemplateFailure, TemplateOptions, @@ -331,8 +330,7 @@ fn outcome<'py>( .unbind()) } -/// Expand every template in `value` and the containers inside it, as Python's -/// `expand_template_data` does. +/// Expand every template in `value` and the containers inside it. pub(crate) fn expanded<'py>( host: &PythonHost<'py>, value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs index b16754c22b..aa76ad8fd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs @@ -1,41 +1,61 @@ //! Macro call arguments parsed natively into a value plan Python builds objects from. +use pyo3::PyErr; +use pyo3::exceptions::PyKeyError; use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyList, PyListMethods, PyTuple}; +use pyo3::types::{PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments as parse; use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; use sqlbuild_render::macro_arguments::types::ArgumentHost; +use std::cell::RefCell; /// `unicodedata` answers character names and identifier normalization as CPython does. struct UnicodeHost<'py> { unicodedata: Bound<'py, PyModule>, + /// An unexpected Python error, raised once parsing returns. + raised: RefCell>, +} + +impl UnicodeHost<'_> { + fn held(&self, result: PyResult) -> Option { + match result { + Ok(value) => Some(value), + Err(error) => { + self.raised.borrow_mut().get_or_insert(error); + None + } + } + } } impl ArgumentHost for UnicodeHost<'_> { fn character_named(&self, name: &str) -> Option { - self.unicodedata - .call_method1("lookup", (name,)) - .and_then(|found| found.extract::()) - .ok() + let py: Python<'_> = self.unicodedata.py(); + match self.unicodedata.call_method1("lookup", (name,)) { + Ok(found) => self.held(found.extract::()), + Err(error) if error.is_instance_of::(py) => None, + Err(error) => self.held(Err(error)), + } } fn identifier(&self, text: &str) -> Option { - let py: Python<'_> = self.unicodedata.py(); - let candidate = text.into_pyobject(py).ok()?; - let valid: bool = candidate - .call_method0("isidentifier") - .and_then(|valid| valid.extract()) - .ok()?; + let candidate = PyString::new(self.unicodedata.py(), text); + let valid: bool = self.held( + candidate + .call_method0("isidentifier") + .and_then(|valid| valid.extract()), + )?; if !valid { return None; } - self.unicodedata - .call_method1("normalize", ("NFKC", candidate)) - .and_then(|normalized| normalized.extract()) - .ok() + self.held( + self.unicodedata + .call_method1("normalize", ("NFKC", candidate)) + .and_then(|normalized| normalized.extract()), + ) } } @@ -48,8 +68,13 @@ fn parse_macro_arguments( ) -> PyResult> { let host = UnicodeHost { unicodedata: py.import("unicodedata")?, + raised: RefCell::new(None), }; - match parse(&host, text, &nested) { + let parsed = parse(&host, text, &nested); + if let Some(error) = host.raised.into_inner() { + return Err(error); + } + match parsed { Ok(arguments) => arguments_row(py, &arguments), Err(error) => Ok( ("error", error.detail, error.help, error.line, error.column) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 9fe00bacbe..ad5b571b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod macro_arguments; pub(crate) mod macro_call_memo; pub(crate) mod macro_call_rows; pub(crate) mod macro_calls; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs new file mode 100644 index 0000000000..d3e3f56667 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -0,0 +1,11 @@ +//! Register the macro call and macro argument functions and classes. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::render::{macro_arguments, macro_calls}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + macro_arguments::register(module)?; + macro_calls::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index acde1b8eab..99a0c902f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -24,8 +24,7 @@ type InterpolationRow = ( Option, ); -/// The project variables, `os.environ`, the context values (or `None`) and the Python that -/// renders a non-string variable as text, raising `ValueError` when it cannot be text. +/// Project variables, the environment, context values (or `None`) and the variable renderer. #[derive(FromPyObject)] struct InterpolationSources<'py>( Bound<'py, PyDict>, @@ -68,7 +67,14 @@ impl InterpolationHost for PythonInterpolationHost<'_> { } let label = format!("SQL variable '@@{name}'"); let py = value.py(); - Some(match self.sources.3.call1((value, label)) { + let arguments = PyDict::new(py); + if let Err(error) = arguments + .set_item("value", value) + .and_then(|()| arguments.set_item("label", label)) + { + return Some(Err(self.hold(error))); + } + Some(match self.sources.3.call((), Some(&arguments)) { Ok(rendered) => self.text(&rendered), Err(error) if error.is_instance_of::(py) => { Err(error.value(py).to_string()) @@ -83,7 +89,7 @@ impl InterpolationHost for PythonInterpolationHost<'_> { .0 .keys() .iter() - .map(|key| key.str().map(|text| text.to_string()).unwrap_or_default()) + .map(|key| key_text(&key)) .collect(); names.sort(); names @@ -108,11 +114,20 @@ impl InterpolationHost for PythonInterpolationHost<'_> { } fn context(&self, name: &str) -> Option> { - let value = self.sources.2.as_ref()?.get_item(name).ok()??; + let value = match self.sources.2.as_ref()?.get_item(name) { + Ok(value) => value?, + Err(error) => { + let _ = self.hold(error); + return None; + } + }; if value.is_none() { return Some(None); } - Some(self.text(&value).ok()) + match self.text(&value) { + Ok(text) => Some(Some(text)), + Err(_held) => Some(None), + } } fn context_names(&self) -> Vec { @@ -120,11 +135,16 @@ impl InterpolationHost for PythonInterpolationHost<'_> { .2 .iter() .flat_map(|context| context.keys()) - .map(|key| key.str().map(|text| text.to_string()).unwrap_or_default()) + .map(|key| key_text(&key)) .collect() } } +/// A variable or context key's text, or empty text when Python cannot render it. +fn key_text(key: &Bound<'_, PyAny>) -> String { + key.str().map(|text| text.to_string()).unwrap_or_default() +} + /// Interpolate each `(sql, file path)`; an error is returned in its row, not raised. #[pyfunction] fn interpolate_sql_batch<'py>( diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index f4cd61114b..0e0d1ed5c9 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -31,8 +31,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; - render::macro_arguments::register(module)?; - render::macro_calls::register(module)?; + render::registration::register(module)?; cache::registration::register(module)?; attachments::registration::register(module)?; register_analysis_stages(module)?; diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs index 7842d856da..44d7d2190f 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/lexer.rs @@ -52,38 +52,14 @@ pub(crate) fn tokenize( let mut tokens: Vec = Vec::new(); let mut index: usize = 0; while index < text.len() { - if let Some(call) = nested.iter().position(|(start, _)| *start == index) { + let (kind, end): (Option, usize) = next_token(host, text, nested, index)?; + if let Some(kind) = kind { tokens.push(Token { - kind: TokenKind::Nested(call), + kind, position: index, }); - index = nested[call].1; - continue; - } - let character: char = text[index]; - match character { - ' ' | '\t' | '\x0c' | '\n' | '\r' => index += 1, - '#' => { - while index < text.len() && !matches!(text[index], '\n' | '\r') { - index += 1; - } - } - '\\' => index = line_continuation(text, index)?, - '\'' | '"' => { - let (piece, end) = string_body(host, text, index, "")?; - tokens.push(Token { - kind: TokenKind::Str(piece), - position: index, - }); - index = end; - } - '0'..='9' => index = push_number(text, index, &mut tokens)?, - '.' if text.get(index + 1).is_some_and(char::is_ascii_digit) => { - index = push_number(text, index, &mut tokens)?; - } - _ if is_name_start(character) => index = push_name(host, text, index, &mut tokens)?, - _ => index = push_operator(text, index, &mut tokens)?, } + index = end; } tokens.push(Token { kind: TokenKind::End, @@ -92,6 +68,39 @@ pub(crate) fn tokenize( Ok(tokens) } +/// The token starting at `index`, or `None` for whitespace and comments, and where it ends. +fn next_token( + host: &dyn ArgumentHost, + text: &[char], + nested: &[(usize, usize)], + index: usize, +) -> Result<(Option, usize), Failure> { + if let Some(call) = nested.iter().position(|(start, _)| *start == index) { + return Ok((Some(TokenKind::Nested(call)), nested[call].1)); + } + let character: char = text[index]; + let (kind, end): (TokenKind, usize) = match character { + ' ' | '\t' | '\x0c' | '\n' | '\r' => return Ok((None, index + 1)), + '#' => { + let end: usize = text[index..] + .iter() + .position(|character| matches!(character, '\n' | '\r')) + .map_or(text.len(), |offset| index + offset); + return Ok((None, end)); + } + '\\' => return Ok((None, line_continuation(text, index)?)), + '\'' | '"' => { + let (piece, end) = string_body(host, text, index, "")?; + (TokenKind::Str(piece), end) + } + '0'..='9' => number_token(text, index)?, + '.' if text.get(index + 1).is_some_and(char::is_ascii_digit) => number_token(text, index)?, + _ if is_name_start(character) => name_token(host, text, index)?, + _ => operator_token(text, index)?, + }; + Ok((Some(kind), end)) +} + fn line_continuation(text: &[char], index: usize) -> Result { match (text.get(index + 1), text.get(index + 2)) { (Some('\n'), _) => Ok(index + 2), @@ -101,83 +110,59 @@ fn line_continuation(text: &[char], index: usize) -> Result { } } -fn push_number(text: &[char], index: usize, tokens: &mut Vec) -> Result { +fn number_token(text: &[char], index: usize) -> Result<(TokenKind, usize), Failure> { let (token, end) = number(text, index)?; - tokens.push(Token { - kind: TokenKind::Number(token), - position: index, - }); - Ok(end) + Ok((TokenKind::Number(token), end)) } -fn push_name( +fn name_token( host: &dyn ArgumentHost, text: &[char], index: usize, - tokens: &mut Vec, -) -> Result { - let mut end: usize = index; - while end < text.len() && is_name_continue(text[end]) { - end += 1; - } +) -> Result<(TokenKind, usize), Failure> { + let end: usize = text[index..] + .iter() + .position(|character| !is_name_continue(*character)) + .map_or(text.len(), |offset| index + offset); let word: String = text[index..end].iter().collect(); let lowered: String = word.to_ascii_lowercase(); if matches!(text.get(end), Some('\'' | '"')) && STRING_PREFIXES.contains(&lowered.as_str()) { let (piece, string_end) = string_body(host, text, end, &lowered)?; - tokens.push(Token { - kind: TokenKind::Str(piece), - position: index, - }); - return Ok(string_end); + return Ok((TokenKind::Str(piece), string_end)); } - let name: String = if word.is_ascii() { - word - } else { - host.identifier(&word).ok_or_else(|| { - let invalid: char = text[index..end] - .iter() - .copied() - .find(|character| { - !character.is_ascii() && host.identifier(&format!("a{character}")).is_none() - }) - .unwrap_or(text[index]); - syntax( - &format!( - "the character '{invalid}' (U+{:04X}) is not valid in a name", - u32::from(invalid) - ), - index, - ) - })? - }; - tokens.push(Token { - kind: TokenKind::Name(name), - position: index, - }); - Ok(end) + if word.is_ascii() { + return Ok((TokenKind::Name(word), end)); + } + let name: String = host.identifier(&word).ok_or_else(|| { + let invalid: char = text[index..end] + .iter() + .copied() + .find(|character| { + !character.is_ascii() && host.identifier(&format!("a{character}")).is_none() + }) + .unwrap_or(text[index]); + syntax( + &format!( + "the character '{invalid}' (U+{:04X}) is not valid in a name", + u32::from(invalid) + ), + index, + ) + })?; + Ok((TokenKind::Name(name), end)) } -fn push_operator(text: &[char], index: usize, tokens: &mut Vec) -> Result { +fn operator_token(text: &[char], index: usize) -> Result<(TokenKind, usize), Failure> { if text[index] == '\0' { return Err(syntax("a NUL character is not allowed", index)); } let rest: String = text[index..text.len().min(index + 3)].iter().collect(); - let kind: TokenKind = OPERATORS + Ok(OPERATORS .iter() .find(|operator| rest.starts_with(**operator)) - .map_or_else( - || TokenKind::Other, - |operator| TokenKind::Operator(operator), - ); - let width: usize = match &kind { - TokenKind::Operator(operator) => operator.len(), - _ => 1, - }; - tokens.push(Token { - kind, - position: index, - }); - Ok(index + width) + .map_or((TokenKind::Other, index + 1), |operator| { + (TokenKind::Operator(operator), index + operator.len()) + })) } /// Python's tokenizer reads ASCII letters, `_` and any non-ASCII character into a name. diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs index 2a5e8c421d..e4887c4078 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs @@ -2,6 +2,7 @@ use crate::macro_arguments::_helpers::failures::{Failure, surrogate, syntax}; use crate::macro_arguments::_helpers::lexer::{NumberToken, StringPiece, is_name_start}; +use crate::macro_arguments::constants::{BINARY_RADIX, DECIMAL_RADIX, HEX_RADIX, OCTAL_RADIX}; use crate::macro_arguments::types::ArgumentHost; /// The string starting at the quote `start` under `prefix`, and the index after it. @@ -42,7 +43,9 @@ pub(crate) fn string_body( index += 2; continue; } - index = escape(host, text, index, &mut value)?; + let (decoded, end) = escape(host, text, index)?; + value.push_str(&decoded); + index = end; } Ok(( StringPiece { @@ -58,13 +61,12 @@ fn closes_triple(text: &[char], index: usize, quote: char) -> bool { text.get(index + 1) == Some("e) && text.get(index + 2) == Some("e) } -/// Decode the escape at the backslash `index` into `value`; return the index after it. +/// The text the escape at the backslash `index` decodes to, and the index after it. fn escape( host: &dyn ArgumentHost, text: &[char], index: usize, - value: &mut String, -) -> Result { +) -> Result<(String, usize), Failure> { let escaped: char = text[index + 1]; let simple: Option = match escaped { '\\' => Some('\\'), @@ -80,43 +82,37 @@ fn escape( _ => None, }; if let Some(character) = simple { - value.push(character); - return Ok(index + 2); + return Ok((character.to_string(), index + 2)); } + let line_end: usize = if text.get(index + 2) == Some(&'\n') { + index + 3 + } else { + index + 2 + }; match escaped { - '\n' => Ok(index + 2), - '\r' => Ok(if text.get(index + 2) == Some(&'\n') { - index + 3 - } else { - index + 2 - }), + '\n' => Ok((String::new(), index + 2)), + '\r' => Ok((String::new(), line_end)), '0'..='7' => { let digits: usize = text[index + 1..] .iter() .take(3) .take_while(|character| matches!(character, '0'..='7')) .count(); - let code: u32 = radix_value(&text[index + 1..index + 1 + digits], 8); - value.push(char::from_u32(code).unwrap_or_default()); - Ok(index + 1 + digits) - } - 'x' => hex_escape(text, index, 2, value), - 'u' => hex_escape(text, index, 4, value), - 'U' => hex_escape(text, index, 8, value), - 'N' => named_escape(host, text, index, value), - _ => { - value.push('\\'); - Ok(index + 1) + let code: u32 = radix_value(&text[index + 1..index + 1 + digits], OCTAL_RADIX); + Ok(( + char::from_u32(code).unwrap_or_default().to_string(), + index + 1 + digits, + )) } + 'x' => hex_escape(text, index, 2), + 'u' => hex_escape(text, index, 4), + 'U' => hex_escape(text, index, 8), + 'N' => named_escape(host, text, index), + _ => Ok(("\\".to_owned(), index + 1)), } } -fn hex_escape( - text: &[char], - index: usize, - width: usize, - value: &mut String, -) -> Result { +fn hex_escape(text: &[char], index: usize, width: usize) -> Result<(String, usize), Failure> { let digits: &[char] = &text[index + 2..text.len().min(index + 2 + width)]; if digits.len() < width || !digits.iter().all(char::is_ascii_hexdigit) { let escape: String = text[index..index + 2].iter().collect(); @@ -125,7 +121,7 @@ fn hex_escape( index, )); } - let code: u32 = radix_value(digits, 16); + let code: u32 = radix_value(digits, HEX_RADIX); let end: usize = index + 2 + width; if (0xD800..=0xDFFF).contains(&code) { let escape: String = text[index..end].iter().collect(); @@ -135,16 +131,14 @@ fn hex_escape( let escape: String = text[index..end].iter().collect(); syntax(&format!("{escape} is not a Unicode code point"), index) })?; - value.push(character); - Ok(end) + Ok((character.to_string(), end)) } fn named_escape( host: &dyn ArgumentHost, text: &[char], index: usize, - value: &mut String, -) -> Result { +) -> Result<(String, usize), Failure> { let close: Option = (text.get(index + 2) == Some(&'{')) .then(|| { text[index + 3..] @@ -159,8 +153,7 @@ fn named_escape( let character: char = host .character_named(&name) .ok_or_else(|| syntax(&format!("\\N{{{name}}} names no Unicode character"), index))?; - value.push(character); - Ok(index + 4 + close) + Ok((character.to_string(), index + 4 + close)) } fn radix_value(digits: &[char], radix: u32) -> u32 { @@ -173,9 +166,9 @@ fn radix_value(digits: &[char], radix: u32) -> u32 { pub(crate) fn number(text: &[char], start: usize) -> Result<(NumberToken, usize), Failure> { let prefixed: Option = (text[start] == '0') .then(|| match text.get(start + 1) { - Some('x' | 'X') => Some(16), - Some('o' | 'O') => Some(8), - Some('b' | 'B') => Some(2), + Some('x' | 'X') => Some(HEX_RADIX), + Some('o' | 'O') => Some(OCTAL_RADIX), + Some('b' | 'B') => Some(BINARY_RADIX), _ => None, }) .flatten(); @@ -194,13 +187,13 @@ pub(crate) fn number(text: &[char], start: usize) -> Result<(NumberToken, usize) let end: usize = reject_trailing_name(text, end, start)?; return Ok((NumberToken::Int { radix, digits }, end)); } - let (whole, mut index) = digit_part(text, start, 10, start)?; + let (whole, mut index) = digit_part(text, start, DECIMAL_RADIX, start)?; let mut float: bool = false; let mut literal: String = whole.clone(); if text.get(index) == Some(&'.') { float = true; literal.push('.'); - let (fraction, end) = digit_part(text, index + 1, 10, start)?; + let (fraction, end) = digit_part(text, index + 1, DECIMAL_RADIX, start)?; literal.push_str(&fraction); index = end; } @@ -211,7 +204,7 @@ pub(crate) fn number(text: &[char], start: usize) -> Result<(NumberToken, usize) exponent.push(*sign); exponent_start += 1; } - let (digits, end) = digit_part(text, exponent_start, 10, start)?; + let (digits, end) = digit_part(text, exponent_start, DECIMAL_RADIX, start)?; if digits.is_empty() { return Err(syntax("a number's exponent has no digits", start)); } @@ -236,7 +229,7 @@ pub(crate) fn number(text: &[char], start: usize) -> Result<(NumberToken, usize) } Ok(( NumberToken::Int { - radix: 10, + radix: DECIMAL_RADIX, digits: whole, }, end, @@ -261,7 +254,7 @@ fn digit_part( && text.get(index + 1).is_some_and(|next| next.is_digit(radix)) { index += 1; - } else if character == '_' || (character.is_ascii_digit() && radix < 10) { + } else if character == '_' || (character.is_ascii_digit() && radix < DECIMAL_RADIX) { return Err(syntax( "a number has a misplaced '_' or digit", number_start, diff --git a/crates/sqlbuild-render/src/macro_arguments/constants.rs b/crates/sqlbuild-render/src/macro_arguments/constants.rs new file mode 100644 index 0000000000..ef0bba4283 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/constants.rs @@ -0,0 +1,6 @@ +//! Radixes of Python's number literals and escapes. + +pub(crate) const BINARY_RADIX: u32 = 2; +pub(crate) const OCTAL_RADIX: u32 = 8; +pub(crate) const DECIMAL_RADIX: u32 = 10; +pub(crate) const HEX_RADIX: u32 = 16; diff --git a/crates/sqlbuild-render/src/macro_arguments/errors.rs b/crates/sqlbuild-render/src/macro_arguments/errors.rs new file mode 100644 index 0000000000..7f28be5dfd --- /dev/null +++ b/crates/sqlbuild-render/src/macro_arguments/errors.rs @@ -0,0 +1,12 @@ +//! Why one macro call's arguments cannot be used. + +/// Why the arguments cannot be used, where, and how to fix them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArgumentError { + /// What is wrong, completing "Macro arguments in '' ...". + pub detail: String, + pub help: String, + /// 1-based line and column, in characters, within the argument text. + pub line: usize, + pub column: usize, +} diff --git a/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs index 8ad1f29c7a..4e9f64cd85 100644 --- a/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs +++ b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs @@ -4,7 +4,8 @@ use crate::macro_arguments::_helpers::evaluation::evaluate_arguments; use crate::macro_arguments::_helpers::failures::Failure; use crate::macro_arguments::_helpers::lexer::tokenize; use crate::macro_arguments::_helpers::syntax_tree::parse_call_arguments; -use crate::macro_arguments::models::{ArgumentError, MacroArguments}; +use crate::macro_arguments::errors::ArgumentError; +use crate::macro_arguments::models::MacroArguments; use crate::macro_arguments::types::ArgumentHost; /// The value plan of `text`, whose nested macro calls fill the `[start, end)` code point spans. diff --git a/crates/sqlbuild-render/src/macro_arguments/mod.rs b/crates/sqlbuild-render/src/macro_arguments/mod.rs index 91fb358a0f..38a9736573 100644 --- a/crates/sqlbuild-render/src/macro_arguments/mod.rs +++ b/crates/sqlbuild-render/src/macro_arguments/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; pub mod main; pub mod models; pub mod types; diff --git a/crates/sqlbuild-render/src/macro_arguments/models.rs b/crates/sqlbuild-render/src/macro_arguments/models.rs index 3a86ab5715..222f37f1a3 100644 --- a/crates/sqlbuild-render/src/macro_arguments/models.rs +++ b/crates/sqlbuild-render/src/macro_arguments/models.rs @@ -1,4 +1,4 @@ -//! The value plan of one macro call's arguments, and why parsing them stopped. +//! The value plan of one macro call's arguments. /// One argument value; Python builds the object, calling nested macros by their index. #[derive(Clone, Debug, PartialEq, Eq)] @@ -38,14 +38,3 @@ pub struct MacroArguments { /// `(function, name)` of every typed reference, in Python's `ast.walk` order. pub typed_references: Vec<(String, String)>, } - -/// Why the arguments cannot be used, where, and how to fix them. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ArgumentError { - /// What is wrong, completing "Macro arguments in '' ...". - pub detail: String, - pub help: String, - /// 1-based line and column, in characters, within the argument text. - pub line: usize, - pub column: usize, -} diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs b/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs index 93887a7f10..cb95753595 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/helpers.rs @@ -1,68 +1,7 @@ use crate::macro_arguments::main::parse_macro_arguments::parse_macro_arguments; -use crate::macro_arguments::models::ArgumentValue; -use crate::macro_arguments::types::ArgumentHost; +use crate::macro_arguments::tests::test_types::TestHost; -/// A host knowing one character name and NFKC-folding the `fi` ligature, rejecting `€`. -struct TestHost; - -impl ArgumentHost for TestHost { - fn character_named(&self, name: &str) -> Option { - (name == "LATIN SMALL LETTER E WITH ACUTE").then_some('é') - } - - fn identifier(&self, text: &str) -> Option { - (!text.contains('€')).then(|| text.replace('fi', "fi")) - } -} - -/// Spell the plan as `positional | keywords | references`, or the error and its location. +/// The parse result of `text` spelled with `{:?}`. pub(crate) fn spelled_arguments(text: &str, nested: &[(usize, usize)]) -> String { - match parse_macro_arguments(&TestHost, text, nested) { - Ok(arguments) => format!( - "{} | {} | {}", - spelled_list(&arguments.positional), - arguments - .keywords - .iter() - .map(|(name, value)| format!("{name}={}", spelled(value))) - .collect::>() - .join(", "), - arguments - .typed_references - .iter() - .map(|(function, name)| format!("{function}:{name}")) - .collect::>() - .join(", "), - ), - Err(error) => format!("error: {} @{}:{}", error.detail, error.line, error.column), - } -} - -fn spelled_list(values: &[ArgumentValue]) -> String { - values.iter().map(spelled).collect::>().join(", ") -} - -fn spelled(value: &ArgumentValue) -> String { - match value { - ArgumentValue::Str(text) => format!("{text:?}"), - ArgumentValue::Int { radix: 10, digits } => digits.clone(), - ArgumentValue::Int { radix, digits } => format!("{digits}/{radix}"), - ArgumentValue::Float(text) => format!("f{text}"), - ArgumentValue::Bool(flag) => if *flag { "True" } else { "False" }.to_owned(), - ArgumentValue::None => "None".to_owned(), - ArgumentValue::NestedCall(call) => format!("@{call}"), - ArgumentValue::TypedReference { function, name } => format!("{function}({name})"), - ArgumentValue::List(items) => format!("[{}]", spelled_list(items)), - ArgumentValue::Tuple(items) => format!("({})", spelled_list(items)), - ArgumentValue::Dict(pairs) => format!( - "{{{}}}", - pairs - .iter() - .map(|(key, value)| format!("{}: {}", spelled(key), spelled(value))) - .collect::>() - .join(", ") - ), - ArgumentValue::Negative(inner) => format!("-{}", spelled(inner)), - ArgumentValue::Positive(inner) => format!("+{}", spelled(inner)), - } + format!("{:?}", parse_macro_arguments(&TestHost, text, nested)) } diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs index f4e36f5d9a..caabd374d6 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs @@ -8,193 +8,193 @@ fn given_argument_text_when_parsing_then_values_match_python_literals() { description: "scalars, containers and keywords", text: "'a', 1, 2.5, True, None, [1, (2,)], {'k': -3}, (), x=+1.0, y=[]", nested: &[], - expected: "\"a\", 1, f2.5, True, None, [1, (2)], {\"k\": -3}, () | x=+f1.0, y=[] | ", + expected_plan: "Ok(MacroArguments { positional: [Str(\"a\"), Int { radix: 10, digits: \"1\" }, Float(\"2.5\"), Bool(true), None, List([Int { radix: 10, digits: \"1\" }, Tuple([Int { radix: 10, digits: \"2\" }])]), Dict([(Str(\"k\"), Negative(Int { radix: 10, digits: \"3\" }))]), Tuple([])], keywords: [(\"x\", Positive(Float(\"1.0\"))), (\"y\", List([]))], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "adjacent, raw, escaped and triple-quoted strings", text: "'it''s', \"a\" 'b', r'\\d', '\\x41\\u00e9\\N{LATIN SMALL LETTER E WITH ACUTE}\\101\\q', '''x\\ny'''", nested: &[], - expected: "\"its\", \"ab\", \"\\\\d\", \"AééA\\\\q\", \"x\\ny\" | | ", + expected_plan: "Ok(MacroArguments { positional: [Str(\"its\"), Str(\"ab\"), Str(\"\\\\d\"), Str(\"A\u{e9}\u{e9}A\\\\q\"), Str(\"x\\ny\")], keywords: [], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "integer radixes, underscores and float forms", text: "0x_ff, 0o17, 0b1_0, 1_000, 00, .5, 5., 1e-3, 1_0.0_1E+2", nested: &[], - expected: "ff/16, 17/8, 10/2, 1000, 00, f.5, f5., f1e-3, f10.01e+2 | | ", + expected_plan: "Ok(MacroArguments { positional: [Int { radix: 16, digits: \"ff\" }, Int { radix: 8, digits: \"17\" }, Int { radix: 2, digits: \"10\" }, Int { radix: 10, digits: \"1000\" }, Int { radix: 10, digits: \"00\" }, Float(\".5\"), Float(\"5.\"), Float(\"1e-3\"), Float(\"10.01e+2\")], keywords: [], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "typed references in ast.walk order", text: "[__ref('a')], __source(\"b\",), k=__seed('c')", nested: &[], - expected: "[__ref(a)], __source(b) | k=__seed(c) | __source:b, __ref:a, __seed:c", + expected_plan: "Ok(MacroArguments { positional: [List([TypedReference { function: \"__ref\", name: \"a\" }]), TypedReference { function: \"__source\", name: \"b\" }], keywords: [(\"k\", TypedReference { function: \"__seed\", name: \"c\" })], typed_references: [(\"__source\", \"b\"), (\"__ref\", \"a\"), (\"__seed\", \"c\")] })", }, ParseMacroArgumentsTestCase { description: "nested calls fill their spans, even under unary minus", text: "@m(1), -@n()", nested: &[(0, 5), (8, 12)], - expected: "@0, -@1 | | ", + expected_plan: "Ok(MacroArguments { positional: [NestedCall(0), Negative(NestedCall(1))], keywords: [], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "a keyword name is NFKC-normalized", text: "field=1", nested: &[], - expected: " | field=1 | ", + expected_plan: "Ok(MacroArguments { positional: [], keywords: [(\"field\", Int { radix: 10, digits: \"1\" })], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "comments and line continuations", text: "1, # note\n 2, \\\n 3", nested: &[], - expected: "1, 2, 3 | | ", + expected_plan: "Ok(MacroArguments { positional: [Int { radix: 10, digits: \"1\" }, Int { radix: 10, digits: \"2\" }, Int { radix: 10, digits: \"3\" }], keywords: [], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "unary signs over booleans and other signs", text: "-True, +-1", nested: &[], - expected: "-True, +-1 | | ", + expected_plan: "Ok(MacroArguments { positional: [Negative(Bool(true)), Positive(Negative(Int { radix: 10, digits: \"1\" }))], keywords: [], typed_references: [] })", }, ParseMacroArgumentsTestCase { description: "a bytes literal is rejected", text: "b'x'", nested: &[], - expected: "error: use a bytes literal @1:1", + expected_plan: "Err(ArgumentError { detail: \"use a bytes literal\", help: \"Pass text as a string without the b prefix, for example \'orders\' instead of b\'orders\'\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "a complex literal is rejected", text: "1, 2j", nested: &[], - expected: "error: use a complex number literal @1:4", + expected_plan: "Err(ArgumentError { detail: \"use a complex number literal\", help: \"Pass int or float numbers; give a complex value\'s real and imaginary parts as two arguments\", line: 1, column: 4 })", }, ParseMacroArgumentsTestCase { description: "an Ellipsis is rejected", text: "...", nested: &[], - expected: "error: use '...' (Ellipsis) @1:1", + expected_plan: "Err(ArgumentError { detail: \"use \'...\' (Ellipsis)\", help: \"Pass None, or a string the macro understands, instead of \'...\'\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "an f-string is rejected", text: "f'{x}'", nested: &[], - expected: "error: use an f-string @1:1", + expected_plan: "Err(ArgumentError { detail: \"use an f-string\", help: \"Pass the parts as separate arguments and format them in the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "bytes joined to a string are rejected", text: "'a' b'c'", nested: &[], - expected: "error: use a bytes literal @1:1", + expected_plan: "Err(ArgumentError { detail: \"use a bytes literal\", help: \"Pass text as a string without the b prefix, for example \'orders\' instead of b\'orders\'\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "an empty typed reference name", text: "__ref('')", nested: &[], - expected: "error: must give __ref() exactly one quoted resource name @1:1", + expected_plan: "Err(ArgumentError { detail: \"must give __ref() exactly one quoted resource name\", help: \"Write one quoted resource name, for example __ref(\\\"orders\\\")\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "a bare name", text: "x", nested: &[], - expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:1", + expected_plan: "Err(ArgumentError { detail: \"must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "a binary operation", text: "1 + 2", nested: &[], - expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:3", + expected_plan: "Err(ArgumentError { detail: \"must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 3 })", }, ParseMacroArgumentsTestCase { description: "a set", text: "{1, 2}", nested: &[], - expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @1:1", + expected_plan: "Err(ArgumentError { detail: \"must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "keyword expansion", text: "**k", nested: &[], - expected: "error: must not use **kwargs expansion syntax @1:3", + expected_plan: "Err(ArgumentError { detail: \"must not use **kwargs expansion syntax\", help: \"Write each argument or key out explicitly\", line: 1, column: 3 })", }, ParseMacroArgumentsTestCase { description: "dict unpacking", text: "{**d}", nested: &[], - expected: "error: must not use dict unpacking @1:4", + expected_plan: "Err(ArgumentError { detail: \"must not use dict unpacking\", help: \"Write each argument or key out explicitly\", line: 1, column: 4 })", }, ParseMacroArgumentsTestCase { description: "unary minus on text", text: "-'a'", nested: &[], - expected: "error: use unary + or - on a value that is not a number @1:1", + expected_plan: "Err(ArgumentError { detail: \"use unary + or - on a value that is not a number\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "a positional argument after a keyword", text: "x=1, 2", nested: &[], - expected: "error: could not be parsed: a positional argument follows a keyword argument @1:6", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a positional argument follows a keyword argument\", help: \"Pass each keyword argument once, after every positional argument\", line: 1, column: 6 })", }, ParseMacroArgumentsTestCase { description: "a repeated keyword", text: "x=1, x=2", nested: &[], - expected: "error: could not be parsed: keyword argument 'x' is repeated @1:6", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: keyword argument \'x\' is repeated\", help: \"Pass each keyword argument once, after every positional argument\", line: 1, column: 6 })", }, ParseMacroArgumentsTestCase { description: "a doubled comma", text: "'amount',,", nested: &[], - expected: "error: could not be parsed: a value is missing here @1:10", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a value is missing here\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 10 })", }, ParseMacroArgumentsTestCase { description: "an unclosed string", text: "'abc", nested: &[], - expected: "error: could not be parsed: a string is not closed @1:1", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a string is not closed\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "an unclosed bracket on a later line", text: "[1,\n 2", nested: &[], - expected: "error: could not be parsed: a bracket is not closed with ']' @1:1", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a bracket is not closed with \']\'\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "an error on a later line", text: "1,\n x", nested: &[], - expected: "error: must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references @2:3", + expected_plan: "Err(ArgumentError { detail: \"must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 2, column: 3 })", }, ParseMacroArgumentsTestCase { description: "a lone surrogate escape", text: "'\\ud800'", nested: &[], - expected: "error: contain the lone surrogate escape '\\ud800' @1:2", + expected_plan: "Err(ArgumentError { detail: \"contain the lone surrogate escape \'\\\\ud800\'\", help: \"Lone surrogates are not valid Unicode text and cannot be sent to a warehouse; write the character itself or the escape of a valid code point\", line: 1, column: 2 })", }, ParseMacroArgumentsTestCase { description: "leading zeros in a decimal integer", text: "007", nested: &[], - expected: "error: could not be parsed: leading zeros in a decimal integer are not allowed; use 0o for octal @1:1", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: leading zeros in a decimal integer are not allowed; use 0o for octal\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "a character Python rejects in a name", text: "1, é€", nested: &[], - expected: "error: could not be parsed: the character '€' (U+20AC) is not valid in a name @1:4", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: the character \'\u{20ac}\' (U+20AC) is not valid in a name\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 4 })", }, ParseMacroArgumentsTestCase { description: "a keyword used as an argument name", text: "True=1", nested: &[], - expected: "error: could not be parsed: 'True' is a Python keyword, not an argument name @1:1", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: \'True\' is a Python keyword, not an argument name\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 1 })", }, ParseMacroArgumentsTestCase { description: "an unknown character name", text: "'\\N{NOPE}'", nested: &[], - expected: "error: could not be parsed: \\N{NOPE} names no Unicode character @1:2", + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: \\\\N{NOPE} names no Unicode character\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 1, column: 2 })", }, ]; for test_case in test_cases { assert_eq!( spelled_arguments(test_case.text, test_case.nested), - test_case.expected, + test_case.expected_plan, "{}", test_case.description ); diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs index c5235931e5..4d4a848364 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs @@ -1,7 +1,22 @@ +use crate::macro_arguments::types::ArgumentHost; + pub(crate) struct ParseMacroArgumentsTestCase { pub(crate) description: &'static str, pub(crate) text: &'static str, pub(crate) nested: &'static [(usize, usize)], - /// The spelled plan, or `error: @line:column`. - pub(crate) expected: &'static str, + /// The parse result spelled with `{:?}`. + pub(crate) expected_plan: &'static str, +} + +/// A host knowing one character name and NFKC-folding the `fi` ligature, rejecting `€`. +pub(crate) struct TestHost; + +impl ArgumentHost for TestHost { + fn character_named(&self, name: &str) -> Option { + (name == "LATIN SMALL LETTER E WITH ACUTE").then_some('é') + } + + fn identifier(&self, text: &str) -> Option { + (!text.contains('€')).then(|| text.replace('fi', "fi")) + } } diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs index 4648692a08..60c0747554 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs @@ -1,7 +1,7 @@ //! Python's `_find_next_reference_start` walk over quoted text and comments. use crate::compiler::_helpers::declaration_references::reference_syntax::{ - ReferenceSyntax, match_reference, + MatchedReference, ReferenceSyntax, match_reference, }; use crate::compiler::models::{ DeclarationReference, DeclarationReferenceScan, DeclarationReferenceStop, @@ -11,8 +11,8 @@ use sqlbuild_core::text::models::PythonText; const SPECIAL_BYTES: &[u8] = b"'\"`$@/-"; /// Where the walk for the next reference start ended. -enum NextStart { - Reference(usize), +enum NextStart<'sql> { + Reference(usize, MatchedReference<'sql>), End, Stop(DeclarationReferenceStop), } @@ -24,22 +24,14 @@ pub(crate) fn scan_references(python: PythonText, sql: &str) -> DeclarationRefer let mut cursor: usize = 0; let mut stop: Option = None; while cursor < sql.len() { - let start: usize = match next_reference_start(python, sql, cursor) { - NextStart::Reference(start) => start, + let (start, matched) = match next_reference_start(python, sql, cursor) { + NextStart::Reference(start, matched) => (start, matched), NextStart::End => break, NextStart::Stop(found) => { stop = Some(found); break; } }; - let matched = match match_reference(python, sql, start) { - ReferenceSyntax::Matched(matched) => matched, - ReferenceSyntax::Malformed(kind) => { - stop = Some(DeclarationReferenceStop::Malformed(kind)); - break; - } - ReferenceSyntax::NotReference => unreachable!("the walk stops only at a reference"), - }; let start_char: usize = offsets.advance(sql, start); let end_char: usize = offsets.advance(sql, matched.end); references.push(DeclarationReference { @@ -55,7 +47,7 @@ pub(crate) fn scan_references(python: PythonText, sql: &str) -> DeclarationRefer } /// The next reference start at or after `start`, as Python's walk finds it. -fn next_reference_start(python: PythonText, sql: &str, start: usize) -> NextStart { +fn next_reference_start(python: PythonText, sql: &str, start: usize) -> NextStart<'_> { let rest: &str = &sql[start..]; if !rest.contains("@enum") && !rest.contains("@const") { return NextStart::End; @@ -83,8 +75,9 @@ fn next_reference_start(python: PythonText, sql: &str, start: usize) -> NextStar }, b'@' => match match_reference(python, sql, index) { ReferenceSyntax::NotReference => Some(index + 1), - ReferenceSyntax::Matched(_) | ReferenceSyntax::Malformed(_) => { - return NextStart::Reference(index); + ReferenceSyntax::Matched(matched) => return NextStart::Reference(index, matched), + ReferenceSyntax::Malformed(kind) => { + return NextStart::Stop(DeclarationReferenceStop::Malformed(kind)); } }, _ => Some(index + 1), diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index 5ef7cf4b4e..f97b85c131 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -9,8 +9,8 @@ use crate::compiler::models::{ AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, InterpolationRead, }; -use crate::compiler::tests::test_types::HeaderNestingTestCase; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::tests::test_types::{HeaderNestingTestCase, MapHost}; +use crate::compiler::types::CharSpan; use sqlbuild_core::text::main::python_text::python_text; use std::collections::BTreeMap; use std::thread; @@ -262,51 +262,6 @@ pub(crate) fn nesting_error_at(position: usize) -> String { format!("values nest deeper than 256 levels at position {position}") } -/// Variables, environment and context for interpolation tests. -pub(crate) struct MapHost { - pub(crate) variables: BTreeMap<&'static str, Result<&'static str, &'static str>>, - pub(crate) environment: BTreeMap<&'static str, &'static str>, - pub(crate) context: Option>>, -} - -impl InterpolationHost for MapHost { - fn variable(&self, name: &str) -> Option> { - self.variables - .get(name) - .map(|value| value.map(str::to_owned).map_err(str::to_owned)) - } - - fn variable_names(&self) -> Vec { - self.variables - .keys() - .map(|name| (*name).to_owned()) - .collect() - } - - fn environment(&self, name: &str) -> Result, String> { - Ok(self.environment.get(name).map(|value| (*value).to_owned())) - } - - fn context_allowed(&self) -> bool { - self.context.is_some() - } - - fn context(&self, name: &str) -> Option> { - self.context - .as_ref()? - .get(name) - .map(|value| value.map(str::to_owned)) - } - - fn context_names(&self) -> Vec { - self.context - .iter() - .flat_map(BTreeMap::keys) - .map(|name| (*name).to_owned()) - .collect() - } -} - /// A host with `region = north`, `revision = 7`, `REGION=eu` and context `this`, `this.schema`. pub(crate) fn interpolation_host(context: bool) -> MapHost { MapHost { @@ -341,8 +296,8 @@ pub(crate) fn interpolation_facts( sql: &str, ) -> (Vec, Vec) { let python = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); - match interpolate_sql(python, host, sql, "models/orders.sql") { - Ok(result) => (result.spans, result.reads), - Err(failure) => (Vec::new(), failure.reads), - } + interpolate_sql(python, host, sql, "models/orders.sql").map_or_else( + |failure| (Vec::new(), failure.reads), + |result| (result.spans, result.reads), + ) } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index 2461676343..7a2eafc102 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -1,6 +1,6 @@ use crate::compiler::models::InterpolationRead; use crate::compiler::tests::helpers::{interpolated, interpolation_facts, interpolation_host}; -use crate::compiler::tests::test_types::InterpolationTestCase; +use crate::compiler::tests::test_types::{InterpolationFactsTestCase, InterpolationTestCase}; #[test] fn given_sql_when_interpolating_then_text_and_errors_match_python() { @@ -11,55 +11,55 @@ fn given_sql_when_interpolating_then_text_and_errors_match_python() { description: "variables in code and quotes; comments and @@@ stay", sql: "SELECT @@revision, '@@region' -- @@region\n/* @@region */, @@@window_start, @@1", context: false, - expected: Ok("SELECT 7, 'north' -- @@region\n/* @@region */, @@@window_start, @@1"), + expected_sql: Ok("SELECT 7, 'north' -- @@region\n/* @@region */, @@@window_start, @@1"), }, InterpolationTestCase { description: "doubled quotes and backticks", sql: "SELECT '@@revision''s', `@@region``@@region`", context: false, - expected: Ok("SELECT '7''s', `north``north`"), + expected_sql: Ok("SELECT '7''s', `north``north`"), }, InterpolationTestCase { description: "dollar-quoted text is quoted text", sql: "SELECT $tag$ it's $$ -- @@region $tag$, @@region, price$1$ -- @@region", context: false, - expected: Ok("SELECT $tag$ it's $$ -- north $tag$, north, price$1$ -- @@region"), + expected_sql: Ok("SELECT $tag$ it's $$ -- north $tag$, north, price$1$ -- @@region"), }, InterpolationTestCase { description: "environment variables, including non-ASCII names", sql: "SELECT '@@ENV:REGION', @@ENV:ÜBER", context: false, - expected: Ok("SELECT 'eu', yes"), + expected_sql: Ok("SELECT 'eu', yes"), }, InterpolationTestCase { description: "non-ASCII variable names", sql: "SELECT @@größe", context: false, - expected: Ok("SELECT large"), + expected_sql: Ok("SELECT large"), }, InterpolationTestCase { description: "context keys take the longest known dotted prefix", sql: "GRANT SELECT ON @@CTX:this.schema.@@CTX:this.extra TO reporting", context: true, - expected: Ok("GRANT SELECT ON sales.orders.extra TO reporting"), + expected_sql: Ok("GRANT SELECT ON sales.orders.extra TO reporting"), }, InterpolationTestCase { description: "an unknown variable lists the available ones", sql: "SELECT '@@missing'", context: false, - expected: Err(unknown_variable), + expected_sql: Err(unknown_variable), }, InterpolationTestCase { description: "a structured variable reports its rendering error", sql: "SELECT @@layout", context: false, - expected: Err("SQL variable '@@layout' is an object"), + expected_sql: Err("SQL variable '@@layout' is an object"), }, InterpolationTestCase { description: "a missing environment variable", sql: "SELECT @@ENV:SALES_REGION", context: false, - expected: Err( + expected_sql: Err( "unknown environment variable '@@ENV:SALES_REGION' in 'models/orders.sql'", ), }, @@ -67,31 +67,31 @@ fn given_sql_when_interpolating_then_text_and_errors_match_python() { description: "an empty environment name", sql: "SELECT @@ENV:-1", context: false, - expected: Err("invalid environment interpolation token in 'models/orders.sql'"), + expected_sql: Err("invalid environment interpolation token in 'models/orders.sql'"), }, InterpolationTestCase { description: "context outside hooks", sql: "SELECT @@CTX:this", context: false, - expected: Err("SQL text in 'models/orders.sql' does not allow @@CTX templates"), + expected_sql: Err("SQL text in 'models/orders.sql' does not allow @@CTX templates"), }, InterpolationTestCase { description: "an empty context name", sql: "SELECT @@CTX:", context: true, - expected: Err("invalid CTX interpolation token in 'models/orders.sql'"), + expected_sql: Err("invalid CTX interpolation token in 'models/orders.sql'"), }, InterpolationTestCase { description: "an unknown context key", sql: "SELECT @@CTX:that", context: true, - expected: Err("SQL text in 'models/orders.sql' references unknown CTX key 'that'"), + expected_sql: Err("SQL text in 'models/orders.sql' references unknown CTX key 'that'"), }, InterpolationTestCase { description: "a context key without a value", sql: "SELECT @@CTX:run_id", context: true, - expected: Err( + expected_sql: Err( "SQL text in 'models/orders.sql' references CTX key 'run_id' but no value is \ available", ), @@ -100,26 +100,29 @@ fn given_sql_when_interpolating_then_text_and_errors_match_python() { description: "an unclosed quote", sql: "SELECT @@revision, 'open", context: false, - expected: Err("SQL interpolation contains an unclosed quoted string"), + expected_sql: Err("SQL interpolation contains an unclosed quoted string"), }, InterpolationTestCase { description: "an unclosed dollar quote", sql: "SELECT @@region, $$ open", context: false, - expected: Err("SQL interpolation contains an unclosed quoted string"), + expected_sql: Err("SQL interpolation contains an unclosed quoted string"), }, InterpolationTestCase { description: "an unclosed block comment", sql: "SELECT @@revision /* open", context: false, - expected: Err("SQL interpolation contains an unclosed block comment"), + expected_sql: Err("SQL interpolation contains an unclosed block comment"), }, ]; for test_case in test_cases { let host = interpolation_host(test_case.context); assert_eq!( interpolated(&host, test_case.sql), - test_case.expected.map(str::to_owned).map_err(str::to_owned), + test_case + .expected_sql + .map(str::to_owned) + .map_err(str::to_owned), "{}", test_case.description ); @@ -129,24 +132,27 @@ fn given_sql_when_interpolating_then_text_and_errors_match_python() { #[test] fn given_tokens_when_interpolating_then_spans_count_code_points_and_reads_keep_order() { let host = interpolation_host(true); - - let (spans, reads) = - interpolation_facts(&host, "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@"); - - assert_eq!( - spans, - vec![ + let test_cases = [InterpolationFactsTestCase { + description: "variables, environment, context and a bare @@ after non-ASCII text", + sql: "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@", + expected_spans: vec![ (12, 19, 12, 17), (21, 33, 19, 21), (35, 45, 23, 29), - (46, 48, 30, 32) - ] - ); - assert_eq!( - reads, - vec![ + (46, 48, 30, 32), + ], + expected_reads: vec![ InterpolationRead::Environment("REGION".to_owned()), InterpolationRead::Context("this".to_owned()), - ] - ); + ], + }]; + + for test_case in test_cases { + assert_eq!( + interpolation_facts(&host, test_case.sql), + (test_case.expected_spans, test_case.expected_reads), + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 5e0b67cfa4..d63a9008c0 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -1,4 +1,6 @@ -use crate::compiler::models::NestingFailure; +use crate::compiler::models::{InterpolationRead, NestingFailure}; +use crate::compiler::types::{CharSpan, InterpolationHost}; +use std::collections::BTreeMap; pub(crate) struct ModelHeaderTokenizationTestCase { pub(crate) description: &'static str, @@ -49,5 +51,57 @@ pub(crate) struct InterpolationTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, pub(crate) context: bool, - pub(crate) expected: Result<&'static str, &'static str>, + pub(crate) expected_sql: Result<&'static str, &'static str>, +} + +pub(crate) struct InterpolationFactsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_spans: Vec, + pub(crate) expected_reads: Vec, +} + +/// Variables, environment and context for interpolation tests. +pub(crate) struct MapHost { + pub(crate) variables: BTreeMap<&'static str, Result<&'static str, &'static str>>, + pub(crate) environment: BTreeMap<&'static str, &'static str>, + pub(crate) context: Option>>, +} + +impl InterpolationHost for MapHost { + fn variable(&self, name: &str) -> Option> { + self.variables + .get(name) + .map(|value| value.map(str::to_owned).map_err(str::to_owned)) + } + + fn variable_names(&self) -> Vec { + self.variables + .keys() + .map(|name| (*name).to_owned()) + .collect() + } + + fn environment(&self, name: &str) -> Result, String> { + Ok(self.environment.get(name).map(|value| (*value).to_owned())) + } + + fn context_allowed(&self) -> bool { + self.context.is_some() + } + + fn context(&self, name: &str) -> Option> { + self.context + .as_ref()? + .get(name) + .map(|value| value.map(str::to_owned)) + } + + fn context_names(&self) -> Vec { + self.context + .iter() + .flat_map(BTreeMap::keys) + .map(|name| (*name).to_owned()) + .collect() + } } diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 2f0612cc46..dd8340322c 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -41,6 +41,12 @@ _RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' +_ENGINE_ERROR_MACROS: str = "macros/cents.py" +_ENGINE_ERROR_CENTS: str = ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + ' return f"({expression} * 100)"\n' +) + def _rules(*codes: str) -> dict[str, str]: return { @@ -597,13 +603,6 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) -_ENGINE_ERROR_MACROS: str = "macros/cents.py" -_ENGINE_ERROR_CENTS: str = ( - 'def cents(expression: str) -> str:\n """Convert to cents."""\n' - ' return f"({expression} * 100)"\n' -) - - def engine_error_cases() -> tuple[FailureCase, ...]: """User-facing errors every engine must report with the exact text; see the README.""" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a89084ae72..852ab2c0b7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -309,7 +309,7 @@ def interpolate_sql_batch( Mapping[str, object], Mapping[str, str], Mapping[str, str | None] | None, - Callable[[object, str], str], + Callable[..., str], ], python_version: tuple[int, int], unicode_version: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py index f7a9b563d3..c7c079ddbc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path from typing import cast @@ -10,12 +10,30 @@ from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( typed_reference_value, ) +from sqlbuild.compiler.compile.constants import MACRO_ARGUMENT_ERROR_TAG from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ParsedMacroArguments from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.types import NativeStage type _ValueRow = tuple[object, ...] +type _BuildContext = tuple[Sequence[object], str] +_BUILDERS: dict[str, Callable[..., object]] = { + "s": lambda *, row, context: row[1], + "b": lambda *, row, context: row[1], + "n": lambda *, row, context: None, + "i": lambda *, row, context: int(cast(str, row[2]), cast(int, row[1])), + "f": lambda *, row, context: float(cast(str, row[1])), + "c": lambda *, row, context: context[0][cast(int, row[1])], + "r": lambda *, row, context: typed_reference_value( + function=cast(str, row[1]), name=cast(str, row[2]) + ), + "l": lambda *, row, context: _items(row=row, context=context), + "t": lambda *, row, context: tuple(_items(row=row, context=context)), + "d": lambda *, row, context: _mapping(row=row, context=context), + "-": lambda *, row, context: -_number(row=row, context=context), + "+": lambda *, row, context: _number(row=row, context=context), +} def parse_macro_call_arguments( @@ -30,7 +48,7 @@ def parse_macro_call_arguments( parsed: tuple[object, ...] = _native.parse_macro_arguments(args_source, list(nested)) label: str = f"the '@{macro_name}' arguments" - if parsed[0] == "error": + if parsed[0] == MACRO_ARGUMENT_ERROR_TAG: _, detail, help_text, line, column = parsed raise CompileInputError( f"Macro arguments in '{file_path}' {detail} at line {line}, column {column} of {label}", @@ -38,11 +56,12 @@ def parse_macro_call_arguments( ) _, positional, keywords, references = parsed report_native_answer(stage=NativeStage.MACRO_CALLS, kind="parsed_arguments") - context: tuple[Sequence[object], str] = (nested_values, f"'{file_path}' in {label}") + context: _BuildContext = (nested_values, f"'{file_path}' in {label}") return ParsedMacroArguments( - args=tuple(_built(row, context) for row in cast(list[_ValueRow], positional)), + args=tuple(_built(row=row, context=context) for row in cast(list[_ValueRow], positional)), kwargs={ - name: _built(row, context) for name, row in cast(list[tuple[str, _ValueRow]], keywords) + name: _built(row=row, context=context) + for name, row in cast(list[tuple[str, _ValueRow]], keywords) }, typed_references=tuple( typed_reference_value(function=function, name=name) @@ -51,33 +70,26 @@ def parse_macro_call_arguments( ) -def _built(row: _ValueRow, context: tuple[Sequence[object], str]) -> object: - tag: object = row[0] - if tag in ("s", "b"): - return row[1] - if tag == "i": - return int(cast(str, row[2]), cast(int, row[1])) - if tag == "f": - return float(cast(str, row[1])) - if tag == "c": - return context[0][cast(int, row[1])] - if tag == "r": - return typed_reference_value(function=cast(str, row[1]), name=cast(str, row[2])) - if tag == "l": - return [_built(item, context) for item in cast(list[_ValueRow], row[1])] - if tag == "t": - return tuple(_built(item, context) for item in cast(list[_ValueRow], row[1])) - if tag == "d": - return { - _built(key, context): _built(value, context) - for key, value in cast(list[tuple[_ValueRow, _ValueRow]], row[1]) - } - if tag in ("-", "+"): - number: object = _built(cast(_ValueRow, row[1]), context) - if not isinstance(number, int | float): - raise CompileInputError( - f"Macro arguments in {context[1]} use unary + or - on a value that is not a number", - help="Apply the sign inside the macro, or pass a number literal", - ) - return -number if tag == "-" else number - return None +def _built(*, row: _ValueRow, context: _BuildContext) -> object: + return _BUILDERS[cast(str, row[0])](row=row, context=context) + + +def _items(*, row: _ValueRow, context: _BuildContext) -> list[object]: + return [_built(row=item, context=context) for item in cast(list[_ValueRow], row[1])] + + +def _mapping(*, row: _ValueRow, context: _BuildContext) -> dict[object, object]: + return { + _built(row=key, context=context): _built(row=value, context=context) + for key, value in cast(list[tuple[_ValueRow, _ValueRow]], row[1]) + } + + +def _number(*, row: _ValueRow, context: _BuildContext) -> int | float: + number: object = _built(row=cast(_ValueRow, row[1]), context=context) + if not isinstance(number, int | float): + raise CompileInputError( + f"Macro arguments in {context[1]} use unary + or - on a value that is not a number", + help="Apply the sign inside the macro, or pass a number literal", + ) + return number diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 0ecdfe8b25..8c5303571f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -36,6 +36,10 @@ ) from sqlbuild.sql_values.types import CollectionRendering +type _InterpolationRow = tuple[ + str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None +] + def expand_authored_sql_result( # noqa: PLR0913 *, @@ -177,20 +181,14 @@ def interpolate_sql_batch( ) -> tuple[SqlInterpolation, ...]: """Interpolate every `(sql, file path)` natively; errors wait in their result.""" - rows = _native.interpolate_sql_batch( + rows: list[_InterpolationRow] = _native.interpolate_sql_batch( [(sql, str(file_path)) for sql, file_path in sqls], - (effective_vars, UnicodeEnvironment(), context_values, _render_variable), + (effective_vars, UnicodeEnvironment(), context_values, render_project_var_text), (sys.version_info[0], sys.version_info[1]), unicodedata.unidata_version, ) return tuple( - SqlInterpolation( - sql=rendered if rendered is not None else sql, - spans=tuple(ExpansionSpan(*span) for span in spans), - reads=tuple(reads), - error=error, - ) - for (sql, _file_path), (rendered, spans, reads, error) in zip(sqls, rows, strict=True) + _interpolation(sql=sql, row=row) for (sql, _file_path), row in zip(sqls, rows, strict=True) ) @@ -203,5 +201,15 @@ def applied_interpolation(interpolation: SqlInterpolation) -> str: return interpolation.sql -def _render_variable(value: object, label: str) -> str: - return render_project_var_text(value=value, label=label) +def _interpolation( + *, + sql: str, + row: _InterpolationRow, +) -> SqlInterpolation: + rendered, spans, reads, error = row + return SqlInterpolation( + sql=rendered if rendered is not None else sql, + spans=tuple(ExpansionSpan(*span) for span in spans), + reads=tuple(reads), + error=error, + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 2fa1b1bc3c..be186aeb85 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -110,6 +110,7 @@ MACRO_TOKEN: str = "@" DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" +MACRO_ARGUMENT_ERROR_TAG: str = "error" SQL_INTERPOLATION_TOKEN: str = "@@" TEMPLATE_OPEN_TOKEN: str = "${" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 6052b8bc76..dad7c3bfbc 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -148,19 +148,18 @@ def discover_declaration_file(*, project_dir: Path, relative_path: str, contents } -_DECLARATION_STATEMENT_DIRECTORIES: dict[str, str] = { - "AUDIT": "audits/generic", - "HOOK": "hooks/sql", +_STATEMENT_FILE_PATHS: dict[str, Callable[[Path], Path]] = { + "AUDIT": lambda project_dir: project_dir / "audits" / "generic" / "orders.sql", + "HOOK": lambda project_dir: project_dir / "hooks" / "sql" / "orders.sql", } def statement_header_file_path(*, statement: str, project_dir: Path) -> Path: """Return where a statement file is parsed from: in the project for declaration kinds.""" - directory: str | None = _DECLARATION_STATEMENT_DIRECTORIES.get(statement) - if directory is None: - return Path("orders.sql") - return project_dir / directory / "orders.sql" + return _STATEMENT_FILE_PATHS.get(statement, lambda _project_dir: Path("orders.sql"))( + project_dir + ) def parse_statement_header_file(*, statement: str, contents: str, file_path: Path) -> object: From 8e69e54403cc30d2906fd423bc53af2b657a8bbe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:27:48 +0100 Subject: [PATCH 11/40] fix(native): address the expansion port review: no secret echo, located macro argument errors, honest native counts --- .../_helpers/checks/python_values.rs | 22 +++ .../_helpers/parsing/constant_files.rs | 3 +- .../tests/test_parse_declaration_text.rs | 9 ++ .../_helpers/render/macro_arguments.rs | 4 +- .../src/macro_arguments/_helpers/failures.rs | 46 +++++- .../src/macro_arguments/_helpers/literals.rs | 25 +++- .../macro_arguments/_helpers/syntax_tree.rs | 34 ++++- .../main/parse_macro_arguments.rs | 2 +- .../tests/test_parse_macro_arguments.rs | 30 ++++ .../src/macro_arguments/tests/test_types.rs | 8 +- .../src/macro_arguments/types.rs | 4 +- .../_helpers/corpus/failure_cases.py | 138 +++++++++++++++--- scripts/compiler_differential/constants.py | 2 +- .../native_fallbacks.toml | 85 +++++------ .../cli/commands/_helpers/compile/output.py | 8 +- .../_helpers/entry/dispatch_errors.py | 11 +- .../compile/_helpers/attachment/core.py | 72 +++++++-- .../_helpers/macro_bridge/expansion.py | 7 +- .../compile/_helpers/render/declarations.py | 5 +- .../_helpers/render/macro_argument_values.py | 69 +++++++-- .../compile/_helpers/render/macros.py | 8 +- .../compile/_helpers/render/templating.py | 22 +-- .../compile/classes/unicode_environment.py | 10 +- src/sqlbuild/compiler/compile/constants.py | 1 + src/sqlbuild/compiler/compile/exceptions.py | 60 ++++++++ src/sqlbuild/compiler/frontier/types.py | 1 + .../compile/test_compile_macro_call_store.py | 4 +- .../engine-error-macro-argument-bytes.json | 10 +- .../engine-error-macro-argument-complex.json | 10 +- .../engine-error-macro-argument-ellipsis.json | 10 +- ...e-error-macro-argument-named-sequence.json | 27 ++++ ...error-macro-argument-nested-call-sign.json | 27 ++++ ...gine-error-macro-argument-second-call.json | 27 ++++ ...e-error-macro-argument-surrogate-pair.json | 27 ++++ .../engine-error-macro-argument-syntax.json | 10 +- ...-error-non-ascii-digit-constant-value.json | 19 +++ ...rror-undecodable-environment-variable.json | 4 +- .../failure/macro-arguments-do-not-parse.json | 10 +- .../failure/macro-non-literal-argument.json | 10 +- .../compiler/compile/_helpers/_test_types.py | 10 ++ .../compile/_helpers/test_templating.py | 74 +++++++++- .../src/content/docs/docs/concepts/macros.mdx | 10 +- 42 files changed, 831 insertions(+), 144 deletions(-) create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-named-sequence.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-nested-call-sign.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-second-call.json create mode 100644 tests/goldens/compiler/failure/engine-error-macro-argument-surrogate-pair.json create mode 100644 tests/goldens/compiler/failure/engine-error-non-ascii-digit-constant-value.json diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs index a2f8922839..477d4aadf5 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs @@ -49,6 +49,28 @@ pub(crate) fn python_type( }) } +/// Reject a bare number written with non-ASCII digits anywhere inside `value`. +pub(crate) fn check_bare_numbers(value: &AuthoredValue, words: WordRules) -> Result<(), ParseStop> { + match value { + AuthoredValue::BareWord(word) => word_type(word, words).map(|_| ()), + AuthoredValue::List(items) | AuthoredValue::Set(items) | AuthoredValue::Tuple(items) => { + items + .iter() + .try_for_each(|item| check_bare_numbers(item, words)) + } + AuthoredValue::Map(entries) + | AuthoredValue::TypedConstant(entries) + | AuthoredValue::NamedSqlHook(_, entries) + | AuthoredValue::PythonHook(_, entries) => entries + .iter() + .try_for_each(|(_, item)| check_bare_numbers(item, words)), + AuthoredValue::Null + | AuthoredValue::Boolean(_) + | AuthoredValue::String(_) + | AuthoredValue::InlineSqlHook(_) => Ok(()), + } +} + /// The text of a value that projects to `str`, or `None` for any other type. pub(crate) fn python_str<'value>( value: &'value AuthoredValue, diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs index b6a5ba6f24..4a940f315c 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs @@ -1,7 +1,7 @@ //! Python's `parse_constant_declaration_file` up to value normalisation, which Python performs. use crate::declaration_files::_helpers::checks::python_values::{ - WordRules, failure, get, python_str, unknown_keys, + WordRules, check_bare_numbers, failure, get, python_str, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; @@ -81,6 +81,7 @@ fn parse_constant( "{file_path} constant '{name}' is missing required value" ))); }; + check_bare_numbers(value, WordRules { python, file_path })?; let explicit_type: Option = constant_option( explicit_type, file_path, diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs index 27090fada5..1d47a86848 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs @@ -52,6 +52,15 @@ fn given_declaration_files_when_parsing_then_values_and_failures_match_python() "Quote it to keep it as text", ], }, + DeclarationTextTestCase { + description: "a constant bare number in non-ASCII digits is rejected, nested too", + kind: CollectionKind::Constants, + contents: "CONSTANT (name limits, value [1, \u{663}]);", + expected_fragments: &[ + "/project/declarations/orders.sql has the bare number '\u{663}', written with non-ASCII digits", + "Quote it to keep it as text", + ], + }, DeclarationTextTestCase { description: "a non-ASCII bare word that is not a number is text", kind: CollectionKind::Enums, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs index aa76ad8fd9..9c5392a429 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs @@ -32,10 +32,10 @@ impl UnicodeHost<'_> { } impl ArgumentHost for UnicodeHost<'_> { - fn character_named(&self, name: &str) -> Option { + fn character_named(&self, name: &str) -> Option { let py: Python<'_> = self.unicodedata.py(); match self.unicodedata.call_method1("lookup", (name,)) { - Ok(found) => self.held(found.extract::()), + Ok(found) => self.held(found.extract::()), Err(error) if error.is_instance_of::(py) => None, Err(error) => self.held(Err(error)), } diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs index 01f1cad354..e1b52f9b76 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/failures.rs @@ -14,19 +14,25 @@ const UNPACKING_HELP: &str = "Write each argument or key out explicitly"; const KEYWORD_HELP: &str = "Pass each keyword argument once, after every positional argument"; const SURROGATE_HELP: &str = "Lone surrogates are not valid Unicode text and cannot be sent to a \ warehouse; write the character itself or the escape of a valid code point"; +const MISSING_COMMA_HELP: &str = "Separate arguments, and the items of lists, tuples and dicts, \ + with commas, for example @cents('amount', 2)"; +const SURROGATE_PAIR_HELP: &str = "Python strings hold code points, not UTF-16 surrogate pairs; \ + write the character as one escape:"; +const NAMED_SEQUENCE_HELP: &str = "Write each character of the sequence with its own \\N{...} \ + escape, or write the characters themselves"; /// One argument error at a code point position of the argument text. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct Failure { pub(crate) detail: String, - pub(crate) help: &'static str, + pub(crate) help: String, pub(crate) position: usize, } -fn failure(detail: &str, help: &'static str, position: usize) -> Failure { +fn failure(detail: &str, help: &str, position: usize) -> Failure { Failure { detail: detail.to_owned(), - help, + help: help.to_owned(), position, } } @@ -74,6 +80,14 @@ pub(crate) fn typed_reference(function: &str, position: usize) -> Failure { ) } +pub(crate) fn missing_comma(position: usize) -> Failure { + failure( + "could not be parsed: a comma is missing between arguments", + MISSING_COMMA_HELP, + position, + ) +} + pub(crate) fn unary(position: usize) -> Failure { failure( "use unary + or - on a value that is not a number", @@ -98,10 +112,32 @@ pub(crate) fn keyword_order(detail: &str, position: usize) -> Failure { failure(detail, KEYWORD_HELP, position) } -pub(crate) fn surrogate(escape: &str, position: usize) -> Failure { +/// A lone surrogate escape; `pair` is the code point a high and low surrogate pair spell. +pub(crate) fn surrogate(escape: &str, pair: Option, position: usize) -> Failure { + let help: String = pair.map_or_else( + || SURROGATE_HELP.to_owned(), + |code| { + format!( + "{SURROGATE_PAIR_HELP} \\U{code:08X} instead of the surrogate pair, or write \ + the character itself" + ) + }, + ); failure( &format!("contain the lone surrogate escape '{escape}'"), - SURROGATE_HELP, + &help, + position, + ) +} + +/// A `\N{name}` naming a named sequence of several characters. +pub(crate) fn named_sequence(name: &str, length: usize, position: usize) -> Failure { + failure( + &format!( + "could not be parsed: \\N{{{name}}} names a sequence of {length} characters, and \ + \\N escapes name one character" + ), + NAMED_SEQUENCE_HELP, position, ) } diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs index e4887c4078..ab0fc8b461 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/literals.rs @@ -1,6 +1,6 @@ //! Python string and number literals: escapes, prefixes, underscores and radixes. -use crate::macro_arguments::_helpers::failures::{Failure, surrogate, syntax}; +use crate::macro_arguments::_helpers::failures::{Failure, named_sequence, surrogate, syntax}; use crate::macro_arguments::_helpers::lexer::{NumberToken, StringPiece, is_name_start}; use crate::macro_arguments::constants::{BINARY_RADIX, DECIMAL_RADIX, HEX_RADIX, OCTAL_RADIX}; use crate::macro_arguments::types::ArgumentHost; @@ -125,7 +125,7 @@ fn hex_escape(text: &[char], index: usize, width: usize) -> Result<(String, usiz let end: usize = index + 2 + width; if (0xD800..=0xDFFF).contains(&code) { let escape: String = text[index..end].iter().collect(); - return Err(surrogate(&escape, index)); + return Err(surrogate(&escape, surrogate_pair(text, code, end), index)); } let character: char = char::from_u32(code).ok_or_else(|| { let escape: String = text[index..end].iter().collect(); @@ -150,10 +150,27 @@ fn named_escape( return Err(syntax("a \\N escape needs a {name}", index)); }; let name: String = text[index + 3..index + 3 + close].iter().collect(); - let character: char = host + let named: String = host .character_named(&name) .ok_or_else(|| syntax(&format!("\\N{{{name}}} names no Unicode character"), index))?; - Ok((character.to_string(), index + 4 + close)) + let length: usize = named.chars().count(); + if length != 1 { + return Err(named_sequence(&name, length, index)); + } + Ok((named, index + 4 + close)) +} + +/// The code point a high surrogate `code` and a `\u` low surrogate escape at `next` spell. +fn surrogate_pair(text: &[char], code: u32, next: usize) -> Option { + let high: bool = (0xD800..=0xDBFF).contains(&code); + let digits: &[char] = text.get(next + 2..next + 6)?; + let low: u32 = (high + && text.get(next..next + 2) == Some(&['\\', 'u'][..]) + && digits.iter().all(char::is_ascii_hexdigit)) + .then(|| radix_value(digits, HEX_RADIX))?; + (0xDC00..=0xDFFF) + .contains(&low) + .then(|| 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00)) } fn radix_value(digits: &[char], radix: u32) -> u32 { diff --git a/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs b/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs index 70d9f8bb02..e5ed3ae8e2 100644 --- a/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs +++ b/crates/sqlbuild-render/src/macro_arguments/_helpers/syntax_tree.rs @@ -1,6 +1,8 @@ //! Parse argument tokens into Python's `ast` node shapes for the literal subset. -use crate::macro_arguments::_helpers::failures::{Failure, keyword_order, syntax, unsupported}; +use crate::macro_arguments::_helpers::failures::{ + Failure, keyword_order, missing_comma, syntax, unsupported, +}; use crate::macro_arguments::_helpers::lexer::{NumberToken, Token, TokenKind}; const PYTHON_KEYWORDS: [&str; 35] = [ @@ -98,6 +100,9 @@ impl Parser<'_> { /// The error for the next token where no supported argument can continue. fn unexpected(&self) -> Failure { let token: &Token = self.peek(); + if self.missing_comma() { + return missing_comma(token.position); + } match &token.kind { TokenKind::End => syntax( "the arguments end where a value is expected", @@ -111,6 +116,33 @@ impl Parser<'_> { } } + /// Whether the next token starts a value right after one ended, so a comma is missing. + fn missing_comma(&self) -> bool { + let starts_value: bool = matches!( + self.peek().kind, + TokenKind::Str(_) + | TokenKind::Number(_) + | TokenKind::Name(_) + | TokenKind::Nested(_) + | TokenKind::Operator("{") + ); + let ends_value: bool = self + .index + .checked_sub(1) + .and_then(|previous| self.tokens.get(previous)) + .is_some_and(|token| { + matches!( + token.kind, + TokenKind::Str(_) + | TokenKind::Number(_) + | TokenKind::Name(_) + | TokenKind::Nested(_) + | TokenKind::Operator(")" | "]" | "}") + ) + }); + starts_value && ends_value + } + /// Arguments up to a `)` or the end, as Python's call grammar orders them. fn arguments(&mut self) -> Result { let mut args: Vec = Vec::new(); diff --git a/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs index 4e9f64cd85..73ddd4905e 100644 --- a/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs +++ b/crates/sqlbuild-render/src/macro_arguments/main/parse_macro_arguments.rs @@ -35,7 +35,7 @@ fn located(characters: &[char], failure: Failure) -> ArgumentError { } ArgumentError { detail: failure.detail, - help: failure.help.to_owned(), + help: failure.help, line, column: failure.position - line_start + 1, } diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs index caabd374d6..eaf557532c 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_parse_macro_arguments.rs @@ -136,6 +136,24 @@ fn given_argument_text_when_parsing_then_values_match_python_literals() { nested: &[], expected_plan: "Err(ArgumentError { detail: \"could not be parsed: keyword argument \'x\' is repeated\", help: \"Pass each keyword argument once, after every positional argument\", line: 1, column: 6 })", }, + ParseMacroArgumentsTestCase { + description: "a missing comma between numbers", + text: "1 2", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a comma is missing between arguments\", help: \"Separate arguments, and the items of lists, tuples and dicts, with commas, for example @cents(\'amount\', 2)\", line: 1, column: 3 })", + }, + ParseMacroArgumentsTestCase { + description: "a missing comma between list items", + text: "[1 'a']", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a comma is missing between arguments\", help: \"Separate arguments, and the items of lists, tuples and dicts, with commas, for example @cents(\'amount\', 2)\", line: 1, column: 4 })", + }, + ParseMacroArgumentsTestCase { + description: "a missing comma between nested calls", + text: "@inner(1) @inner(2)", + nested: &[(0, 9), (10, 19)], + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: a comma is missing between arguments\", help: \"Separate arguments, and the items of lists, tuples and dicts, with commas, for example @cents(\'amount\', 2)\", line: 1, column: 11 })", + }, ParseMacroArgumentsTestCase { description: "a doubled comma", text: "'amount',,", @@ -160,6 +178,18 @@ fn given_argument_text_when_parsing_then_values_match_python_literals() { nested: &[], expected_plan: "Err(ArgumentError { detail: \"must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references\", help: \"Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro\", line: 2, column: 3 })", }, + ParseMacroArgumentsTestCase { + description: "a surrogate pair escape suggests the code point escape", + text: "'\\ud83d\\ude00'", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"contain the lone surrogate escape \'\\\\ud83d\'\", help: \"Python strings hold code points, not UTF-16 surrogate pairs; write the character as one escape: \\\\U0001F600 instead of the surrogate pair, or write the character itself\", line: 1, column: 2 })", + }, + ParseMacroArgumentsTestCase { + description: "a named sequence is not one character", + text: "'\\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}'", + nested: &[], + expected_plan: "Err(ArgumentError { detail: \"could not be parsed: \\\\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE} names a sequence of 2 characters, and \\\\N escapes name one character\", help: \"Write each character of the sequence with its own \\\\N{...} escape, or write the characters themselves\", line: 1, column: 2 })", + }, ParseMacroArgumentsTestCase { description: "a lone surrogate escape", text: "'\\ud800'", diff --git a/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs index 4d4a848364..ce7f8769e2 100644 --- a/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs +++ b/crates/sqlbuild-render/src/macro_arguments/tests/test_types.rs @@ -12,8 +12,12 @@ pub(crate) struct ParseMacroArgumentsTestCase { pub(crate) struct TestHost; impl ArgumentHost for TestHost { - fn character_named(&self, name: &str) -> Option { - (name == "LATIN SMALL LETTER E WITH ACUTE").then_some('é') + fn character_named(&self, name: &str) -> Option { + match name { + "LATIN SMALL LETTER E WITH ACUTE" => Some("é".to_owned()), + "LATIN CAPITAL LETTER A WITH MACRON AND GRAVE" => Some("\u{100}\u{300}".to_owned()), + _ => None, + } } fn identifier(&self, text: &str) -> Option { diff --git a/crates/sqlbuild-render/src/macro_arguments/types.rs b/crates/sqlbuild-render/src/macro_arguments/types.rs index f5dc60fb24..0ca2a683d2 100644 --- a/crates/sqlbuild-render/src/macro_arguments/types.rs +++ b/crates/sqlbuild-render/src/macro_arguments/types.rs @@ -2,8 +2,8 @@ /// Unicode lookups the host Python answers, so names match CPython's own tables. pub trait ArgumentHost { - /// The character `\N{name}` names, or `None` when Python knows no such name. - fn character_named(&self, name: &str) -> Option; + /// `unicodedata.lookup(name)`, several characters for a named sequence, or `None`. + fn character_named(&self, name: &str) -> Option; /// The NFKC form of a non-ASCII identifier, or `None` when Python rejects it as one. fn identifier(&self, text: &str) -> Option; } diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index dd8340322c..a54814abb6 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -46,6 +46,11 @@ 'def cents(expression: str) -> str:\n """Convert to cents."""\n' ' return f"({expression} * 100)"\n' ) +_MACRO_LITERALS_HELP: str = ( + "Macro arguments are Python literals (strings, numbers, True, False, None, lists, " + "tuples and dicts), nested macro calls, and __ref(), __source() or __seed() " + "references; compute anything else inside the macro" +) def _rules(*codes: str) -> dict[str, str]: @@ -633,6 +638,18 @@ def engine_error_cases() -> tuple[FailureCase, ...]: "(column names are case-insensitive)" ), ), + failure_case( + name="engine-error-non-ascii-digit-constant-value", + expected_code="D013", + files={"constants/limits.sql": "CONSTANT (name minimum_amount, value \u0663);\n"}, + expected_message=( + "/constants/limits.sql has the bare number '\u0663', written with " + "non-ASCII digits" + ), + expected_help=( + 'Quote it to keep it as text ("\u0663"), or write the number with ASCII digits 0-9' + ), + ), failure_case( name="engine-error-non-ascii-digit-bare-number", expected_code="D013", @@ -660,14 +677,94 @@ def engine_error_cases() -> tuple[FailureCase, ...]: ), }, expected_message=( - f"Macro arguments in '/{FAILURE_MART_PATH}' " - "could not be parsed: a value is missing here at line 1, column 10 of the " - "'@cents' arguments" + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "could not be parsed: a value is missing here at line 5, column 37" + ), + expected_location=(5, 37), + expected_help=_MACRO_LITERALS_HELP, + ), + failure_case( + name="engine-error-macro-argument-second-call", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, @cents('amount') AS cents,\n" + " @cents('amount' 2) AS more_cents\n" + 'FROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "could not be parsed: a comma is missing between arguments at line 6, column 19" + ), + expected_location=(6, 19), + expected_help=( + "Separate arguments, and the items of lists, tuples and dicts, with commas, for " + "example @cents('amount', 2)" + ), + ), + failure_case( + name="engine-error-macro-argument-nested-call-sign", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, @cents(-@cents('amount')) AS cents\n" + 'FROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' apply unary - " + "to the value of a nested macro call, which is a str and not a number at line 5, " + "column 29" + ), + expected_location=(5, 29), + expected_help=( + "Return a number from the nested macro, or apply the sign inside the macro" + ), + ), + failure_case( + name="engine-error-macro-argument-named-sequence", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, " + "@cents('\\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}') AS cents\n" + 'FROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' could not be " + "parsed: \\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE} names a sequence of 2 " + "characters, and \\N escapes name one character at line 5, column 29" + ), + expected_location=(5, 29), + expected_help=( + "Write each character of the sequence with its own \\N{...} escape, or write the " + "characters themselves" + ), + ), + failure_case( + name="engine-error-macro-argument-surrogate-pair", + expected_code="P001", + files={ + _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, + **mart_body_files( + "SELECT customer_id, @cents('\\ud83d\\ude00') AS cents\n" + 'FROM __ref("stg_orders")\n' + ), + }, + expected_message=( + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' contain the lone " + "surrogate escape '\\ud83d' at line 5, column 29" ), + expected_location=(5, 29), expected_help=( - "Macro arguments are Python literals (strings, numbers, True, False, None, lists, " - "tuples and dicts), nested macro calls, and __ref(), __source() or __seed() " - "references; compute anything else inside the macro" + "Python strings hold code points, not UTF-16 surrogate pairs; write the character " + "as one escape: \\U0001F600 instead of the surrogate pair, or write the character " + "itself" ), ), failure_case( @@ -676,13 +773,14 @@ def engine_error_cases() -> tuple[FailureCase, ...]: files={ _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, **mart_body_files( - "SELECT customer_id, @cents(b'amount') AS cents\\nFROM __ref(\"stg_orders\")\n" + "SELECT customer_id, @cents(b'amount') AS cents\nFROM __ref(\"stg_orders\")\n" ), }, expected_message=( - f"Macro arguments in '/{FAILURE_MART_PATH}' " - "use a bytes literal at line 1, column 1 of the '@cents' arguments" + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "use a bytes literal at line 5, column 28" ), + expected_location=(5, 28), expected_help=( "Pass text as a string without the b prefix, for example 'orders' instead of " "b'orders'" @@ -694,13 +792,14 @@ def engine_error_cases() -> tuple[FailureCase, ...]: files={ _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, **mart_body_files( - 'SELECT customer_id, @cents(2j) AS cents\\nFROM __ref("stg_orders")\n' + 'SELECT customer_id, @cents(2j) AS cents\nFROM __ref("stg_orders")\n' ), }, expected_message=( - f"Macro arguments in '/{FAILURE_MART_PATH}' " - "use a complex number literal at line 1, column 1 of the '@cents' arguments" + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "use a complex number literal at line 5, column 28" ), + expected_location=(5, 28), expected_help=( "Pass int or float numbers; give a complex value's real and imaginary parts as two " "arguments" @@ -712,13 +811,14 @@ def engine_error_cases() -> tuple[FailureCase, ...]: files={ _ENGINE_ERROR_MACROS: _ENGINE_ERROR_CENTS, **mart_body_files( - 'SELECT customer_id, @cents(...) AS cents\\nFROM __ref("stg_orders")\n' + 'SELECT customer_id, @cents(...) AS cents\nFROM __ref("stg_orders")\n' ), }, expected_message=( - f"Macro arguments in '/{FAILURE_MART_PATH}' " - "use '...' (Ellipsis) at line 1, column 1 of the '@cents' arguments" + f"Macro arguments of '@cents' in '/{FAILURE_MART_PATH}' " + "use '...' (Ellipsis) at line 5, column 28" ), + expected_location=(5, 28), expected_help=("Pass None, or a string the macro understands, instead of '...'"), ), failure_case( @@ -729,11 +829,13 @@ def engine_error_cases() -> tuple[FailureCase, ...]: 'FROM __ref("stg_orders")\n' ), expected_message=( - f"Environment variable '{FAILURE_UNDECODABLE_ENV_VAR}' is not valid UTF-8 text" + f"Environment variable '{FAILURE_UNDECODABLE_ENV_VAR}' is not valid UTF-8 text: " + "byte 17 starts an invalid UTF-8 sequence" ), expected_help=( - "Its value is b'\\xff'; set it to UTF-8 text, since SQLBuild cannot send " - "undecodable bytes to a warehouse" + f"Set '{FAILURE_UNDECODABLE_ENV_VAR}' to UTF-8 text, since SQLBuild cannot send " + "undecodable bytes to a warehouse; the value is not shown because it may be a " + "secret" ), ), failure_case( diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index a2b7b2c605..c5daaf8be2 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -333,7 +333,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: FAILURE_UNDECODABLE_ENV_VAR: str = "SQB_FAILURE_UNDECODABLE_REGION" GENERATOR_ENVIRONMENT: dict[str, str] = { GENERATOR_CHANNEL_ENV_VAR: "web-orders", - FAILURE_UNDECODABLE_ENV_VAR: "\udcff", + FAILURE_UNDECODABLE_ENV_VAR: "north-region-key-\udcff", } GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" GENERATOR_CRLF: str = "\r\n" diff --git a/scripts/compiler_differential/native_fallbacks.toml b/scripts/compiler_differential/native_fallbacks.toml index 656fd241c5..37349b0142 100644 --- a/scripts/compiler_differential/native_fallbacks.toml +++ b/scripts/compiler_differential/native_fallbacks.toml @@ -74,7 +74,7 @@ engine = "native" stage = "declaration_files" site = "declaration_files.native" kind = "parsed_files" -counts = { example = 32, failure = 74, fixture = 20, seed = 799 } +counts = { example = 32, failure = 79, fixture = 20, seed = 799 } [[entry]] engine = "native" @@ -83,19 +83,12 @@ site = "declaration_files.native" kind = "seed_files" counts = { example = 8, failure = 3, fixture = 4, seed = 43 } -[[entry]] -engine = "native" -stage = "declaration_scopes" -site = "declaration_scopes.native" -kind = "resolver_lookups" -counts = { example = 12, failure = 231, fixture = 12, seed = 73 } - [[entry]] engine = "native" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 230, fixture = 12, seed = 44 } +counts = { example = 12, failure = 234, fixture = 12, seed = 44 } [[entry]] engine = "native" @@ -109,7 +102,7 @@ engine = "native" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 296, fixture = 12, seed = 50 } +counts = { example = 12, failure = 301, fixture = 12, seed = 50 } [[entry]] engine = "native" @@ -118,19 +111,26 @@ site = "macro_call_store.native" kind = "served_calls" counts = { example = 18, fixture = 15, seed = 192 } +[[entry]] +engine = "native" +stage = "macro_calls" +site = "macro_calls.mocked_evaluation" +kind = "test_mock" +counts = { seed = 29 } + [[entry]] engine = "native" stage = "macro_calls" site = "macro_calls.native" kind = "bridged_calls" -counts = { example = 32, failure = 9, fixture = 28, seed = 748 } +counts = { example = 32, failure = 9, fixture = 28, seed = 719 } [[entry]] engine = "native" stage = "macro_calls" site = "macro_calls.native" kind = "parsed_arguments" -counts = { example = 6, failure = 13, fixture = 5, seed = 369 } +counts = { example = 6, failure = 16, fixture = 5, seed = 369 } [[entry]] engine = "native" @@ -158,14 +158,14 @@ engine = "native" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 307, fixture = 64, seed = 890 } +counts = { example = 56, failure = 311, fixture = 64, seed = 890 } [[entry]] engine = "native" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 245, fixture = 15, seed = 121 } +counts = { example = 29, failure = 249, fixture = 15, seed = 121 } [[entry]] engine = "native" @@ -207,14 +207,14 @@ engine = "native" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } [[entry]] engine = "native" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } [[entry]] engine = "native" @@ -305,7 +305,7 @@ engine = "native-preview" stage = "declaration_files" site = "declaration_files.native" kind = "parsed_files" -counts = { example = 32, failure = 74, fixture = 20, seed = 799 } +counts = { example = 32, failure = 79, fixture = 20, seed = 799 } [[entry]] engine = "native-preview" @@ -314,19 +314,12 @@ site = "declaration_files.native" kind = "seed_files" counts = { example = 8, failure = 3, fixture = 4, seed = 43 } -[[entry]] -engine = "native-preview" -stage = "declaration_scopes" -site = "declaration_scopes.native" -kind = "resolver_lookups" -counts = { example = 12, failure = 231, fixture = 12, seed = 73 } - [[entry]] engine = "native-preview" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 230, fixture = 12, seed = 44 } +counts = { example = 12, failure = 234, fixture = 12, seed = 44 } [[entry]] engine = "native-preview" @@ -340,7 +333,7 @@ engine = "native-preview" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 296, fixture = 12, seed = 50 } +counts = { example = 12, failure = 301, fixture = 12, seed = 50 } [[entry]] engine = "native-preview" @@ -356,19 +349,26 @@ site = "macro_call_store.native" kind = "served_calls" counts = { example = 18, fixture = 15, seed = 192 } +[[entry]] +engine = "native-preview" +stage = "macro_calls" +site = "macro_calls.mocked_evaluation" +kind = "test_mock" +counts = { seed = 29 } + [[entry]] engine = "native-preview" stage = "macro_calls" site = "macro_calls.native" kind = "bridged_calls" -counts = { example = 32, failure = 9, fixture = 28, seed = 748 } +counts = { example = 32, failure = 9, fixture = 28, seed = 719 } [[entry]] engine = "native-preview" stage = "macro_calls" site = "macro_calls.native" kind = "parsed_arguments" -counts = { example = 6, failure = 13, fixture = 5, seed = 369 } +counts = { example = 6, failure = 16, fixture = 5, seed = 369 } [[entry]] engine = "native-preview" @@ -431,14 +431,14 @@ engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 307, fixture = 64, seed = 890 } +counts = { example = 56, failure = 311, fixture = 64, seed = 890 } [[entry]] engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 245, fixture = 15, seed = 121 } +counts = { example = 29, failure = 249, fixture = 15, seed = 121 } [[entry]] engine = "native-preview" @@ -480,14 +480,14 @@ engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } [[entry]] engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } [[entry]] engine = "native-preview" @@ -536,7 +536,7 @@ engine = "python" stage = "declaration_files" site = "declaration_files.native" kind = "parsed_files" -counts = { example = 32, failure = 74, fixture = 20, seed = 799 } +counts = { example = 32, failure = 79, fixture = 20, seed = 799 } [[entry]] engine = "python" @@ -545,19 +545,12 @@ site = "declaration_files.native" kind = "seed_files" counts = { example = 8, failure = 3, fixture = 4, seed = 43 } -[[entry]] -engine = "python" -stage = "declaration_scopes" -site = "declaration_scopes.native" -kind = "resolver_lookups" -counts = { example = 12, failure = 231, fixture = 12, seed = 73 } - [[entry]] engine = "python" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 230, fixture = 12, seed = 44 } +counts = { example = 12, failure = 234, fixture = 12, seed = 44 } [[entry]] engine = "python" @@ -571,21 +564,21 @@ engine = "python" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 296, fixture = 12, seed = 50 } +counts = { example = 12, failure = 301, fixture = 12, seed = 50 } [[entry]] engine = "python" stage = "macro_calls" site = "macro_calls.native" kind = "parsed_arguments" -counts = { example = 32, failure = 13, fixture = 28, seed = 840 } +counts = { example = 32, failure = 16, fixture = 28, seed = 840 } [[entry]] engine = "python" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 245, fixture = 15, seed = 121 } +counts = { example = 29, failure = 249, fixture = 15, seed = 121 } [[entry]] engine = "python" @@ -599,11 +592,11 @@ engine = "python" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 418, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } [[entry]] engine = "python" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 465, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 232054aa0c..e0c6aa3c00 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -220,7 +220,9 @@ def format_compile_json( return json.dumps(result, indent=2) -def format_compile_error_json(*, code: str, message: str, help_text: str | None) -> str: +def format_compile_error_json( + *, code: str, message: str, help_text: str | None, location: SourceLocation | None = None +) -> str: """Serialize a compile that stopped on a raised error as one JSON diagnostic report.""" diagnostic: CompilerDiagnostic = CompilerDiagnostic( @@ -228,6 +230,10 @@ def format_compile_error_json(*, code: str, message: str, help_text: str | None) severity=DiagnosticSeverity.ERROR, code=code, message=message, + path=location.path if location is not None else None, + line=location.line if location is not None else None, + column=location.column if location is not None else None, + location=location, help=help_text, ) result: dict[str, object] = { diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py index 568f633925..aa6062ccfd 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py @@ -29,6 +29,7 @@ from sqlbuild.lint.exceptions import LintError from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.spec.contracts.models import SourceLocation def dispatch_and_handle_errors( @@ -147,7 +148,15 @@ def _write_compile_machine_error( from sqlbuild.cli.commands._helpers.compile.output import format_compile_error_json code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) - print(format_compile_error_json(code=code, message=message, help_text=help_text)) + location: object = getattr(error, "location", None) + print( + format_compile_error_json( + code=code, + message=message, + help_text=help_text, + location=location if isinstance(location, SourceLocation) else None, + ) + ) def _write_diff_machine_error( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 5df4244686..07fa71fd0d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -79,6 +79,7 @@ from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros_result, ) +from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes from sqlbuild.compiler.compile._helpers.render.sql_vars import ( applied_interpolation, expand_authored_sql_result, @@ -94,7 +95,7 @@ MODEL_FULL_REFRESH_CONFIG_KEY, MODEL_SCHEMA_CONFIG_KEY, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroArgumentError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, CompiledSqlExpansion, @@ -106,10 +107,12 @@ DeclarationExpansionResult, DeclarationResolutionContext, DeclarationScopeResolver, + ExpansionSpan, HookExpansionResult, LoadedMacro, MacroContext, MacroExpansionResult, + MappedOffset, ModelConfigBuildRequest, ModelConfigScanCache, ModelHeaderColumnCache, @@ -735,15 +738,32 @@ def _build_model_input( ) report_native_answer(stage=NativeStage.MODEL_LOOP, kind="declaration_expansions") declaration_expanded_sql: str = declaration_expansion.sql - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, - file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=(declaration_context if context.declaration_resolver is not None else None), - consumer=model_identity, - ) + try: + macro_expansion: MacroExpansionResult = expand_sql_macros_result( + sql=declaration_expanded_sql, + file_path=model_file.file_path, + loaded_macros=loaded_macros, + macro_context=macro_context, + declaration_resolver=context.declaration_resolver, + declarations=( + declaration_context if context.declaration_resolver is not None else None + ), + consumer=model_identity, + ) + except MacroArgumentError as error: + location: SourceLocation | None = ( + _authored_model_location( + model_file=model_file, + sql=declaration_expanded_sql, + offset=error.offset, + passes=(declaration_expansion.spans,), + ) + if var_substituted_sql == model_file.query_sql + else None + ) + if location is None: + raise + raise error.located(location) from None expanded_query_sql: str = macro_expansion.sql if native_stage_enabled(NativeStage.MODEL_LOOP): report_native_fallback(site=NativeFallbackSite.CURSOR_INTRINSIC_VALIDATION) @@ -1855,3 +1875,35 @@ def validate_model_config_has_no_macros(*, values: dict[str, object]) -> None: """Reject macro calls in declarative model config while allowing hook SQL strings.""" validate_no_macros_in_config_value(value=values, path=()) + + +def _authored_model_location( + *, + model_file: DiscoveredSqlModelFile, + sql: str, + offset: int, + passes: tuple[tuple[ExpansionSpan, ...], ...], +) -> SourceLocation | None: + """Map an offset in expanded model SQL to its authored file location, if it is authored.""" + + contents: str = model_file.contents + body_start: int | None = get_model_query_start( + contents=contents, query_sql=model_file.query_sql + ) + if body_start is None and contents.lstrip().startswith(model_file.query_sql): + body_start = len(contents) - len(contents.lstrip()) + mapped: MappedOffset = map_through_passes(offset=offset, passes=passes) + authored_offset: int = (body_start or 0) + mapped.offset + if ( + body_start is None + or mapped.generated + or offset >= len(sql) + or authored_offset >= len(contents) + or contents[authored_offset] != sql[offset] + ): + return None + return SourceLocation( + path=model_file.relative_path, + line=contents.count("\n", 0, authored_offset) + 1, + column=authored_offset - (contents.rfind("\n", 0, authored_offset) + 1) + 1, + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py index 82b7cdbbc9..dce51b5f95 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py @@ -87,7 +87,10 @@ def expand_bridged_sql_macros( ) for site, call_class in zip(sites, call_classes, strict=True) ] - report_native_answer(stage=NativeStage.MACRO_CALLS, kind="bridged_calls", units=len(sites)) + mocked: int = sum(bool(state.macro_overrides.keys() & set(site.tree_names)) for site in sites) + report_native_answer( + stage=NativeStage.MACRO_CALLS, kind="bridged_calls", units=len(sites) - mocked + ) return bridge.splice(sql=sql, sites=sites, outputs=outputs) @@ -168,6 +171,8 @@ def _mocked_call_output( ) -> str: """Run a call whose tree a test mocks; its output differs from the shared memo's, so skip it.""" + report_native_fallback(site=NativeFallbackSite.MACRO_CALL_MOCKED, kind="test_mock") + macro_result: object next_index: int macro_result, next_index = _evaluate_macro_call( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index 499e8835b4..3fdbfd73b3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -33,7 +33,6 @@ ModelSchemaDeclaration, ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.model_loop.constants import ( @@ -169,9 +168,7 @@ def build_declaration_scope_resolver( if loaded_macros is not None: for macro in loaded_macros.values(): declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro - if lookup is not None: - report_native_answer(stage=NativeStage.DECLARATION_SCOPES, kind="resolver_lookups") - elif native_stage_enabled(NativeStage.DECLARATION_SCOPES): + if lookup is None and native_stage_enabled(NativeStage.DECLARATION_SCOPES): report_native_fallback(site=NativeFallbackSite.SCOPE_REBIND_LOOKUP) scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup return DeclarationScopeResolver( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py index c7c079ddbc..f6fd2b75ca 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py @@ -10,14 +10,17 @@ from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( typed_reference_value, ) -from sqlbuild.compiler.compile.constants import MACRO_ARGUMENT_ERROR_TAG -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.constants import ( + MACRO_ARGUMENT_ERROR_TAG, + MACRO_ARGUMENT_NESTED_CALL_TAG, +) +from sqlbuild.compiler.compile.exceptions import MacroArgumentError from sqlbuild.compiler.compile.models import ParsedMacroArguments from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.types import NativeStage type _ValueRow = tuple[object, ...] -type _BuildContext = tuple[Sequence[object], str] +type _BuildContext = tuple[Sequence[object], Callable[..., MacroArgumentError]] _BUILDERS: dict[str, Callable[..., object]] = { "s": lambda *, row, context: row[1], "b": lambda *, row, context: row[1], @@ -31,8 +34,8 @@ "l": lambda *, row, context: _items(row=row, context=context), "t": lambda *, row, context: tuple(_items(row=row, context=context)), "d": lambda *, row, context: _mapping(row=row, context=context), - "-": lambda *, row, context: -_number(row=row, context=context), - "+": lambda *, row, context: _number(row=row, context=context), + "-": lambda *, row, context: -_number(row=row, context=context, sign="-"), + "+": lambda *, row, context: _number(row=row, context=context, sign="+"), } @@ -47,16 +50,44 @@ def parse_macro_call_arguments( """Parse `args_source` natively; nested calls at `nested` take their `nested_values`.""" parsed: tuple[object, ...] = _native.parse_macro_arguments(args_source, list(nested)) - label: str = f"the '@{macro_name}' arguments" + + def argument_error(*, offset: int, detail: str, help_text: str) -> MacroArgumentError: + line: int = args_source.count("\n", 0, offset) + 1 + column: int = offset - (args_source.rfind("\n", 0, offset) + 1) + 1 + return MacroArgumentError( + detail=detail, + help=help_text, + macro_name=macro_name, + file_path=file_path, + offset=offset, + relative_position=(line, column), + ) + if parsed[0] == MACRO_ARGUMENT_ERROR_TAG: _, detail, help_text, line, column = parsed - raise CompileInputError( - f"Macro arguments in '{file_path}' {detail} at line {line}, column {column} of {label}", - help=cast(str, help_text), + line_start: int = sum( + len(text) + 1 for text in args_source.split("\n")[: cast(int, line) - 1] + ) + raise argument_error( + offset=line_start + cast(int, column) - 1, + detail=cast(str, detail), + help_text=cast(str, help_text), ) _, positional, keywords, references = parsed report_native_answer(stage=NativeStage.MACRO_CALLS, kind="parsed_arguments") - context: _BuildContext = (nested_values, f"'{file_path}' in {label}") + + def unary_error(*, call: int, sign: str) -> MacroArgumentError: + value: object = nested_values[call] + return argument_error( + offset=nested[call][0], + detail=( + f"apply unary {sign} to the value of a nested macro call, which is a " + f"{type(value).__name__} and not a number" + ), + help_text="Return a number from the nested macro, or apply the sign inside the macro", + ) + + context: _BuildContext = (nested_values, unary_error) return ParsedMacroArguments( args=tuple(_built(row=row, context=context) for row in cast(list[_ValueRow], positional)), kwargs={ @@ -85,11 +116,17 @@ def _mapping(*, row: _ValueRow, context: _BuildContext) -> dict[object, object]: } -def _number(*, row: _ValueRow, context: _BuildContext) -> int | float: - number: object = _built(row=cast(_ValueRow, row[1]), context=context) +def _number(*, row: _ValueRow, context: _BuildContext, sign: str) -> int | float: + operand: _ValueRow = cast(_ValueRow, row[1]) + number: object = _built(row=operand, context=context) if not isinstance(number, int | float): - raise CompileInputError( - f"Macro arguments in {context[1]} use unary + or - on a value that is not a number", - help="Apply the sign inside the macro, or pass a number literal", - ) + raise context[1](call=_nested_call(operand), sign=sign) return number + + +def _nested_call(row: _ValueRow) -> int: + """The nested call index under unary signs; only a nested call yields a non-number here.""" + + while row[0] != MACRO_ARGUMENT_NESTED_CALL_TAG: + row = cast(_ValueRow, row[1]) + return cast(int, row[1]) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index ecb3188b69..c2475554b7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -32,7 +32,11 @@ SQL_OPEN_PAREN_TOKEN, SQL_QUOTE_TOKENS, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroDeclarationLookupError +from sqlbuild.compiler.compile.exceptions import ( + CompileInputError, + MacroArgumentError, + MacroDeclarationLookupError, +) from sqlbuild.compiler.compile.models import ( DeclarationResolutionContext, DeclarationScopeResolver, @@ -1466,6 +1470,8 @@ def _evaluate_macro_call( declarations=declarations, stack=stack, ) + except MacroArgumentError as error: + raise error.shifted(opening_paren_index + 1) from None finally: approved: frozenset[SqlResourceRef] = state.facts.close_call_site() try: diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 3911006f07..0d63a9f4c6 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -26,7 +26,7 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: """Resolve merged effective vars with recursive `${name}` expansion.""" for name, value in raw_values.items(): - _reject_lone_surrogates(name=name, value=value, root=value) + _reject_lone_surrogates(name=name, value=value, path="its value") outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( raw_values, UnicodeEnvironment() ) @@ -37,23 +37,25 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: return cast(dict[str, object], outcome[0]) -def _reject_lone_surrogates(*, name: str, value: object, root: object) -> None: +def _reject_lone_surrogates(*, name: str, value: object, path: str) -> None: if isinstance(value, dict): - for item in (*value.keys(), *value.values()): - _reject_lone_surrogates(name=name, value=item, root=root) + for key, item in value.items(): + _reject_lone_surrogates(name=name, value=key, path=f"{path} key") + _reject_lone_surrogates(name=name, value=item, path=f"{path}[{key!a}]") elif isinstance(value, list | tuple): - for item in value: - _reject_lone_surrogates(name=name, value=item, root=root) + for index, item in enumerate(value): + _reject_lone_surrogates(name=name, value=item, path=f"{path}[{index}]") elif isinstance(value, str): try: _ = value.encode("utf-8") except UnicodeEncodeError as error: + byte_offset: int = len(value[: error.start].encode("utf-8")) raise CompileInputError( - f"Variable '{name}' holds the lone surrogate {value[error.start]!r}, which is not " - "valid Unicode text", + f"Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte " + f"{byte_offset}, which is not valid Unicode text", help=( - f"Set '{name}' to valid Unicode text in --vars or the [vars] table; its " - f"current value is {root!r}" + f"Set '{name}' to valid Unicode text in --vars or the [vars] table; the " + "value is not shown because it may be a secret" ), ) from None diff --git a/src/sqlbuild/compiler/compile/classes/unicode_environment.py b/src/sqlbuild/compiler/compile/classes/unicode_environment.py index f43db63af5..2b8ef3993c 100644 --- a/src/sqlbuild/compiler/compile/classes/unicode_environment.py +++ b/src/sqlbuild/compiler/compile/classes/unicode_environment.py @@ -15,12 +15,14 @@ def __getitem__(self, name: str) -> str: value: str = os.environ[name] try: _ = value.encode("utf-8") - except UnicodeEncodeError: + except UnicodeEncodeError as error: + byte_offset: int = len(value[: error.start].encode("utf-8")) raise CompileInputError( - f"Environment variable '{name}' is not valid UTF-8 text", + f"Environment variable '{name}' is not valid UTF-8 text: byte {byte_offset} " + "starts an invalid UTF-8 sequence", help=( - f"Its value is {value.encode('utf-8', 'surrogateescape')!r}; set it to UTF-8 " - "text, since SQLBuild cannot send undecodable bytes to a warehouse" + f"Set '{name}' to UTF-8 text, since SQLBuild cannot send undecodable bytes " + "to a warehouse; the value is not shown because it may be a secret" ), ) from None return value diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index be186aeb85..3780d44ad3 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -111,6 +111,7 @@ DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" MACRO_ARGUMENT_ERROR_TAG: str = "error" +MACRO_ARGUMENT_NESTED_CALL_TAG: str = "c" SQL_INTERPOLATION_TOKEN: str = "@@" TEMPLATE_OPEN_TOKEN: str = "${" diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 9d7946a898..1b22eeff02 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + from sqlbuild.spec.contracts.models import SourceLocation @@ -25,6 +27,64 @@ def __init__( self.bridge_independent: bool = bridge_independent +class MacroArgumentError(CompileInputError): + """Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.""" + + def __init__( # noqa: PLR0913 + self, + *, + detail: str, + help: str, + macro_name: str, + file_path: Path, + offset: int, + relative_position: tuple[int, int], + location: SourceLocation | None = None, + ) -> None: + line, column = ( + (location.line, location.column) if location is not None else relative_position + ) + scope: str = ( + f"of '@{macro_name}' in '{file_path}'" if location is not None else f"in '{file_path}'" + ) + suffix: str = "" if location is not None else f" of the '@{macro_name}' arguments" + super().__init__( + f"Macro arguments {scope} {detail} at line {line}, column {column}{suffix}", + help=help, + ) + self.detail: str = detail + self.macro_name: str = macro_name + self.file_path: Path = file_path + self.offset: int = offset + self.relative_position: tuple[int, int] = relative_position + self.location: SourceLocation | None = location + + def shifted(self, by: int) -> MacroArgumentError: + """The same error, its offset moved by `by` into the enclosing SQL.""" + + return MacroArgumentError( + detail=self.detail, + help=self.help or "", + macro_name=self.macro_name, + file_path=self.file_path, + offset=self.offset + by, + relative_position=self.relative_position, + ) + + def located(self, location: SourceLocation) -> MacroArgumentError: + """The same error at its authored file location.""" + + return MacroArgumentError( + detail=self.detail, + help=self.help or "", + macro_name=self.macro_name, + file_path=self.file_path, + offset=self.offset, + relative_position=self.relative_position, + location=location, + ) + + class SqlTestReferenceError(CompileInputError): """Raised when a SQL-test CTE calls a reference the test query cannot resolve.""" diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 8ae3eadf74..8bdb406acc 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -59,6 +59,7 @@ class NativeFallbackSite(StrEnum): MACRO_CALL_RESOLUTION = "macro_calls.resolution" MACRO_UNBRIDGED_EXPANSION = "macro_calls.unbridged_expansion" MACRO_BRIDGE_UNAVAILABLE = "macro_calls.bridge_unavailable" + MACRO_CALL_MOCKED = "macro_calls.mocked_evaluation" REFERENCE_SCAN = "reference_extraction.scan" CONFIG_BUILD = "model_config.build" CONFIG_HEADER_METADATA = "model_config.header_metadata" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 87863d58a5..a8734227cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -520,8 +520,8 @@ def test_given_no_proc_filesystem_when_compiling_repeatedly_then_the_store_still compile_args=("--vars", '{"regions": ["\\udcff"]}'), expected_returncodes=(1, 1, 1), expected_report_fragment=( - "Variable 'regions' holds the lone surrogate '\\udcff', which is not valid " - "Unicode text" + "Variable 'regions' holds a lone surrogate in its value[0] at UTF-8 byte 0, " + "which is not valid Unicode text" ), ), ], diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json b/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json index 72b9d3d540..095539f6f0 100644 --- a/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-bytes.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/marts/customer_totals.sql' use a bytes literal at line 1, column 1 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' use a bytes literal at line 5, column 28", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28 + }, "help": "Pass text as a string without the b prefix, for example 'orders' instead of b'orders'" } ] diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json b/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json index b7e1c02328..95d627de7c 100644 --- a/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-complex.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/marts/customer_totals.sql' use a complex number literal at line 1, column 1 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' use a complex number literal at line 5, column 28", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28 + }, "help": "Pass int or float numbers; give a complex value's real and imaginary parts as two arguments" } ] diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json b/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json index ca37c846b6..2e0374e9a9 100644 --- a/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-ellipsis.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/marts/customer_totals.sql' use '...' (Ellipsis) at line 1, column 1 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' use '...' (Ellipsis) at line 5, column 28", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 28 + }, "help": "Pass None, or a string the macro understands, instead of '...'" } ] diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-named-sequence.json b/tests/goldens/compiler/failure/engine-error-macro-argument-named-sequence.json new file mode 100644 index 0000000000..63158e9632 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-named-sequence.json @@ -0,0 +1,27 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' could not be parsed: \\N{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE} names a sequence of 2 characters, and \\N escapes name one character at line 5, column 29", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29 + }, + "help": "Write each character of the sequence with its own \\N{...} escape, or write the characters themselves" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-nested-call-sign.json b/tests/goldens/compiler/failure/engine-error-macro-argument-nested-call-sign.json new file mode 100644 index 0000000000..a2f7e6ff59 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-nested-call-sign.json @@ -0,0 +1,27 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' apply unary - to the value of a nested macro call, which is a str and not a number at line 5, column 29", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29 + }, + "help": "Return a number from the nested macro, or apply the sign inside the macro" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-second-call.json b/tests/goldens/compiler/failure/engine-error-macro-argument-second-call.json new file mode 100644 index 0000000000..3bcad05662 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-second-call.json @@ -0,0 +1,27 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' could not be parsed: a comma is missing between arguments at line 6, column 19", + "path": "models/marts/customer_totals.sql", + "line": 6, + "column": 19, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 6, + "column": 19 + }, + "help": "Separate arguments, and the items of lists, tuples and dicts, with commas, for example @cents('amount', 2)" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-surrogate-pair.json b/tests/goldens/compiler/failure/engine-error-macro-argument-surrogate-pair.json new file mode 100644 index 0000000000..50966dd72f --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-surrogate-pair.json @@ -0,0 +1,27 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' contain the lone surrogate escape '\\ud83d' at line 5, column 29", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 29 + }, + "help": "Python strings hold code points, not UTF-16 surrogate pairs; write the character as one escape: \\U0001F600 instead of the surrogate pair, or write the character itself" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json b/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json index 05bbb6a514..c52e809dcc 100644 --- a/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json +++ b/tests/goldens/compiler/failure/engine-error-macro-argument-syntax.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/marts/customer_totals.sql' could not be parsed: a value is missing here at line 1, column 10 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/marts/customer_totals.sql' could not be parsed: a value is missing here at line 5, column 37", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 37, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 37 + }, "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" } ] diff --git a/tests/goldens/compiler/failure/engine-error-non-ascii-digit-constant-value.json b/tests/goldens/compiler/failure/engine-error-non-ascii-digit-constant-value.json new file mode 100644 index 0000000000..21436b9879 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-non-ascii-digit-constant-value.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D013", + "message": "/constants/limits.sql has the bare number '٣', written with non-ASCII digits", + "help": "Quote it to keep it as text (\"٣\"), or write the number with ASCII digits 0-9" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json b/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json index 17af644d67..17e3a408b6 100644 --- a/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json +++ b/tests/goldens/compiler/failure/engine-error-undecodable-environment-variable.json @@ -8,8 +8,8 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Environment variable 'SQB_FAILURE_UNDECODABLE_REGION' is not valid UTF-8 text", - "help": "Its value is b'\\xff'; set it to UTF-8 text, since SQLBuild cannot send undecodable bytes to a warehouse" + "message": "Environment variable 'SQB_FAILURE_UNDECODABLE_REGION' is not valid UTF-8 text: byte 17 starts an invalid UTF-8 sequence", + "help": "Set 'SQB_FAILURE_UNDECODABLE_REGION' to UTF-8 text, since SQLBuild cannot send undecodable bytes to a warehouse; the value is not shown because it may be a secret" } ] } diff --git a/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json b/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json index 4f1b942ed5..53ae65edff 100644 --- a/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json +++ b/tests/goldens/compiler/failure/macro-arguments-do-not-parse.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/staging/stg_orders.sql' could not be parsed: a value is missing here at line 1, column 10 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/staging/stg_orders.sql' could not be parsed: a value is missing here at line 5, column 63", + "path": "models/staging/stg_orders.sql", + "line": 5, + "column": 63, + "location": { + "path": "models/staging/stg_orders.sql", + "line": 5, + "column": 63 + }, "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" } ] diff --git a/tests/goldens/compiler/failure/macro-non-literal-argument.json b/tests/goldens/compiler/failure/macro-non-literal-argument.json index 1be2bcd2d5..17230b1aba 100644 --- a/tests/goldens/compiler/failure/macro-non-literal-argument.json +++ b/tests/goldens/compiler/failure/macro-non-literal-argument.json @@ -8,7 +8,15 @@ "phase": "compile", "severity": "error", "code": "P001", - "message": "Macro arguments in '/models/staging/stg_orders.sql' must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references at line 1, column 1 of the '@cents' arguments", + "message": "Macro arguments of '@cents' in '/models/staging/stg_orders.sql' must use only Python literals, nested macro calls, and __ref(), __source(), or __seed() references at line 5, column 54", + "path": "models/staging/stg_orders.sql", + "line": 5, + "column": 54, + "location": { + "path": "models/staging/stg_orders.sql", + "line": 5, + "column": 54 + }, "help": "Macro arguments are Python literals (strings, numbers, True, False, None, lists, tuples and dicts), nested macro calls, and __ref(), __source() or __seed() references; compute anything else inside the macro" } ] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index cc27baeb6d..39fb797bb2 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -1164,3 +1164,13 @@ class AuthoredOffsetTestCase: contents: str marker: str expected_line_column: tuple[int, int] + + +@dataclass(frozen=True) +class UndecodableSecretTestCase: + """An env or var value holding a secret-like prefix and invalid text after it.""" + + description: str + value: object + secret: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index 815f9444e4..90e9062385 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -2,13 +2,20 @@ import pytest -from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data +from sqlbuild.compiler.compile._helpers.render.templating import ( + expand_effective_vars, + expand_template_data, +) +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandTemplateDataErrorTestCase, ExpandTemplateDataTestCase, + UndecodableSecretTestCase, ) +_SECRET: str = "warehouse-pass-7f3a" + @pytest.mark.parametrize( "test_case", @@ -216,3 +223,68 @@ def test_given_invalid_template_expressions_when_expanding_then_raises_clear_err preserve_context_tokens=test_case.preserve_context_tokens, preserve_unknown_context=test_case.preserve_unknown_context, ) + + +@pytest.mark.parametrize( + "test_case", + [ + UndecodableSecretTestCase( + description="undecodable byte after the secret", + value=f"{_SECRET}\udcff", + secret=_SECRET, + expected_message=( + "Environment variable 'SQB_ORDERS_PASSWORD' is not valid UTF-8 text: byte 19 " + "starts an invalid UTF-8 sequence" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_undecodable_env_value_when_reading_then_error_never_shows_the_value( + test_case: UndecodableSecretTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SQB_ORDERS_PASSWORD", str(test_case.value)) + + with pytest.raises(CompileInputError) as raised: + _ = UnicodeEnvironment()["SQB_ORDERS_PASSWORD"] + + assert ( + raised.value.message, + test_case.secret in f"{raised.value.message} {raised.value.help}", + ) == (test_case.expected_message, False) + + +@pytest.mark.parametrize( + "test_case", + [ + UndecodableSecretTestCase( + description="top-level text", + value=f"{_SECRET}\ud800", + secret=_SECRET, + expected_message=( + "Variable 'orders_password' holds a lone surrogate in its value at UTF-8 byte " + "19, which is not valid Unicode text" + ), + ), + UndecodableSecretTestCase( + description="nested list item", + value={"credentials": [_SECRET, f"{_SECRET}\udcff"]}, + secret=_SECRET, + expected_message=( + "Variable 'orders_password' holds a lone surrogate in its value['credentials'][1] " + "at UTF-8 byte 19, which is not valid Unicode text" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_lone_surrogate_var_when_expanding_then_error_never_shows_the_value( + test_case: UndecodableSecretTestCase, +) -> None: + with pytest.raises(CompileInputError) as raised: + expand_effective_vars({"orders_password": test_case.value}) + + assert ( + raised.value.message, + test_case.secret in f"{raised.value.message} {raised.value.help}", + ) == (test_case.expected_message, False) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 2b4e2d5744..391b18c812 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -69,8 +69,14 @@ Macro calls accept Python literal values: - `__ref("model")`, `__source("source")`, and `__seed("seed")` references, alone or inside lists and dictionaries -Bytes (`b"text"`), complex numbers (`2j`), f-strings, and `...` are rejected. An argument error -names its line and column within the call's arguments. +These inputs are rejected with an error that names the file line and column: + +- bytes (`b"text"`), complex numbers (`2j`), f-strings, and `...` +- a keyword argument passed twice, such as `@lit(a=1, a=2)`, which Python also rejects +- lone surrogate escapes such as `"\ud800"`; write a character outside the Basic Multilingual + Plane as one escape, such as `"\U0001F600"`, rather than as a surrogate pair +- `\N{...}` naming a multi-character named sequence; `\N` escapes name one character +- two values with no comma between them Positional and keyword arguments are supported: From 2390d0ff192e256809cdc8e99dda32d31cdd5ded Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:36:36 +0100 Subject: [PATCH 12/40] docs(macros): say where macro argument errors are located --- website/src/content/docs/docs/concepts/macros.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 391b18c812..5887782337 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -69,7 +69,8 @@ Macro calls accept Python literal values: - `__ref("model")`, `__source("source")`, and `__seed("seed")` references, alone or inside lists and dictionaries -These inputs are rejected with an error that names the file line and column: +These inputs are rejected with an error. In a model body the error names the file line and +column; elsewhere it names the position within the macro's arguments: - bytes (`b"text"`), complex numbers (`2j`), f-strings, and `...` - a keyword argument passed twice, such as `@lit(a=1, a=2)`, which Python also rejects From 12064cdcc0dad5ab1650134daa27e08e5ea4fe1b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:23:13 +0100 Subject: [PATCH 13/40] feat(native): scan SQL references natively under every engine; delete the Python reference scanner --- .../_helpers/sqltext/sql_references.rs | 15 +- .../bindings/_helpers/sqltext/static_sql.rs | 13 - .../src/compiler/_helpers/mod.rs | 1 - .../_helpers/sql_references/extraction.rs | 85 ----- .../compiler/_helpers/sql_references/mod.rs | 1 - .../sqlbuild-sqltext/src/compiler/main/mod.rs | 1 - .../src/compiler/main/sql_references.rs | 7 - crates/sqlbuild-sqltext/src/compiler/tests.rs | 2 - .../src/compiler/tests/helpers.rs | 49 --- .../src/compiler/tests/test_sql_references.rs | 40 --- .../src/compiler/tests/test_types.rs | 6 - crates/sqlbuild-sqltext/src/compiler/types.rs | 2 - crates/sqlbuild-sqltext/src/constants.rs | 2 - .../src/sql_references/_helpers/scan.rs | 255 ++++++-------- .../src/sql_references/constants.rs | 2 - .../main/extract_sql_references.rs | 30 +- .../src/sql_references/models.rs | 6 +- .../src/sql_references/tests/helpers.rs | 7 - .../tests/test_extract_sql_references.rs | 21 +- src/sqlbuild/_native.pyi | 4 - .../compiler/compile/_helpers/refs/native.py | 10 +- .../compile/_helpers/refs/references.py | 322 +----------------- src/sqlbuild/compiler/frontier/types.py | 1 - .../compiler/sql_analysis/constants.py | 1 + src/sqlbuild/compiler/sql_analysis/models.py | 11 + .../scripts/compiler_differential/helpers.py | 6 +- .../test_native_fallback_gate.py | 18 +- .../compile/_helpers/refs/_test_types.py | 31 +- .../compiler/compile/_helpers/refs/helpers.py | 320 +---------------- .../_helpers/refs/test_native_references.py | 171 +++------- .../frontier/test_native_fallback_recorder.py | 14 +- 31 files changed, 217 insertions(+), 1237 deletions(-) delete mode 100644 crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs delete mode 100644 crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs delete mode 100644 crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs delete mode 100644 crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index b16159e080..698eb60d99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -15,11 +15,11 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; type ReferenceRow = (&'static str, String, Option, Option); /// `(kind, call, start, message, help, corrected_call)` for one rejected call. type InvalidCallRow = (&'static str, String, usize, String, String, String); -/// The references and rejected calls, or Python's error and its start; `None` defers. -type ExtractionRow = Option<( +/// The references and rejected calls, or the scan error and its start. +type ExtractionRow = ( Option<(Vec, Vec)>, Option<(String, usize)>, -)>; +); /// Extracts references under one adapter's lexical rules, read once per syntax. #[pyclass(module = "sqlbuild._native", frozen)] @@ -36,11 +36,11 @@ impl SqlReferenceScanner { } } - /// Return `((references, rejected calls), None)`, `(None, (message, start))`, or `None`. + /// Return `((references, rejected calls), None)` or `(None, (message, start))`. fn extract(&self, sql: &str) -> PyResult { compiler_guard(|| { Ok(match extract_sql_references(sql, &self.syntax) { - ReferenceExtraction::Extracted(scan) => Some(( + ReferenceExtraction::Extracted(scan) => ( Some(( scan.references.into_iter().map(reference_row).collect(), scan.invalid_calls @@ -49,11 +49,10 @@ impl SqlReferenceScanner { .collect(), )), None, - )), + ), ReferenceExtraction::Failed(failure) => { - Some((None, Some((failure.message, failure.start)))) + (None, Some((failure.message, failure.start))) } - ReferenceExtraction::Deferred => None, }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 99a0c902f3..70621c736b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -11,7 +11,6 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; -use crate::bindings::_helpers::boundary::panics::compiler_guard; const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; @@ -189,19 +188,7 @@ fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { .collect() } -#[pyfunction] -fn extract_static_sql_references( - sql: &str, -) -> PyResult>> { - compiler_guard(|| { - Ok(sqlbuild_sqltext::compiler::main::sql_references::extract( - sql, - )) - }) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(interpolate_sql_batch, module)?)?; - module.add_function(wrap_pyfunction!(extract_static_sql_references, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs index 934c174aee..15fa52285e 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs @@ -1,4 +1,3 @@ pub mod declaration_references; pub mod model_headers; pub mod sql_interpolation; -pub mod sql_references; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs deleted file mode 100644 index e685f55c4a..0000000000 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Conservative native fast path for logical SQL reference extraction. - -use crate::compiler::types::StaticReference; -use crate::constants::{DBT_REFERENCE_KIND, TABLE_FUNCTION_REFERENCE_KIND}; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::models::QuotePolicy; - -const PREFIXES: [(&str, &str); 6] = [ - ("__dbt_ref(", "dbt_ref"), - ("__table_fn(", "table_function"), - ("__source(", "source"), - ("__seed(", "seed"), - ("__udf(", "udf"), - ("__ref(", "ref"), -]; - -pub(crate) fn extract(sql: &str) -> Option> { - let bytes = sql.as_bytes(); - let mut references: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - match non_code_end(bytes, index, QuotePolicy::COMPILER) { - Ok(Some(end)) => { - index = end; - continue; - } - Ok(None) => {} - Err(_) => return None, - } - let Some((prefix, kind)) = PREFIXES - .iter() - .find(|(prefix, _)| bytes[index..].starts_with(prefix.as_bytes())) - else { - index += 1; - continue; - }; - if *kind == TABLE_FUNCTION_REFERENCE_KIND { - return None; - } - let (first_name, first_end) = reference_name(sql, index + prefix.len())?; - let mut cursor = first_end; - let mut package = None; - let name; - let separator = ascii_whitespace_end(bytes, cursor); - if *kind == DBT_REFERENCE_KIND && bytes.get(separator).copied() == Some(b',') { - let second_start = ascii_whitespace_end(bytes, separator + 1); - let (second_name, second_end) = reference_name(sql, second_start)?; - cursor = second_end; - package = Some(first_name); - name = second_name; - } else { - name = first_name; - } - if bytes.get(cursor).copied() != Some(b')') { - return None; - } - references.push(((*kind).to_owned(), name, package, None)); - index = cursor + 1; - } - Some(references) -} - -/// Read one bare `"name"` argument, the only form compile replacement accepts. -fn reference_name(sql: &str, start: usize) -> Option<(String, usize)> { - let bytes = sql.as_bytes(); - if bytes.get(start).copied() != Some(b'"') { - return None; - } - let close = start + 1 + bytes[start + 1..].iter().position(|value| *value == b'"')?; - if close == start + 1 { - return None; - } - Some((sql[start + 1..close].to_owned(), close + 1)) -} - -fn ascii_whitespace_end(bytes: &[u8], start: usize) -> usize { - let mut end = start; - while bytes - .get(end) - .is_some_and(|value| value.is_ascii_whitespace()) - { - end += 1; - } - end -} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs deleted file mode 100644 index ae851a60d0..0000000000 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod extraction; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index f5e0b71fd8..f488f56683 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -6,5 +6,4 @@ pub mod model_header_parsing; pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod sql_interpolation; -pub mod sql_references; pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs deleted file mode 100644 index d48a41008c..0000000000 --- a/crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Public compiler entry point for conservative logical SQL references. - -use crate::compiler::types::StaticReference; - -pub fn extract(sql: &str) -> Option> { - crate::compiler::_helpers::sql_references::extraction::extract(sql) -} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests.rs b/crates/sqlbuild-sqltext/src/compiler/tests.rs index 759eb2c611..7744d6b316 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests.rs @@ -10,7 +10,5 @@ mod model_header_nesting; mod model_header_tokenization; #[path = "tests/test_sql_interpolation.rs"] mod sql_interpolation; -#[path = "tests/test_sql_references.rs"] -mod sql_references; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index f97b85c131..8822423b6e 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -1,7 +1,6 @@ use crate::compiler::_helpers::model_headers::tokenization::{ MAX_TOKENIZER_WORKERS, TOKENIZER_WORKER_STACK_BYTES, build_tokenizer_pool, parse_batch, }; -use crate::compiler::_helpers::sql_references::extraction::extract; use crate::compiler::main::declaration_references::scan_declaration_references; use crate::compiler::main::model_header_single_parsing::parse_one; use crate::compiler::main::sql_interpolation::interpolate_sql; @@ -15,54 +14,6 @@ use sqlbuild_core::text::main::python_text::python_text; use std::collections::BTreeMap; use std::thread; -pub(crate) fn simple_references_preserve_authored_order() -> bool { - extract( - "SELECT * FROM __source(\"orders\") UNION ALL SELECT * FROM __dbt_ref(\"shop\" , \"customers\")", - ) == Some(vec![ - ("source".to_owned(), "orders".to_owned(), None, None), - ( - "dbt_ref".to_owned(), - "customers".to_owned(), - Some("shop".to_owned()), - None, - ), - ]) -} - -pub(crate) fn comments_and_quoted_text_hide_references() -> bool { - extract("-- __ref(\"ignored\")\nSELECT '__seed(\"also_ignored\")' FROM __ref(\"orders\")") - == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) -} - -pub(crate) fn dollar_quoted_text_hides_references() -> bool { - extract(concat!( - "SELECT $$Customer's order -- __ref(\"ignored\") $5$$ AS label, ", - "$tag$ $$ __seed(\"also_ignored\") $tag$ AS note, price$1$ ", - "FROM __ref(\"orders\")" - )) == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) -} - -pub(crate) fn complex_or_malformed_sql_requests_fallback() -> bool { - [ - "SELECT * FROM __table_fn(\"orders\")(1)", - "SELECT * FROM __ref(concat('ord', 'ers'))", - "SELECT * FROM __ref(\"orders\"", - "SELECT * FROM __ref(\"orders\") /* unterminated", - "SELECT * FROM __ref(\"orders\") WHERE note = 'unterminated", - "SELECT * FROM __ref(\"orders\") WHERE note = $$unterminated", - "SELECT * FROM __ref(örders)", - "SELECT * FROM __ref(orders)", - "SELECT * FROM __ref('orders')", - "SELECT * FROM __ref( \"orders\" )", - "SELECT * FROM __ref(/* upstream */ \"orders\")", - "SELECT * FROM __ref(\"\")", - "SELECT * FROM __ref(\"ord\"\"ers\")", - "SELECT * FROM __dbt_ref(\"shop\", \"customers\" )", - ] - .into_iter() - .all(|sql| extract(sql).is_none()) -} - pub(crate) fn nested_authored_headers_preserve_values_and_offsets() -> bool { let headers = vec![ " columns (café (type DECIMAL(10,2))), enabled true".to_owned(), diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs deleted file mode 100644 index 50ecf02ba2..0000000000 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::compiler::tests::helpers::{ - comments_and_quoted_text_hide_references, complex_or_malformed_sql_requests_fallback, - dollar_quoted_text_hides_references, simple_references_preserve_authored_order, -}; -use crate::compiler::tests::test_types::StaticSqlOperationTestCase; - -#[test] -fn given_sql_reference_cases_when_extracting_then_expected_behavior_holds() { - let test_cases = [ - StaticSqlOperationTestCase { - description: "simple references preserve authored order", - run: simple_references_preserve_authored_order, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "comments and quoted text hide embedded references", - run: comments_and_quoted_text_hide_references, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "dollar-quoted text hides embedded references", - run: dollar_quoted_text_hides_references, - expected_success: true, - }, - StaticSqlOperationTestCase { - description: "complex and malformed SQL requests Python fallback", - run: complex_or_malformed_sql_requests_fallback, - expected_success: true, - }, - ]; - - for test_case in test_cases { - let actual_success = (test_case.run)(); - assert_eq!( - actual_success, test_case.expected_success, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index d63a9008c0..043592355b 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -8,12 +8,6 @@ pub(crate) struct ModelHeaderTokenizationTestCase { pub(crate) expected_success: bool, } -pub(crate) struct StaticSqlOperationTestCase { - pub(crate) description: &'static str, - pub(crate) run: fn() -> bool, - pub(crate) expected_success: bool, -} - pub(crate) struct ModelHeaderMatchTestCase { pub(crate) description: &'static str, pub(crate) contents: &'static str, diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index 545ba2ca48..a16c3efdff 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1,5 +1,3 @@ -pub type StaticReference = (String, String, Option, Option); - /// One substitution in code points: `(source start, source end, output start, output end)`. pub type CharSpan = (usize, usize, usize, usize); diff --git a/crates/sqlbuild-sqltext/src/constants.rs b/crates/sqlbuild-sqltext/src/constants.rs index 76a1c686d9..f1e65ca5bb 100644 --- a/crates/sqlbuild-sqltext/src/constants.rs +++ b/crates/sqlbuild-sqltext/src/constants.rs @@ -1,11 +1,9 @@ pub(crate) const COLUMNS_KEY: &str = "columns"; pub(crate) const CONSTANT_CALL: &str = "constant"; -pub(crate) const DBT_REFERENCE_KIND: &str = "dbt_ref"; pub(crate) const INLINE_SQL_HOOK: &str = "inline_sql"; /// The deepest authored header value nesting; YAML and JSON readers bound theirs the same way. pub(crate) const MAX_HEADER_NESTING_DEPTH: usize = 256; pub(crate) const OUTSIDE_KEY: &str = "outside"; pub(crate) const SQL_HOOK: &str = "sql"; -pub(crate) const TABLE_FUNCTION_REFERENCE_KIND: &str = "table_function"; pub(crate) const THRESHOLDS_KEY: &str = "thresholds"; pub(crate) const TYPE_KEY: &str = "type"; diff --git a/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs b/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs index 733926471e..181d352203 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs @@ -12,24 +12,11 @@ use crate::sql_references::types::ReferencePrefix; use crate::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use crate::sql_scan::models::{LexicalSyntax, Unclosed}; -/// Why the scan stopped before reaching the end of the SQL. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum Stop { - /// Python raises `CompileInputError` with this message. - Failed(String), - /// A character or construct whose classification only Python reproduces. - Deferred, -} - -type Scan = Result; +/// The `CompileInputError` message that stopped the scan. +type Scan = Result; -/// Why the scan of one text stopped, with the byte offset Python's error points at. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum Stopped { - /// Python raises this message for the quote or comment, or the call, starting at the offset. - Failed(String, usize), - Deferred, -} +/// The scan error and the byte offset of the quote or comment, or the call, it points at. +pub(crate) type Stopped = (String, usize); const NON_CODE_START: [bool; 256] = byte_table(NON_CODE_START_BYTES, false); const REFERENCE_SCAN_START: [bool; 256] = byte_table(NON_CODE_START_BYTES, true); @@ -54,15 +41,16 @@ enum Parsed { /// Return every reference and rejected call in `sql` in authored order. pub(crate) fn scan_references( - sql: &[u8], + text: &str, syntax: &LexicalSyntax, ) -> Result { + let sql: &[u8] = text.as_bytes(); let mut scan: ReferenceScan = ReferenceScan::default(); let mut offsets: CharOffsets = CharOffsets::default(); let mut index = 0; while index < sql.len() { - if let Some(end) = non_code_end(sql, index, syntax, REFERENCE_CONTEXT) - .map_err(|stop| stopped_at(stop, index))? + if let Some(end) = + non_code_end(sql, index, syntax, REFERENCE_CONTEXT).map_err(|stop| (stop, index))? { index = end; continue; @@ -82,13 +70,11 @@ pub(crate) fn scan_references( continue; }; let (parsed, next) = - parse_reference(sql, index, call, syntax).map_err(|stop| stopped_at(stop, index))?; + parse_reference(text, index, call, syntax).map_err(|stop| (stop, index))?; match parsed { Parsed::Reference(reference) => scan.references.push(reference), Parsed::Invalid(invalid_call) => scan.invalid_calls.push(InvalidReferenceCall { - start: offsets - .advance(sql, index) - .map_err(|stop| stopped_at(stop, index))?, + start: offsets.advance(text, index), ..invalid_call }), } @@ -99,33 +85,34 @@ pub(crate) fn scan_references( /// Parse the reference call at `start` as Python `_parse_reference_at` does. fn parse_reference( - sql: &[u8], + text: &str, start: usize, call: &ReferencePrefix, syntax: &LexicalSyntax, ) -> Scan<(Parsed, usize)> { + let sql: &[u8] = text.as_bytes(); let (prefix, kind) = *call; let open = start + prefix.len() - 1; let close = matching_paren(sql, open, syntax, REFERENCE_CONTEXT)?; - let arguments = &sql[open + 1..close]; - let call_text = &sql[start..=close]; + let arguments: &str = &text[open + 1..close]; + let call_text: &str = &text[start..=close]; let names = if kind == DBT_REFERENCE_KIND { - dbt_reference_names(arguments)? + dbt_reference_names(arguments) } else { - bare_name(arguments)?.map(|name| (name, None)) + bare_name(arguments).map(|name| (name, None)) }; let Some((name, package)) = names else { - let invalid = invalid_arguments(kind, sql, (start, open, close), syntax)?; + let invalid = invalid_arguments(kind, text, (start, open, close), syntax); return Ok((Parsed::Invalid(invalid), close + 1)); }; let mut call_argument_count = None; if kind == TABLE_FUNCTION_REFERENCE_KIND { - let suffix = python_whitespace_end(sql, close + 1)?; + let suffix = python_whitespace_end(text, close + 1); if sql.get(suffix) != Some(&b'(') { let corrected_call = format!("{}()", example_call(kind, &[name.as_str()])); let invalid = InvalidReferenceCall { kind, - call: utf8(call_text)?, + call: call_text.to_owned(), start: 0, message: format!( "{} must be followed by an argument list", @@ -140,7 +127,7 @@ fn parse_reference( return Ok((Parsed::Invalid(invalid), close + 1)); } let suffix_close = matching_paren(sql, suffix, syntax, TABLE_FUNCTION_CALL_CONTEXT)?; - call_argument_count = Some(split_arguments(sql, suffix + 1, suffix_close, syntax)?.len()); + call_argument_count = Some(split_arguments(text, suffix + 1, suffix_close, syntax)?.len()); } Ok(( Parsed::Reference(SqlReference { @@ -156,16 +143,16 @@ fn parse_reference( /// Python `_invalid_reference_arguments`: the P012 message, help and corrected call. fn invalid_arguments( kind: &'static str, - sql: &[u8], + text: &str, (start, open, close): (usize, usize, usize), syntax: &LexicalSyntax, -) -> Scan { +) -> InvalidReferenceCall { let allowed: &[usize] = if kind == DBT_REFERENCE_KIND { &[1, 2] } else { &[1] }; - let authored: Option> = authored_names(sql, open + 1, close, syntax)? + let authored: Option> = authored_names(text, open + 1, close, syntax) .filter(|names| allowed.contains(&names.len())); let names: Vec = authored.unwrap_or_else(|| { PLACEHOLDER_NAMES @@ -186,13 +173,13 @@ fn invalid_arguments( } else { "exactly one double-quoted name" }; - let call = utf8(&sql[start..=close])?; + let call: String = text[start..=close].to_owned(); let words: Vec<&str> = call .split(is_python_space) .filter(|word| !word.is_empty()) .collect(); let prefix = function_name(kind); - Ok(InvalidReferenceCall { + InvalidReferenceCall { kind, message: format!("{} is not a valid {prefix}() call", words.join(" ")), help: format!( @@ -202,39 +189,34 @@ fn invalid_arguments( corrected_call, call, start: 0, - }) + } } /// Python `_authored_reference_names`: each argument as a plain or quoted name, else `None`. fn authored_names( - sql: &[u8], + text: &str, start: usize, end: usize, syntax: &LexicalSyntax, -) -> Scan>> { - let split = match split_arguments(sql, start, end, syntax) { - Ok(split) => split, - Err(Stop::Failed(_)) => return Ok(None), - Err(Stop::Deferred) => return Err(Stop::Deferred), - }; - let mut names: Vec = Vec::with_capacity(split.len()); - for argument in split { - let name: &[u8] = match (argument.first(), argument.last()) { - (Some(first), Some(last)) - if argument.len() >= QUOTED_NAME_MINIMUM_BYTES - && first == last - && NAME_QUOTE_BYTES.contains(first) => - { - &argument[1..argument.len() - 1] - } - _ => &argument, - }; - if !is_authored_name(name) { - return Ok(None); - } - names.push(utf8(name)?); - } - Ok(Some(names)) +) -> Option> { + split_arguments(text, start, end, syntax) + .ok()? + .iter() + .map(|argument| { + let bytes: &[u8] = argument.as_bytes(); + let name: &str = match (bytes.first(), bytes.last()) { + (Some(first), Some(last)) + if bytes.len() >= QUOTED_NAME_MINIMUM_BYTES + && first == last + && NAME_QUOTE_BYTES.contains(first) => + { + &argument[1..argument.len() - 1] + } + _ => argument, + }; + is_authored_name(name.as_bytes()).then(|| name.to_owned()) + }) + .collect() } /// Python `[A-Za-z_][A-Za-z0-9_.]*` matched against the whole name. @@ -261,32 +243,31 @@ fn example_call(kind: &str, names: &[&str]) -> String { } /// Python `"([^"]+)"` matched against the whole argument text; `None` when it does not match. -fn bare_name(arguments: &[u8]) -> Scan> { - match quoted_name_end(arguments, 0) { - Some(end) if end == arguments.len() => utf8(&arguments[1..end - 1]).map(Some), - _ => Ok(None), +fn bare_name(arguments: &str) -> Option { + match quoted_name_end(arguments.as_bytes(), 0) { + Some(end) if end == arguments.len() => Some(arguments[1..end - 1].to_owned()), + _ => None, } } /// Python `"([^"]+)"(?:\s*,\s*"([^"]+)")?` over the whole text: name and package, or `None`. -fn dbt_reference_names(arguments: &[u8]) -> Scan)>> { - let Some(first_end) = quoted_name_end(arguments, 0) else { - return Ok(None); - }; +fn dbt_reference_names(arguments: &str) -> Option<(String, Option)> { + let bytes: &[u8] = arguments.as_bytes(); + let first_end = quoted_name_end(bytes, 0)?; + let first: String = arguments[1..first_end - 1].to_owned(); if first_end == arguments.len() { - return Ok(Some((utf8(&arguments[1..first_end - 1])?, None))); + return Some((first, None)); } - let separator = python_whitespace_end(arguments, first_end)?; - if arguments.get(separator) != Some(&b',') { - return Ok(None); + let separator = python_whitespace_end(arguments, first_end); + if bytes.get(separator) != Some(&b',') { + return None; } - let second_start = python_whitespace_end(arguments, separator + 1)?; - match quoted_name_end(arguments, second_start) { - Some(end) if end == arguments.len() => Ok(Some(( - utf8(&arguments[second_start + 1..end - 1])?, - Some(utf8(&arguments[1..first_end - 1])?), - ))), - _ => Ok(None), + let second_start = python_whitespace_end(arguments, separator + 1); + match quoted_name_end(bytes, second_start) { + Some(end) if end == arguments.len() => { + Some((arguments[second_start + 1..end - 1].to_owned(), Some(first))) + } + _ => None, } } @@ -304,26 +285,27 @@ fn quoted_name_end(arguments: &[u8], start: usize) -> Option { } /// Python `_split_top_level_arguments` over `sql[start..end]`: comments read as one space. +/// +/// The walk matches `matching_paren`'s from the same start, so no quote or comment it skips can +/// run past `end`. fn split_arguments( - sql: &[u8], + text: &str, start: usize, end: usize, syntax: &LexicalSyntax, -) -> Scan>> { - let mut arguments: Vec> = Vec::new(); - let mut current: Vec = Vec::new(); +) -> Scan> { + let sql: &[u8] = text.as_bytes(); + let mut arguments: Vec = Vec::new(); + let mut current: String = String::new(); let mut depth = 0isize; let mut saw_separator = false; let mut index = start; while index < end { if let Some(non_code) = non_code_end(sql, index, syntax, REFERENCE_CONTEXT)? { - if non_code > end { - return Err(Stop::Deferred); - } if QUOTE_BYTES.contains(&sql[index]) { - current.extend_from_slice(&sql[index..non_code]); + current.push_str(&text[index..non_code]); } else { - current.push(b' '); + current.push(' '); } index = non_code; continue; @@ -332,13 +314,7 @@ fn split_arguments( b'(' => depth += 1, b')' => depth -= 1, b',' if depth == 0 => { - let argument = python_strip(¤t)?; - if argument.is_empty() { - return Err(failed(format!( - "{REFERENCE_CONTEXT} contains an empty argument" - ))); - } - arguments.push(argument.to_vec()); + arguments.push(stripped_argument(¤t)?); current.clear(); saw_separator = true; index += 1; @@ -346,48 +322,38 @@ fn split_arguments( } _ => {} } - current.push(sql[index]); - index += 1; + let character: char = text[index..].chars().next().unwrap_or_default(); + current.push(character); + index += character.len_utf8(); } - let argument = python_strip(¤t)?; + let argument: &str = current.trim_matches(is_python_space); if !argument.is_empty() { - arguments.push(argument.to_vec()); + arguments.push(argument.to_owned()); } else if saw_separator { - return Err(failed(format!( - "{REFERENCE_CONTEXT} contains an empty argument" - ))); + return Err(empty_argument()); } Ok(arguments) } -/// Python `str.strip()`, which strips `str.isspace()` characters. -fn python_strip(value: &[u8]) -> Scan<&[u8]> { - let text: &str = std::str::from_utf8(value).map_err(|_| Stop::Deferred)?; - Ok(text.trim_matches(is_python_space).as_bytes()) +/// Python `str.strip()` of one argument, which must not be empty. +fn stripped_argument(argument: &str) -> Scan { + let stripped: &str = argument.trim_matches(is_python_space); + if stripped.is_empty() { + return Err(empty_argument()); + } + Ok(stripped.to_owned()) } -/// Python `_skip_whitespace`, which skips `str.isspace()` characters. -fn python_whitespace_end(sql: &[u8], start: usize) -> Scan { - let mut index = start; - while index < sql.len() { - let character: char = char_at(sql, index)?; - if !is_python_space(character) { - break; - } - index += character.len_utf8(); - } - Ok(index) +fn empty_argument() -> String { + format!("{REFERENCE_CONTEXT} contains an empty argument") } -/// The character starting at byte `index` of UTF-8 text. -fn char_at(text: &[u8], index: usize) -> Scan { - let width: usize = (text[index].leading_ones() as usize).max(1); - let encoded: &[u8] = text.get(index..index + width).ok_or(Stop::Deferred)?; - std::str::from_utf8(encoded) - .map_err(|_| Stop::Deferred)? - .chars() - .next() - .ok_or(Stop::Deferred) +/// Python `_skip_whitespace`, which skips `str.isspace()` characters. +fn python_whitespace_end(text: &str, start: usize) -> usize { + text[start..] + .char_indices() + .find(|(_, character)| !is_python_space(*character)) + .map_or(text.len(), |(offset, _)| start + offset) } fn non_code_end( @@ -427,21 +393,13 @@ fn matching_paren(sql: &[u8], open: usize, syntax: &LexicalSyntax, context: &str Err(unclosed_error(context, Unclosed::Parenthesis)) } -fn unclosed_error(context: &str, unclosed: Unclosed) -> Stop { +fn unclosed_error(context: &str, unclosed: Unclosed) -> String { let construct = match unclosed { Unclosed::BlockComment => "block comment", Unclosed::Quote => "quoted string", Unclosed::Parenthesis => "parenthesis", }; - failed(format!("{context} contains an unclosed {construct}")) -} - -fn utf8(value: &[u8]) -> Scan { - String::from_utf8(value.to_vec()).map_err(|_| Stop::Deferred) -} - -fn failed(message: String) -> Stop { - Stop::Failed(message) + format!("{context} contains an unclosed {construct}") } /// Converts increasing byte offsets of UTF-8 text into Python code-point offsets. @@ -452,20 +410,9 @@ struct CharOffsets { } impl CharOffsets { - fn advance(&mut self, text: &[u8], byte: usize) -> Scan { - self.char += std::str::from_utf8(&text[self.byte..byte]) - .map_err(|_| Stop::Deferred)? - .chars() - .count(); + fn advance(&mut self, text: &str, byte: usize) -> usize { + self.char += text[self.byte..byte].chars().count(); self.byte = byte; - Ok(self.char) - } -} - -/// Point a stop at `start`: the top-level quote or comment, or the call being parsed. -fn stopped_at(stop: Stop, start: usize) -> Stopped { - match stop { - Stop::Failed(message) => Stopped::Failed(message, start), - Stop::Deferred => Stopped::Deferred, + self.char } } diff --git a/crates/sqlbuild-sqltext/src/sql_references/constants.rs b/crates/sqlbuild-sqltext/src/sql_references/constants.rs index 5bd768c96e..c0812999ab 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/constants.rs @@ -15,8 +15,6 @@ pub(crate) const DBT_REFERENCE_KIND: &str = "dbt_ref"; pub(crate) const TABLE_FUNCTION_REFERENCE_KIND: &str = "table_fn"; pub(crate) const REFERENCE_CONTEXT: &str = "SQL reference"; pub(crate) const TABLE_FUNCTION_CALL_CONTEXT: &str = "SQL table function call"; -/// Line comment prefixes Python's reference and parenthesis scans both stop at. -pub(crate) const SUPPORTED_LINE_COMMENT_PREFIXES: [&str; 3] = ["--", "//", "#"]; /// Characters that may start a comment or quoted text under a supported syntax. pub(crate) const NON_CODE_START_BYTES: &[u8] = b"-/#'\"`$"; /// Non-code openers whose text an argument keeps; comments become one space. diff --git a/crates/sqlbuild-sqltext/src/sql_references/main/extract_sql_references.rs b/crates/sqlbuild-sqltext/src/sql_references/main/extract_sql_references.rs index 5bcc0c0c80..8376cf085c 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/main/extract_sql_references.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/main/extract_sql_references.rs @@ -1,30 +1,16 @@ -//! Logical SQL references under one adapter's lexical rules, or a deferral to Python. +//! Logical SQL references under one adapter's lexical rules. -use crate::sql_references::_helpers::scan::{Stopped, scan_references}; -use crate::sql_references::constants::SUPPORTED_LINE_COMMENT_PREFIXES; +use crate::sql_references::_helpers::scan::scan_references; use crate::sql_references::models::{ReferenceExtraction, ReferenceScanFailure}; use crate::sql_scan::models::LexicalSyntax; -/// Return the references and rejected calls Python finds in `sql`, or its error message. +/// Return the references and rejected calls in `sql`, or the error that stops the scan. pub fn extract_sql_references(sql: &str, syntax: &LexicalSyntax) -> ReferenceExtraction { - if !syntax - .line_comment_prefixes - .iter() - .all(|prefix| SUPPORTED_LINE_COMMENT_PREFIXES.contains(&prefix.as_str())) - { - return ReferenceExtraction::Deferred; - } - match scan_references(sql.as_bytes(), syntax) { + match scan_references(sql, syntax) { Ok(scan) => ReferenceExtraction::Extracted(scan), - Err(Stopped::Failed(message, start)) => { - sql.get(..start) - .map_or(ReferenceExtraction::Deferred, |prefix| { - ReferenceExtraction::Failed(ReferenceScanFailure { - message, - start: prefix.chars().count(), - }) - }) - } - Err(Stopped::Deferred) => ReferenceExtraction::Deferred, + Err((message, start)) => ReferenceExtraction::Failed(ReferenceScanFailure { + message, + start: sql[..start].chars().count(), + }), } } diff --git a/crates/sqlbuild-sqltext/src/sql_references/models.rs b/crates/sqlbuild-sqltext/src/sql_references/models.rs index 72acbeb3ce..c6029db8d5 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/models.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/models.rs @@ -39,13 +39,11 @@ pub struct ReferenceScanFailure { pub start: usize, } -/// The references of one SQL text, Python's error for it, or a deferral to Python. +/// The references of one SQL text, or the error that stops the scan. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ReferenceExtraction { /// Every reference and rejected call in authored order, exactly as Python scans them. Extracted(ReferenceScan), - /// The `CompileInputError` Python raises for this text. + /// The `CompileInputError` raised for this text. Failed(ReferenceScanFailure), - /// A character or syntax rule the scan cannot classify exactly as Python does. - Deferred, } diff --git a/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs index 794647a5be..75f366d800 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs @@ -21,13 +21,6 @@ pub(crate) fn backslash_hash_comments() -> LexicalSyntax { } } -pub(crate) fn unsupported_comments() -> LexicalSyntax { - LexicalSyntax { - line_comment_prefixes: vec!["--".to_string(), "%".to_string()], - ..LexicalSyntax::default() - } -} - pub(crate) fn reference(kind: &'static str, name: &str) -> SqlReference { SqlReference { kind, diff --git a/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs b/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs index 2abecb7591..33bcb61ae8 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs @@ -1,8 +1,7 @@ use crate::sql_references::main::extract_sql_references::extract_sql_references; -use crate::sql_references::models::ReferenceExtraction; use crate::sql_references::tests::helpers::{ backslash_hash_comments, dbt_reference, extracted, failed, generic, invalid_call, reference, - rejected, table_function, unsupported_comments, + rejected, table_function, }; use crate::sql_references::tests::test_types::ExtractSqlReferencesTestCase; @@ -362,10 +361,20 @@ fn given_sql_when_extracting_references_then_matches_python_scanner() { expected_extraction: extracted(vec![table_function("f", 1)]), }, ExtractSqlReferencesTestCase { - description: "unsupported line comment prefix defers", - sql: "SELECT * FROM __ref(\"a\")", - syntax: unsupported_comments(), - expected_extraction: ReferenceExtraction::Deferred, + description: "non-ASCII text inside and around a rejected call keeps its characters", + sql: "SELECT 'é' FROM __ref(größe, \u{3000}\"b\")", + syntax: generic(), + expected_extraction: rejected( + vec![], + vec![invalid_call( + "ref", + "__ref(größe, \u{3000}\"b\")", + 16, + "__ref(größe, \"b\") is not a valid __ref() call", + "__ref() takes exactly one double-quoted name, with no comments or extra spaces inside the parentheses: __ref(\"model_name\")", + "__ref(\"model_name\")", + )], + ), }, ]; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 852ab2c0b7..aa85d3549e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -236,7 +236,6 @@ class SqlReferenceScanner: None, ] | tuple[None, tuple[str, int]] - | None ): ... def scope_expected_model_names( @@ -316,9 +315,6 @@ def interpolate_sql_batch( ) -> list[ tuple[str | None, list[tuple[int, int, int, int]], list[tuple[str, str]], str | None] ]: ... -def extract_static_sql_references( - sql: str, -) -> list[tuple[str, str, str | None, int | None]] | None: ... def scan_macro_call_sites( sql: str, python_version: tuple[int, int], unicode_version: str ) -> list[tuple[int, int, str, list[str], bool]] | None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/native.py b/src/sqlbuild/compiler/compile/_helpers/refs/native.py index febe9223ac..bcb6cb5038 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/native.py @@ -3,6 +3,7 @@ from __future__ import annotations from functools import cache +from typing import cast import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( @@ -23,21 +24,18 @@ None, ] | tuple[None, tuple[str, int]] - | None ) def extract_native_sql_references( *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure | None: - """Return the references and rejected calls, Python's located error, or None for Python.""" +) -> SqlReferenceScan | SqlReferenceScanFailure: + """Return the references and rejected calls, or the scan error and its start.""" extraction: _NativeExtraction = _scanner(syntax).extract(sql) - if extraction is None: - return None scanned, failure = extraction if scanned is None: - return failure + return cast(SqlReferenceScanFailure, failure) references, invalid_calls = scanned return SqlReferenceScan( references=tuple( diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/references.py b/src/sqlbuild/compiler/compile/_helpers/refs/references.py index 1d362798ee..5c41378829 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/references.py @@ -2,20 +2,13 @@ from __future__ import annotations -import re from pathlib import Path -import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes from sqlbuild.compiler.compile.constants import ( REFERENCE_CALL_SYNTAX_CODE, - SQL_ARGUMENT_SEPARATOR_TOKEN, - SQL_CLOSE_PAREN_TOKEN, - SQL_OPEN_PAREN_TOKEN, - SQL_QUOTE_TOKENS, - SQL_REFERENCE_NAME_QUOTE_TOKENS, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( @@ -33,43 +26,11 @@ DiagnosticSeverity, SqlReferenceScanFailure, ) -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer -from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SourceLocation -_CONTEXT: str = "SQL reference" -_PAIRED_QUOTE_CHARACTER_COUNT: int = 2 -_DBT_REF_PACKAGE_ARGUMENT_COUNT: int = 2 -_NAME_ARGUMENT_PATTERN: re.Pattern[str] = re.compile(r'"([^"]+)"') -_DBT_REF_ARGUMENTS_PATTERN: re.Pattern[str] = re.compile(r'"([^"]+)"(?:\s*,\s*"([^"]+)")?') -_AUTHORED_NAME_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_.]*") -_PLACEHOLDER_NAMES: dict[SqlReferenceKind, tuple[str, ...]] = { - SqlReferenceKind.REF: ("model_name",), - SqlReferenceKind.SOURCE: ("source_name",), - SqlReferenceKind.SEED: ("seed_name",), - SqlReferenceKind.UDF: ("function_name",), - SqlReferenceKind.TABLE_FUNCTION: ("function_name",), - SqlReferenceKind.DBT_REF: ("package_name", "model_name"), -} -_REFERENCE_PREFIXES: tuple[tuple[str, SqlReferenceKind], ...] = ( - ("__dbt_ref(", SqlReferenceKind.DBT_REF), - ("__table_fn(", SqlReferenceKind.TABLE_FUNCTION), - ("__source(", SqlReferenceKind.SOURCE), - ("__seed(", SqlReferenceKind.SEED), - ("__udf(", SqlReferenceKind.UDF), - ("__ref(", SqlReferenceKind.REF), -) -_REFERENCE_PREFIX_BY_KIND: dict[SqlReferenceKind, str] = { - ref_kind: prefix[:-1] for prefix, ref_kind in _REFERENCE_PREFIXES -} -_REFERENCE_SCAN_PATTERN: re.Pattern[str] = re.compile(r"__|--|/\*|//|#|'|\"|`|\$") - def extract_sql_references( *, sql: str, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin | None = None @@ -97,33 +58,11 @@ def scan_sql_reference_calls( def _reference_scan_outcome( *, sql: str, syntax: SqlLexicalSyntax ) -> SqlReferenceScan | SqlReferenceScanFailure: - if native_stage_enabled(NativeStage.REFERENCE_EXTRACTION): - native: SqlReferenceScan | SqlReferenceScanFailure | None = extract_native_sql_references( - sql=sql, syntax=syntax - ) - if native is not None: - report_native_answer(stage=NativeStage.REFERENCE_EXTRACTION, kind="reference_scans") - return native - report_native_fallback(site=NativeFallbackSite.REFERENCE_SCAN) - return python_reference_scan(sql=sql, syntax=syntax) - if syntax.reads_differently_from_generic(sql): - return python_reference_scan(sql=sql, syntax=syntax) - native_references: list[tuple[str, str, str | None, int | None]] | None = ( - _native.extract_static_sql_references(sql) + outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( + sql=sql, syntax=syntax ) - if native_references is not None: - return SqlReferenceScan( - references=tuple( - CompileSqlReference( - ref_kind=SqlReferenceKind(kind), - ref_name=name, - ref_package=package, - call_argument_count=call_argument_count, - ) - for kind, name, package, call_argument_count in native_references - ) - ) - return python_reference_scan(sql=sql, syntax=syntax) + report_native_answer(stage=NativeStage.REFERENCE_EXTRACTION, kind="reference_scans") + return outcome def _located_scan_error( @@ -301,254 +240,3 @@ def reference_call_location(*, path: Path, text: str, start: int, call: str) -> end_line=text.count("\n", 0, end) + 1, end_column=end - (text.rfind("\n", 0, end) + 1) + 1, ) - - -def python_reference_scan( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - """Scan with the Python scanner; an error carries the start of its quote, comment or call.""" - - references: list[CompileSqlReference] = [] - invalid_calls: list[InvalidSqlReferenceCall] = [] - index: int = 0 - length: int = len(sql) - while index < length: - index = _next_reference_scan_position(sql=sql, start=index) - if index >= length: - break - try: - non_code_end: int | None = dialect_non_code_end( - sql=sql, start=index, syntax=syntax, context=_CONTEXT - ) - except CompileInputError as error: - return (error.message, index) - if non_code_end is not None: - index = non_code_end - continue - - try: - parsed: tuple[CompileSqlReference | InvalidSqlReferenceCall, int] | None = ( - _parse_reference_at(sql=sql, start=index, syntax=syntax) - ) - except CompileInputError as error: - return (error.message, index) - if parsed is None: - index += 1 - continue - if isinstance(parsed[0], InvalidSqlReferenceCall): - invalid_calls.append(parsed[0]) - else: - references.append(parsed[0]) - index = parsed[1] - return SqlReferenceScan(references=tuple(references), invalid_calls=tuple(invalid_calls)) - - -def _next_reference_scan_position(*, sql: str, start: int) -> int: - match: re.Match[str] | None = _REFERENCE_SCAN_PATTERN.search(sql, start) - return match.start() if match is not None else len(sql) - - -def _parse_reference_at( - *, sql: str, start: int, syntax: SqlLexicalSyntax -) -> tuple[CompileSqlReference | InvalidSqlReferenceCall, int] | None: - ref_kind: SqlReferenceKind | None = None - prefix: str - for prefix, candidate_kind in _REFERENCE_PREFIXES: - if sql.startswith(prefix, start): - ref_kind = candidate_kind - break - if ref_kind is None: - return None - - open_paren_index: int = start + len(_REFERENCE_PREFIX_BY_KIND[ref_kind]) - closing_paren_index: int = find_matching_paren( - sql=sql, open_paren_index=open_paren_index, context=_CONTEXT, syntax=syntax - ) - raw_arguments: str = sql[open_paren_index + 1 : closing_paren_index] - call: str = sql[start : closing_paren_index + 1] - argument_match: re.Match[str] | None = ( - _DBT_REF_ARGUMENTS_PATTERN - if ref_kind == SqlReferenceKind.DBT_REF - else _NAME_ARGUMENT_PATTERN - ).fullmatch(raw_arguments) - if argument_match is None: - return ( - _invalid_reference_arguments( - ref_kind=ref_kind, - call=call, - start=start, - raw_arguments=raw_arguments, - syntax=syntax, - ), - closing_paren_index + 1, - ) - if ref_kind == SqlReferenceKind.DBT_REF and argument_match.group(2) is not None: - return ( - CompileSqlReference( - ref_kind=ref_kind, - ref_package=argument_match.group(1), - ref_name=argument_match.group(2), - ), - closing_paren_index + 1, - ) - call_argument_count: int | None = None - if ref_kind == SqlReferenceKind.TABLE_FUNCTION: - call_suffix_start: int = _skip_whitespace(sql=sql, start=closing_paren_index + 1) - if call_suffix_start >= len(sql) or sql[call_suffix_start] != SQL_OPEN_PAREN_TOKEN: - corrected_call: str = ref_kind.example_call(argument_match.group(1), quote='"') - return ( - InvalidSqlReferenceCall( - ref_kind=ref_kind, - call=call, - start=start, - message=f"{ref_prefix(ref_kind)} must be followed by an argument list", - help=( - "pass the function arguments in a second set of parentheses, using () " - f"for no arguments: {corrected_call}()" - ), - corrected_call=f"{corrected_call}()", - ), - closing_paren_index + 1, - ) - call_suffix_end: int = find_matching_paren( - sql=sql, - open_paren_index=call_suffix_start, - context="SQL table function call", - syntax=syntax, - ) - call_argument_count = len( - _split_top_level_arguments( - raw_arguments=sql[call_suffix_start + 1 : call_suffix_end], syntax=syntax - ) - ) - return ( - CompileSqlReference( - ref_kind=ref_kind, - ref_name=argument_match.group(1), - call_argument_count=call_argument_count, - ), - closing_paren_index + 1, - ) - - -def _invalid_reference_arguments( - *, - ref_kind: SqlReferenceKind, - call: str, - start: int, - raw_arguments: str, - syntax: SqlLexicalSyntax, -) -> InvalidSqlReferenceCall: - """Describe a reference call that later compile stages could not replace.""" - - names: tuple[str, ...] | None = _authored_reference_names( - raw_arguments=raw_arguments, syntax=syntax - ) - allowed_counts: frozenset[int] = ( - frozenset({1, _DBT_REF_PACKAGE_ARGUMENT_COUNT}) - if ref_kind == SqlReferenceKind.DBT_REF - else frozenset({1}) - ) - if names is None or len(names) not in allowed_counts: - names = _PLACEHOLDER_NAMES[ref_kind] - corrected_call: str = ref_kind.example_call(*names, quote='"') - if ref_kind == SqlReferenceKind.TABLE_FUNCTION: - corrected_call += "(...)" - accepted: str = ( - "one double-quoted model name, or a double-quoted package name and model name " - "separated by a comma" - if ref_kind == SqlReferenceKind.DBT_REF - else "exactly one double-quoted name" - ) - return InvalidSqlReferenceCall( - ref_kind=ref_kind, - call=call, - start=start, - message=f"{' '.join(call.split())} is not a valid {ref_prefix(ref_kind)}() call", - help=( - f"{ref_prefix(ref_kind)}() takes {accepted}, with no comments or extra spaces " - f"inside the parentheses: {corrected_call}" - ), - corrected_call=corrected_call, - ) - - -def _authored_reference_names( - *, raw_arguments: str, syntax: SqlLexicalSyntax -) -> tuple[str, ...] | None: - """Return the names the author meant, when each argument is a plain name or string.""" - - try: - arguments: tuple[str, ...] = _split_top_level_arguments( - raw_arguments=raw_arguments, syntax=syntax - ) - except CompileInputError: - return None - names: list[str] = [] - for argument in arguments: - name: str = argument - if ( - len(argument) >= _PAIRED_QUOTE_CHARACTER_COUNT - and argument[0] == argument[-1] - and argument[0] in SQL_REFERENCE_NAME_QUOTE_TOKENS - ): - name = argument[1:-1] - if _AUTHORED_NAME_PATTERN.fullmatch(name) is None: - return None - names.append(name) - return tuple(names) - - -def ref_prefix(ref_kind: SqlReferenceKind | str) -> str: - normalized_ref_kind: SqlReferenceKind = SqlReferenceKind(ref_kind) - return _REFERENCE_PREFIX_BY_KIND[normalized_ref_kind] - - -def _skip_whitespace(*, sql: str, start: int) -> int: - index: int = start - while index < len(sql) and sql[index].isspace(): - index += 1 - return index - - -def _split_top_level_arguments(*, raw_arguments: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: - arguments: list[str] = [] - current: list[str] = [] - depth: int = 0 - index: int = 0 - saw_separator: bool = False - while index < len(raw_arguments): - non_code_end: int | None = dialect_non_code_end( - sql=raw_arguments, start=index, syntax=syntax, context=_CONTEXT - ) - if non_code_end is not None: - current.append( - raw_arguments[index:non_code_end] - if raw_arguments[index] in SQL_QUOTE_TOKENS - else " " - ) - index = non_code_end - continue - character: str = raw_arguments[index] - if character == SQL_OPEN_PAREN_TOKEN: - depth += 1 - elif character == SQL_CLOSE_PAREN_TOKEN: - depth -= 1 - elif character == SQL_ARGUMENT_SEPARATOR_TOKEN and depth == 0: - argument_text: str = "".join(current).strip() - if not argument_text: - raise CompileInputError(f"{_CONTEXT} contains an empty argument") - arguments.append(argument_text) - current = [] - saw_separator = True - index += 1 - continue - current.append(character) - index += 1 - - final_argument: str = "".join(current).strip() - if final_argument: - arguments.append(final_argument) - elif saw_separator: - raise CompileInputError(f"{_CONTEXT} contains an empty argument") - return tuple(arguments) diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 8bdb406acc..4b4a9be988 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -60,7 +60,6 @@ class NativeFallbackSite(StrEnum): MACRO_UNBRIDGED_EXPANSION = "macro_calls.unbridged_expansion" MACRO_BRIDGE_UNAVAILABLE = "macro_calls.bridge_unavailable" MACRO_CALL_MOCKED = "macro_calls.mocked_evaluation" - REFERENCE_SCAN = "reference_extraction.scan" CONFIG_BUILD = "model_config.build" CONFIG_HEADER_METADATA = "model_config.header_metadata" CONFIG_PRESENCE_SCAN = "model_config.presence_scan" diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 87156fe212..7de538663f 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -17,6 +17,7 @@ SQL_STRING_PREFIX_MAX_LENGTH: int = 2 SQL_TRIPLE_QUOTE_LENGTH: int = 3 SQL_LINE_COMMENT_PREFIX: str = "--" +SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) SQL_BLOCK_COMMENT_OPEN: str = "/*" SQL_BLOCK_COMMENT_CLOSE: str = "*/" SQL_DOLLAR_QUOTE_DELIMITER: str = "$$" diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py index e002f357eb..576e6b175f 100644 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ b/src/sqlbuild/compiler/sql_analysis/models.py @@ -12,6 +12,7 @@ SQL_ESCAPE_CHARACTER, SQL_LINE_COMMENT_PREFIX, SQL_TRIPLE_QUOTE_LENGTH, + SUPPORTED_SQL_LINE_COMMENT_PREFIXES, ) @@ -59,6 +60,16 @@ class SqlLexicalSyntax: nested_block_comments: bool = False line_comment_prefixes: frozenset[str] = frozenset({SQL_LINE_COMMENT_PREFIX}) + def __post_init__(self) -> None: + unsupported: frozenset[str] = ( + self.line_comment_prefixes - SUPPORTED_SQL_LINE_COMMENT_PREFIXES + ) + if unsupported: + raise ValueError( + f"SQL line comment prefixes {sorted(unsupported)} are not supported; SQLBuild " + f"scans only {sorted(SUPPORTED_SQL_LINE_COMMENT_PREFIXES)}" + ) + @property def cache_key(self) -> str: """Return a process-stable text key for these lexical rules.""" diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index 0c39b1c9cc..f58bd67b3c 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -99,10 +99,10 @@ def _perturbed(self, **changes): _record.write(json.dumps({"kind": "legacy_fallback", "site": "orders.sql"}) + "\\n") """ -REFERENCE_SCAN_SABOTAGE: str = """ -import sqlbuild.compiler.compile._helpers.refs.references as references +MACRO_RESOLUTION_SABOTAGE: str = """ +from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge -references.extract_native_sql_references = lambda **_: None +MacroBridge.call_class = lambda self, **_: None """ MODEL_ANALYSIS_SABOTAGE: str = """ diff --git a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py index 27073a525e..f0691862f7 100644 --- a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py +++ b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py @@ -9,8 +9,8 @@ from scripts.compiler_differential.main.differential import run_compiler_differential from tests.e2e.scripts.compiler_differential._test_types import NativeFallbackGateTestCase from tests.e2e.scripts.compiler_differential.helpers import ( + MACRO_RESOLUTION_SABOTAGE, MODEL_ANALYSIS_SABOTAGE, - REFERENCE_SCAN_SABOTAGE, harness_arguments, stage_disable_sabotage, write_native_perturbation, @@ -32,8 +32,8 @@ "fallback disappeared but native answers did not appear; the stage may be switched off" ) _VANISHED_ENTRY: str = ( - '\n[[entry]]\nengine = "native-preview"\nstage = "reference_extraction"\n' - 'site = "reference_extraction.scan"\nkind = "deferred"\ncounts = { project = 1 }\n' + '\n[[entry]]\nengine = "native-preview"\nstage = "macro_calls"\n' + 'site = "macro_calls.resolution"\nkind = "deferred"\ncounts = { project = 1 }\n' ) @@ -53,18 +53,18 @@ appended_entries=_VANISHED_ENTRY, expected_exit_code=1, expected_lines=( - "Native fallback allow-list: native-preview reference_extraction " - f"reference_extraction.scan deferred (project): {_FALLBACK_WITHOUT_ANSWERS}", + "Native fallback allow-list: native-preview macro_calls " + f"macro_calls.resolution deferred (project): {_FALLBACK_WITHOUT_ANSWERS}", ), ), NativeFallbackGateTestCase( - description="shipped_reference_scan_sabotaged_to_python_fails", - perturbation=REFERENCE_SCAN_SABOTAGE, + description="shipped_macro_resolution_sabotaged_to_python_fails", + perturbation=MACRO_RESOLUTION_SABOTAGE, appended_entries="", expected_exit_code=1, expected_lines=( - "Native fallback allow-list: native-preview reference_extraction " - "reference_extraction.scan deferred (project): ", + "Native fallback allow-list: native-preview macro_calls " + "macro_calls.resolution deferred (project): ", "not on the allow-list; port it or list it with a reason", ), ), diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py index 177744b936..115edaf939 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py @@ -7,8 +7,8 @@ @dataclass(frozen=True) -class CraftedReferenceParityTestCase: - """One SQL text whose native extraction must equal Python's under every lexical syntax.""" +class CraftedReferenceTestCase: + """One SQL text and how many calls the native scan rejects or fails on.""" description: str sql: str @@ -16,21 +16,6 @@ class CraftedReferenceParityTestCase: expected_failed: int = 0 -@dataclass(frozen=True) -class GeneratedReferenceParityTestCase: - """Seeded reference SQL compared with Python, with minimum coverage of each outcome.""" - - description: str - syntax: str - seed: int - count: int - expected_minimum_extracted: int - expected_minimum_failed: int - expected_minimum_table_functions: int - expected_minimum_rejected: int - expected_maximum_deferred: int - - @dataclass(frozen=True) class ReferenceEngineTestCase: """Reference extraction through the compiler entry point under one engine.""" @@ -55,18 +40,6 @@ class NativeReferenceErrorTestCase: expected_bridge_independent: bool -@dataclass(frozen=True) -class ReferenceDiagnosticParityTestCase: - """Generated authored files whose rejected calls both engines must report identically.""" - - description: str - syntax: str - seed: int - count: int - expected_minimum_located: int - expected_minimum_located_errors: int - - @dataclass(frozen=True) class ReferenceScanBoundTestCase: """A worst-case SQL shape the native scan must finish within a strict time limit.""" diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py index 9091cd6beb..7169985f1c 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py @@ -1,324 +1,8 @@ -"""Generated reference SQL and native-versus-Python comparison for reference extraction.""" +"""Source maps for located reference extraction errors.""" from __future__ import annotations -import random -import re -from dataclasses import dataclass -from itertools import chain, compress -from operator import attrgetter, itemgetter -from pathlib import Path -from typing import cast - -import pytest - -from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics -from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references -from sqlbuild.compiler.compile._helpers.refs.references import ( - extract_sql_references, - python_reference_scan, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - CompileSqlReference, - ExpansionSpan, - SqlReferenceOrigin, - SqlReferenceScan, - SqlReferenceSourceMap, -) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlReferenceScanFailure -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.compiler.frontier.types import CompilerEngine -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from tests.integration.src.sqlbuild.compiler.helpers import mismatches - -LEXICAL_SYNTAXES: dict[str, SqlLexicalSyntax] = { - "generic": SqlLexicalSyntax(), - "bigquery": BigQueryAdapter.sql_lexical_syntax, - "databricks": DatabricksAdapter.sql_lexical_syntax, - "duckdb": DuckDbAdapter.sql_lexical_syntax, - "postgres": PostgresAdapter.sql_lexical_syntax, - "snowflake": SnowflakeAdapter.sql_lexical_syntax, - "sqlserver": SqlServerAdapter.sql_lexical_syntax, -} -_NAMES: tuple[str, ...] = ( - '"orders"', - '"customers"', - '"order items"', - '"größe"', -) -_REJECTED_NAMES: tuple[str, ...] = ( - "'customers'", - "products", - " inventory\t", - "'commandés'", - "orders\x1c", - "orders /* note */", - "-- note\norders", - ' "orders" ', - ' "order""s" ', - '"a" "b"', - '""', - '/* note */ "orders"', -) -_MALFORMED_NAMES: tuple[str, ...] = ( - "'x\"", - "1orders", - "_orders", - "order-items", - "commandés", - "\u00a0orders", - "concat('a', 'b')", - '__ref("orders")', - "", - " ", - "/* only */", - "'orders'", - "orders", -) -_NO_SECOND_ARGUMENT: tuple[str, ...] = ("",) -_DBT_SECOND_ARGUMENTS: tuple[str, ...] = ("", ', "products"', ' ,\x1c"orders"', ", products") -_MALFORMED_SECOND_ARGUMENTS: tuple[str, ...] = (",", ", ", ",, products", ", products", ' ,"a"') -_NO_CALL_SUFFIX: tuple[str, ...] = ("",) -_CALL_SUFFIXES: tuple[str, ...] = ( - "()", - "(1)", - " (1, 2)", - "\n(f(1, 2), 'a,b')", - "((SELECT 1), $$a,b$$)", - '(__ref("orders"))', - "(1 /* , */, 2 -- ,\n)", - "\x1c(1)", -) -_MALFORMED_CALL_SUFFIXES: tuple[str, ...] = ( - "(1,)", - "(, 1)", - "/* gap */ (1)", - "\u00a0(1)", - "", - "(1", - " x", -) -_FILLERS: tuple[str, ...] = ( - "SELECT * FROM ", - " JOIN ", - "\n", - "'__ref(\"hidden\")'", - '"quoted__ref(x)"', - "`tick__ref(x)`", - "$$ __ref(hidden) $$", - "$tag$ __table_fn(hidden) $tag$", - "-- __ref(hidden)\n", - "/* __ref(hidden) */", - "'it''s'", - "a$$b", - "é", - "/* unclosed", - "'unclosed", -) -_RISKY_FILLERS: tuple[str, ...] = ( - "# __ref(hidden)\n", - "// __ref(hidden)\n", - "/* outer /* inner */ __ref(nested) */", - "'it\\'s __ref(escaped)'", - "E'it\\'s __ref(escaped)'", - "r'C:\\' __ref(raw)", - "'''it's __ref(triple)'''", - "(", - ")", - "'unclosed", - "/* unclosed", - "$$unclosed", -) -_USUAL_AND_RARE_WEIGHTS: tuple[int, int] = (94, 6) - - -@dataclass(frozen=True) -class _CallShape: - call: str - names: tuple[str, ...] - second_arguments: tuple[str, ...] - call_suffixes: tuple[str, ...] - - -_ANY_NAMES: tuple[str, ...] = (*(_NAMES * 9), *_REJECTED_NAMES) -_CALL_SHAPES: tuple[_CallShape, ...] = ( - _CallShape("__ref(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _NO_CALL_SUFFIX), - _CallShape("__source(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _NO_CALL_SUFFIX), - _CallShape("__seed(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _NO_CALL_SUFFIX), - _CallShape("__udf(", _ANY_NAMES, _NO_SECOND_ARGUMENT, ("(x)",)), - _CallShape("__dbt_ref(", _ANY_NAMES, _DBT_SECOND_ARGUMENTS, _NO_CALL_SUFFIX), - _CallShape("__table_fn(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _CALL_SUFFIXES), - _CallShape("__table_fn(", _NAMES, _NO_SECOND_ARGUMENT, _CALL_SUFFIXES), - _CallShape("___ref(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _NO_CALL_SUFFIX), - _CallShape("x__ref(", _ANY_NAMES, _NO_SECOND_ARGUMENT, _NO_CALL_SUFFIX), -) - - -def _pick(*, rng: random.Random, usual: tuple[str, ...], rare: tuple[str, ...]) -> str: - return rng.choice(rng.choices((usual, rare), weights=_USUAL_AND_RARE_WEIGHTS)[0]) - - -@dataclass(frozen=True) -class ReferenceParity: - """How native extraction compared with Python over one corpus; counts are native's.""" - - mismatches: list[tuple[object, object, object]] - extracted: int - failed: int - deferred: int - rejected: int - table_functions: int - - -def generated_reference_sqls(*, rng: random.Random, count: int) -> list[str]: - """Return seeded SQL mixing valid, malformed, hidden and dialect-sensitive reference calls.""" - - return [_generated_reference_sql(rng=rng) for _ in range(count)] - - -def _generated_reference_sql(*, rng: random.Random) -> str: - return "".join(_generated_reference_call(rng=rng) for _ in range(rng.randint(1, 4))) - - -def _generated_reference_call(*, rng: random.Random) -> str: - shape: _CallShape = rng.choice(_CALL_SHAPES) - return "".join( - ( - _pick(rng=rng, usual=_FILLERS, rare=_RISKY_FILLERS), - shape.call, - _pick(rng=rng, usual=shape.names, rare=_MALFORMED_NAMES), - _pick(rng=rng, usual=shape.second_arguments, rare=_MALFORMED_SECOND_ARGUMENTS), - ")", - _pick(rng=rng, usual=shape.call_suffixes, rare=_MALFORMED_CALL_SUFFIXES), - ) - ) - - -def python_outcome( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - """Return the oracle's references and rejected calls, or its error and where it points.""" - - return python_reference_scan(sql=sql, syntax=syntax) - - -def reference_parity(*, sqls: list[str], syntax: SqlLexicalSyntax) -> ReferenceParity: - """Compare native extraction, rejected calls and errors with Python wherever native scans.""" - - native: list[SqlReferenceScan | SqlReferenceScanFailure | None] = [ - extract_native_sql_references(sql=sql, syntax=syntax) for sql in sqls - ] - python: list[SqlReferenceScan | SqlReferenceScanFailure] = [ - python_outcome(sql=sql, syntax=syntax) for sql in sqls - ] - scanned: list[bool] = [outcome is not None for outcome in native] - extracted: list[SqlReferenceScan] = cast( - list[SqlReferenceScan], - list(compress(native, [isinstance(outcome, SqlReferenceScan) for outcome in native])), - ) - return ReferenceParity( - mismatches=mismatches( - inputs=list(compress(sqls, scanned)), - expected=list(compress(python, scanned)), - actual=list(compress(native, scanned)), - ), - extracted=sum(not scan.invalid_calls for scan in extracted), - failed=sum(isinstance(outcome, tuple) for outcome in native), - deferred=native.count(None), - rejected=sum(len(scan.invalid_calls) for scan in extracted), - table_functions=sum( - reference.ref_kind is SqlReferenceKind.TABLE_FUNCTION - for reference in chain.from_iterable(map(attrgetter("references"), extracted)) - ), - ) - - -_LOCATED_ERROR_PATTERN: re.Pattern[str] = re.compile(r"[^:]+:\d+:\d+: ") -_FILE_HEADERS: tuple[str, ...] = ( - 'MODEL (description "Orders");\n\n', - 'MODEL (\n description "Orders",\n);\n-- notes\n\n', - 'MODEL (description "Ordres é");\n/* commentaire */\n', - 'MODEL (description "Orders over 12\\" boxes, by customer\'s totals");\n\n', -) - - -def generated_reference_files(*, rng: random.Random, count: int) -> list[tuple[str, str]]: - """Return seeded authored files as `(relative path, SQL body)` pairs.""" - - return [ - (f"models/area_{index % 7}/orders_{index}.sql", body) - for index, body in enumerate(generated_reference_sqls(rng=rng, count=count)) - ] - - -def reported_reference_outcomes( - *, - files: list[tuple[str, str]], - syntax: SqlLexicalSyntax, - engine: CompilerEngine, - monkeypatch: pytest.MonkeyPatch, -) -> list[object]: - """Extract each file's body under `engine`, keeping its references, diagnostics and error.""" - - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) - return [ - _reported_reference_outcome( - origin=SqlReferenceOrigin( - file_path=Path("/project") / relative_path, - relative_path=Path(relative_path), - contents=_FILE_HEADERS[index % len(_FILE_HEADERS)] + body, - resource_type=CompiledResourceType.MODEL, - resource_name=Path(relative_path).stem, - source_map=source_map_at(body_start=len(_FILE_HEADERS[index % len(_FILE_HEADERS)])), - ), - sql=body, - syntax=syntax, - ) - for index, (relative_path, body) in enumerate(files) - ] - - -def _reported_reference_outcome( - *, origin: SqlReferenceOrigin, sql: str, syntax: SqlLexicalSyntax -) -> tuple[tuple[CompileSqlReference, ...], tuple[CompilerDiagnostic, ...], str, str | None]: - references: tuple[CompileSqlReference, ...] = () - with collect_compile_diagnostics() as collected: - try: - references = extract_sql_references(sql=sql, syntax=syntax, origin=origin) - except CompileInputError as error: - return (references, tuple(collected.diagnostics), str(error), error.code) - return (references, tuple(collected.diagnostics), "", None) - - -def located_error_count(*, outcomes: list[object]) -> int: - """Count the scan errors that carry an authored line and column.""" - - errors: list[str] = list(map(itemgetter(2), cast(list[tuple[object, object, str]], outcomes))) - return sum(bool(_LOCATED_ERROR_PATTERN.match(error)) for error in errors) - - -def located_diagnostic_count(*, outcomes: list[object]) -> int: - """Count the reported diagnostics that carry an authored location.""" - - diagnostics: list[tuple[CompilerDiagnostic, ...]] = list( - map(itemgetter(1), cast(list[tuple[object, tuple[CompilerDiagnostic, ...]]], outcomes)) - ) - return sum(diagnostic.location is not None for diagnostic in chain.from_iterable(diagnostics)) - - -def python_scan_not_expected(**_: object) -> SqlReferenceScan | SqlReferenceScanFailure: - """A replacement for the Python scanner that fails the test when it is called.""" - - raise AssertionError("the Python reference scanner ran") +from sqlbuild.compiler.compile.models import ExpansionSpan, SqlReferenceSourceMap def source_map_at( diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index 3aa18da59a..b494b42b5b 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -1,8 +1,7 @@ -"""Native reference extraction returns Python's references, rejected calls and errors exactly.""" +"""Native reference extraction: references, rejected calls, located errors and scan bounds.""" from __future__ import annotations -import random import time from collections.abc import Callable from itertools import product @@ -22,28 +21,14 @@ from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( - CraftedReferenceParityTestCase, - GeneratedReferenceParityTestCase, + CraftedReferenceTestCase, NativeReferenceErrorTestCase, - ReferenceDiagnosticParityTestCase, ReferenceEngineTestCase, ReferenceScanBoundTestCase, ) -from tests.integration.src.sqlbuild.compiler.compile._helpers.refs.helpers import ( - LEXICAL_SYNTAXES, - ReferenceParity, - generated_reference_files, - generated_reference_sqls, - located_diagnostic_count, - located_error_count, - python_scan_not_expected, - reference_parity, - reported_reference_outcomes, - source_map_at, -) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.compile._helpers.refs.helpers import source_map_at -_GENERIC_SYNTAX: SqlLexicalSyntax = LEXICAL_SYNTAXES["generic"] +_GENERIC_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() _ENGINE_SQLS: tuple[ tuple[str, str, tuple[tuple[str, str, str | None, int | None], ...], str], ... ] = ( @@ -78,7 +63,10 @@ _OUTCOME_SUMMARIES: dict[type, Callable[[Any], int | str]] = { SqlReferenceScan: lambda scan: len(scan.references) + len(scan.invalid_calls), tuple: itemgetter(0), - type(None): repr, +} +_REJECTED_AND_FAILED: dict[type, Callable[[Any], tuple[int, int]]] = { + SqlReferenceScan: lambda scan: (len(scan.invalid_calls), 0), + tuple: lambda _failure: (0, 1), } _MANY_REFERENCES: str = "SELECT 1 FROM " + " JOIN ".join( f'__ref("orders_{index}")' for index in range(50_000) @@ -90,7 +78,7 @@ @pytest.mark.parametrize( "test_case", [ - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="every_kind_in_authored_order", sql=( 'SELECT * FROM __ref("orders") JOIN __source("raw_orders") ' @@ -98,25 +86,25 @@ 'WHERE __udf("clean")(x) > 0 OR __dbt_ref("orders") IS NULL' ), ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="table_function_argument_counts", sql=( "SELECT * FROM __table_fn(\"expand_orders\") ( 1, f(2, 3), 'a,b' /* , */ ) " 'JOIN __table_fn("daily_orders")() JOIN __table_fn("by_day")\n(1 -- ,\n, 2)' ), ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="references_nested_in_table_function_arguments", sql='SELECT * FROM __table_fn("orders_for")((SELECT id FROM __ref("customers")))', ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="hidden_in_comments_strings_and_dollar_quotes", sql=( "-- __ref(a)\nSELECT '__ref(b)', $$__table_fn(\"c\")$$, $t$ __ref(d) $t$, " '/* __ref(e) */ "__ref(f)", `__ref(g)` FROM __ref("h")' ), ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="dialect_sensitive_text", sql=( "SELECT 'it\\'s __ref(a)', E'x\\'__ref(b)', r'C:\\' AS p, '''__ref(c)''', " @@ -124,136 +112,95 @@ ), expected_rejected=4, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="comments_inside_calls_are_rejected", sql='SELECT * FROM __ref("a" /* note */ "b") JOIN __ref(-- c\n"orders")', expected_rejected=2, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="non_ascii_quoted_names_and_text", sql="SELECT 'é' FROM __ref(\"commandés\") JOIN __table_fn(\"größe\")('ü')", ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="wrong_reference_argument_count_is_rejected", sql="SELECT * FROM __ref(orders, customers)", expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unquoted_and_single_quoted_names_are_rejected", sql="SELECT * FROM __ref(orders) JOIN __source('raw_orders')", expected_rejected=2, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="spaces_inside_calls_are_rejected", sql='SELECT * FROM __seed( "regions" ) JOIN __dbt_ref("shop", "orders" )', expected_rejected=2, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unclosed_text_after_a_rejected_call_fails", sql="SELECT * FROM __ref(orders) WHERE note = 'unclosed", expected_failed=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unclosed_text_before_a_rejected_call_fails", sql='SELECT * FROM __ref("orders" WHERE __ref(customers)', expected_failed=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="wrong_dbt_reference_argument_count_is_rejected", sql="SELECT * FROM __dbt_ref(shop, orders, extra)", expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="empty_table_function_call_argument", sql='SELECT * FROM __table_fn("orders_for")(1,,2)', expected_failed=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="table_function_without_call_is_rejected", sql='SELECT * FROM __table_fn("orders_for") /* gap */ (1)', expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unquoted_table_function_name_is_rejected", sql="SELECT * FROM __table_fn(orders_for)(1)", expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unclosed_table_function_call", sql='SELECT * FROM __table_fn("orders_for")(1', expected_failed=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="nested_reference_as_name_is_rejected", sql="SELECT * FROM __ref(__ref(orders))", expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="non_ascii_identifier_is_rejected", sql="SELECT * FROM __ref(commandés)", expected_rejected=1, ), - CraftedReferenceParityTestCase( + CraftedReferenceTestCase( description="unicode_whitespace_before_call_is_skipped", sql='SELECT * FROM __table_fn("orders_for")\u00a0(1)', ), ], ids=lambda case: case.description, ) -def test_given_crafted_reference_sql_when_extracting_natively_then_matches_python( - test_case: CraftedReferenceParityTestCase, +def test_given_crafted_reference_sql_when_extracting_then_rejected_and_failed_counts_match( + test_case: CraftedReferenceTestCase, ) -> None: - parity: ReferenceParity = reference_parity(sqls=[test_case.sql], syntax=_GENERIC_SYNTAX) - dialect_mismatches: list[list[tuple[object, object, object]]] = [ - reference_parity(sqls=[test_case.sql], syntax=syntax).mismatches - for syntax in LEXICAL_SYNTAXES.values() - ] + outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( + sql=test_case.sql, syntax=_GENERIC_SYNTAX + ) - assert (dialect_mismatches, parity.deferred, parity.rejected, parity.failed) == ( - [[]] * len(LEXICAL_SYNTAXES), - 0, + assert _REJECTED_AND_FAILED[type(outcome)](outcome) == ( test_case.expected_rejected, test_case.expected_failed, ) -@pytest.mark.parametrize( - "test_case", - [ - GeneratedReferenceParityTestCase( - description=f"{syntax}_syntax", - syntax=syntax, - seed=20261007 + offset, - count=3000, - expected_minimum_extracted=700, - expected_minimum_failed=500, - expected_minimum_table_functions=300, - expected_minimum_rejected=1400, - expected_maximum_deferred=0, - ) - for offset, syntax in enumerate(LEXICAL_SYNTAXES) - ], - ids=lambda case: case.description, -) -def test_given_generated_reference_sql_when_extracting_natively_then_matches_python( - test_case: GeneratedReferenceParityTestCase, -) -> None: - sqls: list[str] = generated_reference_sqls( - rng=random.Random(test_case.seed), count=test_case.count - ) - - parity: ReferenceParity = reference_parity(sqls=sqls, syntax=LEXICAL_SYNTAXES[test_case.syntax]) - - assert ( - parity.mismatches, - parity.extracted >= test_case.expected_minimum_extracted, - parity.failed >= test_case.expected_minimum_failed, - parity.table_functions >= test_case.expected_minimum_table_functions, - parity.rejected >= test_case.expected_minimum_rejected, - parity.deferred <= test_case.expected_maximum_deferred, - ) == ([], True, True, True, True, True), parity - - @pytest.mark.parametrize( "test_case", [ @@ -263,7 +210,7 @@ def test_given_generated_reference_sql_when_extracting_natively_then_matches_pyt sql=sql, expected_references=expected_references, expected_error=expected_error, - expected_native_calls=int(engine is not CompilerEngine.PYTHON), + expected_native_calls=1, ) for engine, (name, sql, expected_references, expected_error) in product( CompilerEngine, _ENGINE_SQLS @@ -271,18 +218,18 @@ def test_given_generated_reference_sql_when_extracting_natively_then_matches_pyt ], ids=lambda case: case.description, ) -def test_given_engine_when_extracting_references_then_only_native_engines_run_native_scanner( +def test_given_engine_when_extracting_references_then_every_engine_runs_native_scanner( test_case: ReferenceEngineTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) native_calls: list[str] = [] - native_extract: Callable[..., SqlReferenceScan | SqlReferenceScanFailure | None] = ( + native_extract: Callable[..., SqlReferenceScan | SqlReferenceScanFailure] = ( references.extract_native_sql_references ) def counting_extract( *, sql: str, syntax: SqlLexicalSyntax - ) -> SqlReferenceScan | SqlReferenceScanFailure | None: + ) -> SqlReferenceScan | SqlReferenceScanFailure: native_calls.append(sql) return native_extract(sql=sql, syntax=syntax) @@ -391,11 +338,10 @@ def counting_extract( ], ids=lambda case: case.description, ) -def test_given_native_reference_error_when_extracting_then_raised_located_without_python_rescan( +def test_given_native_reference_error_when_extracting_then_raised_located( test_case: NativeReferenceErrorTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) - monkeypatch.setattr(references, "python_reference_scan", python_scan_not_expected) with pytest.raises(CompileInputError) as raised: _ = extract_sql_references( @@ -424,43 +370,6 @@ def test_given_native_reference_error_when_extracting_then_raised_located_withou ) -@pytest.mark.parametrize( - "test_case", - [ - ReferenceDiagnosticParityTestCase( - description=f"{syntax}_syntax", - syntax=syntax, - seed=20261009 + offset, - count=400, - expected_minimum_located=120, - expected_minimum_located_errors=90, - ) - for offset, syntax in enumerate(LEXICAL_SYNTAXES) - ], - ids=lambda case: case.description, -) -def test_given_generated_files_when_reporting_rejected_calls_then_engines_report_identically( - test_case: ReferenceDiagnosticParityTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - files: list[tuple[str, str]] = generated_reference_files( - rng=random.Random(test_case.seed), count=test_case.count - ) - syntax: SqlLexicalSyntax = LEXICAL_SYNTAXES[test_case.syntax] - - python: list[object] = reported_reference_outcomes( - files=files, syntax=syntax, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch - ) - preview: list[object] = reported_reference_outcomes( - files=files, syntax=syntax, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch - ) - - assert ( - mismatches(inputs=[body for _, body in files], expected=python, actual=preview), - located_diagnostic_count(outcomes=python) >= test_case.expected_minimum_located, - located_error_count(outcomes=python) >= test_case.expected_minimum_located_errors, - ) == ([], True, True) - - @pytest.mark.parametrize( "test_case", [ @@ -501,7 +410,7 @@ def test_given_worst_case_reference_sql_when_extracting_natively_then_finishes_q test_case: ReferenceScanBoundTestCase, ) -> None: started: float = time.thread_time() - outcome: SqlReferenceScan | SqlReferenceScanFailure | None = extract_native_sql_references( + outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( sql=test_case.sql, syntax=_GENERIC_SYNTAX ) elapsed: float = time.thread_time() - started diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py index 20c7960c22..9ec4a839cb 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_fallback_recorder.py @@ -20,16 +20,16 @@ NativeFallbackRecorderTestCase( description="counts_per_site_and_kind_sorted", recorded=( - (NativeFallbackSite.CONFIG_BUILD, "deferred"), - (NativeFallbackSite.REFERENCE_SCAN, "deferred"), - (NativeFallbackSite.CONFIG_BUILD, "deferred"), - (NativeFallbackSite.CONFIG_BUILD, "error"), + (NativeFallbackSite.PROJECT_ASSEMBLY, "deferred"), + (NativeFallbackSite.MACRO_CALL_RESOLUTION, "deferred"), + (NativeFallbackSite.PROJECT_ASSEMBLY, "deferred"), + (NativeFallbackSite.PROJECT_ASSEMBLY, "error"), ), expected_files=[ [ - ["model_config.build", "deferred", 2], - ["model_config.build", "error", 1], - ["reference_extraction.scan", "deferred", 1], + ["macro_calls.resolution", "deferred", 1], + ["project_assembly.assembly", "deferred", 2], + ["project_assembly.assembly", "error", 1], ] ], ), From da69a4f91352e7cbabc646d4df6260e20e34e7ec Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:42:25 +0100 Subject: [PATCH 14/40] feat(native): build and validate model config natively under every engine; delete the Python config builder and validators --- .../src/text/_helpers/code_point_ranges.rs | 10 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + .../src/text/main/python_decimal_value.rs | 15 + .../src/text/main/python_text.rs | 1 + crates/sqlbuild-core/src/text/models.rs | 9 + .../config_presence/_helpers/macro_calls.rs | 35 +- .../src/config_presence/_helpers/scan.rs | 36 +- .../_helpers/template_tokens.rs | 9 +- .../main/contains_macro_call.rs | 5 +- .../config_presence/main/contains_template.rs | 5 +- .../config_presence/main/first_macro_path.rs | 12 +- .../src/config_presence/models.rs | 13 +- .../tests/test_config_presence.rs | 43 +- .../src/config_presence/tests/test_types.rs | 5 +- .../src/model_validation/_helpers/config.rs | 172 +- .../_helpers/cursor_bounds.rs | 394 ++--- .../src/model_validation/_helpers/decimals.rs | 175 ++ .../model_validation/_helpers/durations.rs | 67 +- .../model_validation/_helpers/incremental.rs | 77 +- .../model_validation/_helpers/isoformat.rs | 478 ++++++ .../_helpers/materialization.rs | 57 +- .../model_validation/_helpers/microbatch.rs | 106 +- .../src/model_validation/_helpers/mod.rs | 3 +- .../model_validation/_helpers/references.rs | 23 +- .../src/model_validation/_helpers/snapshot.rs | 6 +- .../src/model_validation/_helpers/text.rs | 35 - .../src/model_validation/constants.rs | 26 +- .../src/model_validation/main/mod.rs | 1 + .../main/retention_override.rs | 47 +- .../main/table_type_override.rs | 12 +- .../main/validate_model_config.rs | 4 +- .../main/validate_model_references.rs | 14 + .../src/model_validation/models.rs | 26 +- .../src/model_validation/tests/helpers.rs | 22 +- .../tests/test_storage_overrides.rs | 10 +- .../tests/test_validate_model_config.rs | 156 +- .../src/model_validation/types.rs | 2 +- .../src/tests/test_types.rs | 50 +- crates/sqlbuild-model-config/src/types.rs | 10 +- .../_helpers/model_config/authored_nodes.rs | 34 +- .../_helpers/model_config/config_presence.rs | 35 - .../src/bindings/_helpers/model_config/mod.rs | 1 - .../model_config/model_config_builder.rs | 98 +- .../_helpers/model_config/model_validator.rs | 102 +- .../_helpers/model_config/registration.rs | 4 +- .../bindings/_helpers/sqltext/static_sql.rs | 1 - .../src/engine/tests/test_sql_scanning.rs | 23 - .../src/engine/tests/test_types.rs | 1 - .../_helpers/corpus/failure_cases.py | 36 + .../native_fallbacks.toml | 97 +- src/sqlbuild/_native.pyi | 13 +- .../compile/_helpers/attachment/core.py | 243 +-- .../_helpers/attachment/model_config.py | 288 +--- .../compile/_helpers/attachment/references.py | 122 +- .../_helpers/config/model_validation.py | 1417 ----------------- .../compile/_helpers/config/table_type.py | 37 - src/sqlbuild/compiler/compile/constants.py | 7 - src/sqlbuild/compiler/compile/models.py | 48 +- src/sqlbuild/compiler/frontier/types.py | 3 - .../_native_config_contains_macro_call.py | 11 - .../main/_native_config_contains_template.py | 11 - .../main/compile/test_native_config_errors.py | 3 - ...ngine-error-non-ascii-replay-duration.json | 19 + .../engine-error-oversized-retention.json | 19 + .../native_model_config/_test_types.py | 44 +- .../compile/native_model_config/helpers.py | 648 +------- .../test_native_model_config_session.py | 155 +- .../compiler/model_config/_test_types.py | 11 - .../sqlbuild/compiler/model_config/helpers.py | 101 +- .../model_config/test_native_model_config.py | 47 +- .../compile/_helpers/config/helpers.py | 127 +- .../_helpers/config/test_table_type.py | 8 +- .../_helpers/test_microbatch_redesign.py | 28 +- .../_helpers/test_model_config_validation.py | 61 +- .../compile/_helpers/test_path_defaults.py | 2 +- .../sqlbuild/compiler/compile/test_main.py | 42 +- 76 files changed, 2090 insertions(+), 4029 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/main/python_decimal_value.rs create mode 100644 crates/sqlbuild-model-config/src/model_validation/_helpers/decimals.rs create mode 100644 crates/sqlbuild-model-config/src/model_validation/_helpers/isoformat.rs delete mode 100644 crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs create mode 100644 crates/sqlbuild-model-config/src/model_validation/main/validate_model_references.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs delete mode 100644 src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py delete mode 100644 src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py create mode 100644 tests/goldens/compiler/failure/engine-error-non-ascii-replay-duration.json create mode 100644 tests/goldens/compiler/failure/engine-error-oversized-retention.json diff --git a/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs index 2a0a763184..7b51cd55b2 100644 --- a/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs +++ b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs @@ -8,3 +8,13 @@ pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { .get(index) .is_some_and(|(start, _)| *start <= code_point) } + +/// The start of the range holding one character, or `None` when no range holds it. +pub(crate) fn range_start(ranges: &[(u32, u32)], character: char) -> Option { + let code_point: u32 = u32::from(character); + let index: usize = ranges.partition_point(|(_, end)| *end < code_point); + ranges + .get(index) + .filter(|(start, _)| *start <= code_point) + .map(|(start, _)| *start) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index 2b20809e36..5834d53ef8 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -6,5 +6,6 @@ pub mod is_python_decimal; pub mod is_python_space; pub mod is_python_word; pub mod python_cleandoc; +pub mod python_decimal_value; pub mod python_strip; pub mod python_text; diff --git a/crates/sqlbuild-core/src/text/main/python_decimal_value.rs b/crates/sqlbuild-core/src/text/main/python_decimal_value.rs new file mode 100644 index 0000000000..05f484a894 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_decimal_value.rs @@ -0,0 +1,15 @@ +//! The digit value of one Python decimal character, from a generated table. + +use crate::text::_helpers::code_point_ranges::range_start; +use crate::text::models::PythonText; + +/// The digit Python's `unicodedata.decimal()` gives one character, or `None` for a non-digit. +/// +/// Decimal digits come in contiguous runs from zero to nine, so a merged table range starts at a +/// zero and the value is the distance from it, modulo ten. +pub fn python_decimal_value(python: PythonText, character: char) -> Option { + if character.is_ascii() { + return character.to_digit(10); + } + range_start(python.decimal_ranges, character).map(|start| (u32::from(character) - start) % 10) +} diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs index 5253b9853c..f8d2bcfcdf 100644 --- a/crates/sqlbuild-core/src/text/main/python_text.rs +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -27,5 +27,6 @@ pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option u8 { + self.minor_version + } } /// The characters `inspect.cleandoc` strips from line starts. diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs index 2d8517fa20..f48d060455 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs @@ -1,18 +1,17 @@ //! Python's `@[A-Za-z_][A-Za-z0-9_]*\s*\(` search over one string. -use crate::config_presence::models::Presence; +use sqlbuild_core::text::main::is_python_space::is_python_space; -/// Search `text` for a macro call opening, deferring where Python's `\s` decides. -pub(crate) fn macro_call(text: &str) -> Presence { +/// Whether `text` holds a macro call opening; `\s` is Python's Unicode whitespace class. +pub(crate) fn macro_call(text: &str) -> bool { let bytes = text.as_bytes(); - let mut deferred = false; - for (at, _) in text.match_indices('@') { + text.match_indices('@').any(|(at, _)| { let mut index = at + 1; if !bytes .get(index) .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') { - continue; + return false; } while bytes .get(index) @@ -20,23 +19,9 @@ pub(crate) fn macro_call(text: &str) -> Presence { { index += 1; } - while bytes.get(index).copied().is_some_and(is_python_ascii_space) { - index += 1; - } - match bytes.get(index) { - Some(b'(') => return Presence::Present, - Some(byte) if !byte.is_ascii() => deferred = true, - _ => {} - } - } - if deferred { - Presence::Deferred - } else { - Presence::Absent - } -} - -/// ASCII characters Python's `\s` matches in a `str` pattern. -fn is_python_ascii_space(byte: u8) -> bool { - matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') + text[index..] + .chars() + .find(|character| !is_python_space(*character)) + == Some('(') + }) } diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs index 34669a21a2..892fd6fea8 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs @@ -1,36 +1,18 @@ //! The recursive config scan shared by every presence check. -use crate::config_presence::models::Presence; use crate::types::{AuthoredNode, NodeKind}; /// Scan mapping values, list and tuple items and strings like Python's recursive config scans. -pub(crate) fn scan( - node: &N, - text_presence: &impl Fn(&str) -> Presence, -) -> Presence { +pub(crate) fn scan(node: &N, text_presence: &impl Fn(&str) -> bool) -> bool { match node.kind() { - NodeKind::Str => node.with_text(text_presence).unwrap_or(Presence::Deferred), - NodeKind::Map => any_present(node.entries().iter().map(|(_, value)| value), text_presence), - NodeKind::List | NodeKind::Tuple => any_present(node.items().iter(), text_presence), - _ => Presence::Absent, - } -} - -fn any_present<'node, N: AuthoredNode + 'node>( - nodes: impl Iterator, - text_presence: &impl Fn(&str) -> Presence, -) -> Presence { - let mut deferred = false; - for node in nodes { - match scan(node, text_presence) { - Presence::Present => return Presence::Present, - Presence::Deferred => deferred = true, - Presence::Absent => {} + NodeKind::Str => node.with_text(text_presence).unwrap_or(false), + NodeKind::Map => node + .entries() + .iter() + .any(|(_, value)| scan(value, text_presence)), + NodeKind::List | NodeKind::Tuple => { + node.items().iter().any(|item| scan(item, text_presence)) } - } - if deferred { - Presence::Deferred - } else { - Presence::Absent + _ => false, } } diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs index dcf2a1d7f5..6691ba666b 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs @@ -1,13 +1,8 @@ //! Python's `TEMPLATE_OPEN_TOKEN in value` over one string. use crate::config_presence::constants::TEMPLATE_OPEN_TOKEN; -use crate::config_presence::models::Presence; /// Return whether `text` holds a template opening. -pub(crate) fn template_token(text: &str) -> Presence { - if text.contains(TEMPLATE_OPEN_TOKEN) { - Presence::Present - } else { - Presence::Absent - } +pub(crate) fn template_token(text: &str) -> bool { + text.contains(TEMPLATE_OPEN_TOKEN) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs index ba6326a391..924ec8d1c0 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs @@ -2,10 +2,9 @@ use crate::config_presence::_helpers::macro_calls::macro_call; use crate::config_presence::_helpers::scan::scan; -use crate::config_presence::models::Presence; use crate::types::AuthoredNode; -/// Return whether `MACRO_CALL_PATTERN` matches any nested string, or defer to Python. -pub fn contains_macro_call(node: &N) -> Presence { +/// Return whether `MACRO_CALL_PATTERN` matches any nested string. +pub fn contains_macro_call(node: &N) -> bool { scan(node, ¯o_call) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs index bacc702131..c4f3a3832d 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs @@ -2,10 +2,9 @@ use crate::config_presence::_helpers::scan::scan; use crate::config_presence::_helpers::template_tokens::template_token; -use crate::config_presence::models::Presence; use crate::types::AuthoredNode; -/// Return whether `contains_template_data` finds `${` in any nested string, or defer to Python. -pub fn contains_template(node: &N) -> Presence { +/// Return whether `contains_template_data` finds `${` in any nested string. +pub fn contains_template(node: &N) -> bool { scan(node, &template_token) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs index 2490eb32ae..d3ec7c465a 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs @@ -1,7 +1,7 @@ //! The first config field, in Python's walk order, whose string holds a macro call. use crate::config_presence::_helpers::macro_calls::macro_call; -use crate::config_presence::models::{MacroPath, Presence}; +use crate::config_presence::models::MacroPath; use crate::types::{AuthoredNode, NodeKind}; /// Walk config values as `validate_no_macros_in_config_value` does, skipping the `skipped` keys. @@ -13,11 +13,9 @@ pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroP fn walk(node: &N, path: &[String]) -> MacroPath { match node.kind() { - NodeKind::Str => match node.with_text(macro_call) { - Some(Presence::Present) => MacroPath::Found(path.to_vec()), - Some(Presence::Absent) => MacroPath::Absent, - Some(Presence::Deferred) | None => MacroPath::Deferred, - }, + NodeKind::Str if node.with_text(macro_call).unwrap_or(false) => { + MacroPath::Found(path.to_vec()) + } NodeKind::Map => walk_entries(&node.entries(), path), NodeKind::List | NodeKind::Tuple => { for item in node.items() { @@ -38,7 +36,7 @@ fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPa continue; } let Some(name) = key.text() else { - return MacroPath::Deferred; + continue; }; let mut child: Vec = path.to_vec(); child.push(name); diff --git a/crates/sqlbuild-model-config/src/config_presence/models.rs b/crates/sqlbuild-model-config/src/config_presence/models.rs index 38b270b275..6eba20ef0f 100644 --- a/crates/sqlbuild-model-config/src/config_presence/models.rs +++ b/crates/sqlbuild-model-config/src/config_presence/models.rs @@ -1,13 +1,4 @@ -//! Whether authored config contains a construct, or that only Python can tell. - -/// The answer Python's recursive config scan gives, or a deferral to that scan. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Presence { - Present, - Absent, - /// A string needs Python's Unicode `\s` class to decide. - Deferred, -} +//! Where a config walk finds a macro call. /// Where Python's config walk first finds a macro call. #[derive(Clone, Debug, PartialEq, Eq)] @@ -15,6 +6,4 @@ pub enum MacroPath { /// The mapping keys leading to the first string holding a macro call. Found(Vec), Absent, - /// A string before any match needs Python's Unicode rules to decide. - Deferred, } diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index 5bf585a298..00bba4ca11 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -1,11 +1,10 @@ use crate::config_presence::main::contains_macro_call::contains_macro_call; use crate::config_presence::main::contains_template::contains_template; -use crate::config_presence::models::Presence; use crate::config_presence::tests::test_types::PresenceTestCase; use crate::tests::test_types::Value; #[test] -fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { +fn given_config_values_when_scanning_then_presence_matches_python() { let test_cases = [ PresenceTestCase { description: "plain scalars contain nothing", @@ -15,8 +14,8 @@ fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { Value::Null, Value::Float, ]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Absent, + expected_template: false, + expected_macro_call: false, }, PresenceTestCase { description: "nested mapping values are scanned, keys are not", @@ -27,38 +26,44 @@ fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { Value::Tuple(vec![Value::Str("@cents (amount)")]), ), ]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { description: "a template token anywhere in a string", value: Value::Str("prefix_${target.schema}"), - expected_template: Presence::Present, - expected_macro_call: Presence::Absent, + expected_template: true, + expected_macro_call: false, }, PresenceTestCase { description: "an at-name without a call or with an invalid start is no macro", value: Value::Str("mail@example.com @1x( @ (x)"), - expected_template: Presence::Absent, - expected_macro_call: Presence::Absent, + expected_template: false, + expected_macro_call: false, }, PresenceTestCase { description: "Python separators count as whitespace before the call", value: Value::Str("@m\u{1f}\t("), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { - description: "non-ASCII after a macro name defers to Python's whitespace class", - value: Value::Str("@m\u{a0}("), - expected_template: Presence::Absent, - expected_macro_call: Presence::Deferred, + description: "Unicode whitespace after a macro name counts as Python's \\s", + value: Value::Str("@m\u{a0}\u{3000}("), + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { - description: "a later call outweighs an earlier deferral", + description: "other non-ASCII after a macro name is no call", + value: Value::Str("@m\u{e9}( @n\u{200b}("), + expected_template: false, + expected_macro_call: false, + }, + PresenceTestCase { + description: "a later call is found after Unicode whitespace that is no call", value: Value::List(vec![Value::Str("@m\u{2003}("), Value::Str("@n(")]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, ]; diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs index 917d2a3746..cb04f91f0a 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -1,10 +1,9 @@ -use crate::config_presence::models::Presence; use crate::tests::test_types::Value; /// One authored config value and what each recursive scan finds in it. pub(super) struct PresenceTestCase { pub(super) description: &'static str, pub(super) value: Value, - pub(super) expected_template: Presence, - pub(super) expected_macro_call: Presence, + pub(super) expected_template: bool, + pub(super) expected_macro_call: bool, } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs index 5c69012b41..51e5962bca 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs @@ -3,9 +3,13 @@ use std::collections::HashMap; use std::fmt::Display; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; + use crate::errors::ConfigError; -use crate::model_validation::_helpers::text::{python_lower, python_strip}; -use crate::model_validation::models::{Rejected, ValidationStop}; +use crate::model_validation::_helpers::durations::{Duration, DurationNumberError, parse_duration}; +use crate::model_validation::constants::SETTING_SNIPPET_INDENT; +use crate::model_validation::models::ValidationStop; use crate::types::{AuthoredNode, NodeKind}; /// Config values by string key; Python's `dict.get` never matches other keys. @@ -13,18 +17,24 @@ pub(crate) struct ConfigView<'a, N> { values: HashMap, /// The model name every validator message starts with. pub(crate) model_name: &'a str, + /// The Python string semantics the validators follow. + pub(crate) python: PythonText, } impl<'a, N: AuthoredNode> ConfigView<'a, N> { /// View the entries of one config mapping for the model `model_name`. - pub(crate) fn new(entries: Vec<(N, N)>, model_name: &'a str) -> Self { + pub(crate) fn new(entries: Vec<(N, N)>, model_name: &'a str, python: PythonText) -> Self { let mut values: HashMap = HashMap::with_capacity(entries.len()); for (key, value) in entries { if let Some(text) = key.text() { values.insert(text, value); } } - Self { values, model_name } + Self { + values, + model_name, + python, + } } /// Return `values[key]` when the key is present, even when its value is `None`. @@ -53,7 +63,7 @@ impl<'a, N: AuthoredNode> ConfigView<'a, N> { pub(crate) fn string(&self, key: &str) -> Result, ValidationStop> { match self.get(key) { None => Ok(None), - Some(value) if value.kind() == NodeKind::Str => Ok(Some(value.text().ok_or(Rejected)?)), + Some(value) if value.kind() == NodeKind::Str => Ok(value.text()), Some(_) => Err(ValidationStop::Error(ConfigError::config_value_type(key))), } } @@ -72,69 +82,63 @@ impl<'a, N: AuthoredNode> ConfigView<'a, N> { pub(crate) fn config_error(&self, text: impl Display) -> ConfigError { ConfigError::compile(format!("model '{}': {text}", self.model_name)) } -} -/// Return the text of a string value, or reject any other type. -pub(crate) fn string_value(value: &N) -> Result { - if value.kind() == NodeKind::Str { - value.text().ok_or(Rejected) - } else { - Err(Rejected) + /// Parse `key`'s duration `text`, raising for a duration SQLBuild cannot read exactly. + pub(crate) fn duration( + &self, + key: &str, + text: &str, + ) -> Result, ValidationStop> { + self.duration_written_as(key, text, "7d") + } + + /// Parse `key`'s duration `text`; a raised error's help shows `key ''`. + pub(crate) fn duration_written_as( + &self, + key: &str, + text: &str, + example: &str, + ) -> Result, ValidationStop> { + parse_duration(self.python, text).map_err(|error| { + let (problem, purpose) = match error { + DurationNumberError::NonAsciiDigits => ( + "uses digits outside ASCII", + format!("write {key} with ASCII digits 0-9"), + ), + DurationNumberError::TooLarge => ( + "has a number larger than a 64-bit integer", + format!("use a {key} whose numbers fit in 64 bits"), + ), + }; + ValidationStop::Error( + self.config_error(format!("{key} '{text}' {problem}")) + .with_help(model_header_help(&purpose, &format!("{key} '{example}'"))), + ) + }) + } + + /// The error for an integer `key` beyond a signed 64-bit integer. + pub(crate) fn integer_too_large(&self, key: &str, value: &N) -> ValidationStop { + ValidationStop::Error( + self.config_error(format!( + "{key} {} is larger than a 64-bit integer", + value.python_str() + )) + .with_help(model_header_help( + &format!("use a {key} that fits in 64 bits"), + &format!("{key} {}", i64::MAX), + )), + ) } } -/// Return the text of a string value, `None` for another type, or reject unreadable text. -pub(crate) fn text_if_string(value: &N) -> Result, Rejected> { +/// Return the text of a string value, or `None` for another type. +pub(crate) fn text_if_string(value: &N) -> Option { if value.kind() == NodeKind::Str { - value.text().map(Some).ok_or(Rejected) + value.text() } else { - Ok(None) - } -} - -/// Return Python's `str(value)` for strings, integers, booleans and `None`, or reject. -pub(crate) fn python_str(value: &N) -> Result { - match value.kind() { - NodeKind::Str => value.text().ok_or(Rejected), - NodeKind::Int { .. } => value - .integer() - .map(|number| number.to_string()) - .ok_or(Rejected), - NodeKind::Bool(flag) => Ok(if flag { "True" } else { "False" }.to_owned()), - NodeKind::Null => Ok("None".to_owned()), - _ => Err(Rejected), - } -} - -/// Return Python's `repr(value)` for printable ASCII strings and scalars, or reject. -pub(crate) fn python_repr(value: &N) -> Result { - if value.kind() != NodeKind::Str { - return python_str(value); + None } - let text = value.text().ok_or(Rejected)?; - python_text_repr(&text) -} - -/// Return Python's `repr(text)` for printable ASCII text, or reject. -pub(crate) fn python_text_repr(text: &str) -> Result { - if !text.bytes().all(|byte| (b' '..=b'~').contains(&byte)) { - return Err(Rejected); - } - let quote = if text.contains('\'') && !text.contains('"') { - '"' - } else { - '\'' - }; - let mut repr = String::with_capacity(text.len() + 2); - repr.push(quote); - for character in text.chars() { - if character == '\\' || character == quote { - repr.push('\\'); - } - repr.push(character); - } - repr.push(quote); - Ok(repr) } /// Return whether `text in vocabulary`. @@ -149,17 +153,14 @@ pub(crate) fn sorted_values(vocabulary: &[&str]) -> String { values.join(", ") } -/// Read `_string_sequence`: a string, or the strings of a list or tuple; reject unreadable text. -pub(crate) fn string_sequence(value: Option<&N>) -> Result, Rejected> { +/// Read `_string_sequence`: a string, or the strings of a list or tuple. +pub(crate) fn string_sequence(value: Option<&N>) -> Vec { match value.map(|node| (node.kind(), node)) { - Some((NodeKind::Str, node)) => string_value(node).map(|text| vec![text]), - Some((NodeKind::List | NodeKind::Tuple, node)) => node - .items() - .iter() - .filter(|item| item.kind() == NodeKind::Str) - .map(string_value) - .collect(), - _ => Ok(Vec::new()), + Some((NodeKind::Str, node)) => node.text().into_iter().collect(), + Some((NodeKind::List | NodeKind::Tuple, node)) => { + node.items().iter().filter_map(text_if_string).collect() + } + _ => Vec::new(), } } @@ -170,15 +171,26 @@ pub(crate) fn has_config_value(value: Option<&N>) -> bool { }) } -/// Check that a value is a string whose Python `strip()` is non-empty, or reject. -pub(crate) fn non_blank_string_check(value: &N) -> Result { - match text_if_string(value)? { - Some(text) => Ok(!python_strip(&text)?.is_empty()), - None => Ok(false), - } +/// Check that a value is a string whose Python `strip()` is non-empty. +pub(crate) fn non_blank_string_check(value: &N) -> bool { + text_if_string(value).is_some_and(|text| !python_strip(&text).is_empty()) +} + +/// Return whether two names are equal under Python's `str.lower()`. +pub(crate) fn same_lowered(left: &str, right: &str) -> bool { + python_lower(left) == python_lower(right) +} + +/// Return Python's `text.lower()`. +pub(crate) fn python_lower(text: &str) -> String { + text.to_lowercase() } -/// Return whether two names are equal under Python's `str.lower()`, or reject. -pub(crate) fn same_lowered(left: &str, right: &str) -> Result { - Ok(python_lower(left)? == python_lower(right)?) +/// Return `model_header_help`: the purpose, then the exact MODEL header entry to add. +pub(crate) fn model_header_help(purpose: &str, entry: &str) -> String { + let indent = SETTING_SNIPPET_INDENT; + format!( + "{purpose}, add this to the MODEL header:\n{indent}MODEL (\n{indent} {entry},\n\ + {indent} ...\n{indent});" + ) } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs index 53f83a1324..8e0420370f 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs @@ -1,44 +1,31 @@ //! `cursor_start` and `cursor_end` checks with the errors Python raises, and their ordering keys. -use crate::model_validation::_helpers::config::{ConfigView, python_text_repr, text_if_string}; -use crate::model_validation::_helpers::text::python_strip; -use crate::model_validation::constants::{ - CLOCK_FIELD_LENGTH, CLOCK_PART_COUNTS, CLOCK_PARTS_WITH_SECONDS, DECIMAL_SPECIAL_PREFIXES, - INTEGER_CURSOR, ISO_DATE_LENGTH, ISO_DATE_SEPARATOR_OFFSETS, LAST_SHIFTED_MONTH, MAX_HOUR, - MAX_MINUTE, MAX_YEAR, MIN_YEAR, TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH, +use std::cmp::Ordering; + +use crate::model_validation::_helpers::config::ConfigView; +use crate::model_validation::_helpers::config::model_header_help; +use crate::model_validation::_helpers::decimals::{DecimalText, FiniteDecimal, decimal_text}; +use crate::model_validation::_helpers::isoformat::{ + IsoDateTime, IsoError, datetime_fromisoformat, epoch_micros, local_micros, supported_micros, + utc_isoformat, }; -use crate::model_validation::models::{Rejected, ValidationStop}; +use crate::model_validation::constants::{INTEGER_CURSOR, TIMESTAMP_CURSOR}; +use crate::model_validation::models::ValidationStop; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; -const MICROS_PER_SECOND: i128 = 1_000_000; -const SECONDS_PER_DAY: i128 = 86_400; -const SECONDS_PER_HOUR: i128 = 3_600; -const SECONDS_PER_MINUTE: i128 = 60; -const CIVIL_EPOCH_OFFSET_DAYS: i128 = 719_468; -const DAYS_PER_ERA: i128 = 146_097; -const FRACTION_DIGITS: [usize; 2] = [3, 6]; -const MICROSECOND_DIGITS: u32 = 6; -/// January's month index when months count from March. -const MARCH_BASED_JANUARY: i128 = 10; -const DECIMAL_SYNTAX: &str = "0123456789+-.eE"; +const MICROS_PER_SECOND: i64 = 1_000_000; /// A bound's `_cursor_contract_key`, or the overflow Python reports while shifting it to UTC. pub(crate) enum BoundKey { - /// UTC microseconds, or the integer itself. - Key(i128), + /// `Decimal(str(value))` of an integer cursor. + Integer(FiniteDecimal), + /// `Decimal(str(utc.timestamp()))`: the float seconds since the Unix epoch. + Timestamp(f64), /// The UTC shift leaves years 1-9999; the text is the value's ISO form at UTC. Overflow(String), } -/// A parsed ISO timestamp. -struct Timestamp { - days: i128, - clock_micros: i128, - /// The UTC offset in seconds, or `None` for a naive value. - offset_seconds: Option, -} - /// Check one bound as `_validate_timestamp_cursor_start` or `_validate_integer_cursor_start` does. pub(crate) fn check_bound( config: &ConfigView<'_, N>, @@ -53,29 +40,27 @@ pub(crate) fn check_bound( } fn check_timestamp_bound(config: &ConfigView<'_, N>, bound: &N) -> Check { - match bound.kind() { - NodeKind::Str => { - let text = bound.text().ok_or(Rejected)?; - parse_timestamp(&text) - .map(|_| ()) - .or_else(|Rejected| invalid_timestamp(config, &text)) - } - NodeKind::Other => Err(ValidationStop::Defer), - _ => Err(config - .error("cursor_start for cursor_type=timestamp must be a string or date-like value")), + if bound.is_date_like() { + return Ok(()); } -} - -/// Raise Python's error for an ASCII string without digits, which no ISO form parses; else defer. -fn invalid_timestamp(config: &ConfigView<'_, N>, text: &str) -> Check { - if !text.is_ascii() || text.bytes().any(|byte| byte.is_ascii_digit()) { - return Err(ValidationStop::Defer); + let Some(text) = bound.text() else { + return Err(config + .error("cursor_start for cursor_type=timestamp must be a string or date-like value")); + }; + match datetime_fromisoformat(&text, config.python.minor_version()) { + Ok(_) => Ok(()), + Err(error) => { + let reason: String = match error { + IsoError::InvalidString => { + format!("Invalid isoformat string: {}", bound.python_repr()) + } + IsoError::Message(message) => message, + }; + Err(config.error(format!( + "cursor_start value '{text}' is not a valid ISO timestamp: {reason}" + ))) + } } - let repr = python_text_repr(text)?; - Err(config.error(format!( - "cursor_start value '{text}' is not a valid ISO timestamp: Invalid isoformat string: \ - {repr}" - ))) } fn check_integer_bound(config: &ConfigView<'_, N>, bound: &N) -> Check { @@ -83,272 +68,85 @@ fn check_integer_bound(config: &ConfigView<'_, N>, bound: &N) - NodeKind::Bool(_) => { Err(config.error("cursor_start for cursor_type=integer must be an integer")) } + NodeKind::Int { .. } if bound.integer().is_none() => { + Err(config.integer_too_large("cursor_start", bound)) + } NodeKind::Int { .. } => Ok(()), NodeKind::Str => { - let text = bound.text().ok_or(Rejected)?; - match plain_decimal(&text) { - Some((_, fraction)) if fraction.bytes().all(|byte| byte == b'0') => Ok(()), - Some(_) => { + let text: String = bound.text().unwrap_or_default(); + match decimal_text(config.python, &text) { + DecimalText::Finite(decimal) if decimal.is_integral() => Ok(()), + DecimalText::Finite(_) => { Err(config.error(format!("cursor_start value '{text}' is not a whole number"))) } - None if certainly_not_decimal(&text)? => Err(config.error(format!( + DecimalText::NonAsciiDigit => Err(ValidationStop::Error( + config + .config_error(format!( + "cursor_start value '{text}' uses digits outside ASCII" + )) + .with_help(model_header_help( + "write cursor_start with ASCII digits 0-9", + "cursor_start '0'", + )), + )), + DecimalText::Special | DecimalText::Invalid => Err(config.error(format!( "cursor_start value '{text}' is not a valid integer" ))), - None => Err(ValidationStop::Defer), } } _ => Err(config.error("cursor_start for cursor_type=integer must be a string or integer")), } } -/// Split `[+-]digits[.digits]` into its signed whole part and fraction digits. -fn plain_decimal(text: &str) -> Option<(&str, &str)> { - let unsigned = text.strip_prefix(['+', '-']).unwrap_or(text); - let (whole, fraction) = unsigned.split_once('.').unwrap_or((unsigned, "")); - let digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit()); - let has_fraction = unsigned.contains('.'); - if whole.is_empty() - || !digits(whole) - || !digits(fraction) - || (has_fraction && fraction.is_empty()) - { - return None; - } - Some((&text[..text.len() - unsigned.len() + whole.len()], fraction)) -} - -/// Return whether `Decimal(text)` certainly raises `InvalidOperation`, or reject non-ASCII text. -fn certainly_not_decimal(text: &str) -> Result { - let stripped: String = python_strip(text)?.replace('_', ""); - if stripped.is_empty() { - return Ok(true); - } - let special = stripped.trim_start_matches(['+', '-']).to_ascii_lowercase(); - Ok(stripped - .chars() - .any(|character| !DECIMAL_SYNTAX.contains(character)) - && !DECIMAL_SPECIAL_PREFIXES - .iter() - .any(|prefix| special.starts_with(prefix))) -} - -/// Return the `_cursor_contract_key` of a bound that passed its checks, or reject. +/// Return the `_cursor_contract_key` of a bound that passed its checks. pub(crate) fn bound_key( + config: &ConfigView<'_, N>, value: &N, cursor_type: &str, -) -> Result { - if let Some(number) = value.integer().filter(|_| cursor_type == INTEGER_CURSOR) { - return Ok(BoundKey::Key(i128::from(number))); - } - let text = text_if_string(value)?.ok_or(Rejected)?; - match cursor_type { - INTEGER_CURSOR => integer_key(&text).map(BoundKey::Key), - TIMESTAMP_CURSOR => timestamp_key(&text), - _ => Err(Rejected), - } -} - -/// Return whether `start < end` as Python compares the keys, or reject when floats may tie. -pub(crate) fn keys_ordered(start: i128, end: i128, cursor_type: &str) -> Result { - if cursor_type != TIMESTAMP_CURSOR || start >= end { - return Ok(start < end); - } - if end - start >= MICROS_PER_SECOND { - Ok(true) - } else { - Err(Rejected) - } -} - -fn integer_key(text: &str) -> Result { - let (whole, fraction) = plain_decimal(text).ok_or(Rejected)?; - if !fraction.bytes().all(|byte| byte == b'0') { - return Err(Rejected); - } - whole - .strip_prefix('+') - .unwrap_or(whole) - .parse() - .map_err(|_| Rejected) -} - -fn timestamp_key(text: &str) -> Result { - let timestamp = parse_timestamp(text)?; - let local_micros = - timestamp.days * SECONDS_PER_DAY * MICROS_PER_SECOND + timestamp.clock_micros; - let utc_micros = local_micros - timestamp.offset_seconds.unwrap_or(0) * MICROS_PER_SECOND; - let day_micros = SECONDS_PER_DAY * MICROS_PER_SECOND; - let first_micros = days_from_civil(MIN_YEAR, 1, 1) * day_micros; - let end_micros = days_from_civil(MAX_YEAR + 1, 1, 1) * day_micros; - if utc_micros < first_micros || utc_micros >= end_micros { - return Ok(BoundKey::Overflow(utc_isoformat(×tamp))); - } - Ok(BoundKey::Key(utc_micros)) -} - -/// Return `parsed.replace(tzinfo=UTC).isoformat()`. -fn utc_isoformat(timestamp: &Timestamp) -> String { - let (year, month, day) = civil_from_days(timestamp.days); - let seconds = timestamp.clock_micros / MICROS_PER_SECOND; - let micros = timestamp.clock_micros % MICROS_PER_SECOND; - let clock = format!( - "{:02}:{:02}:{:02}", - seconds / SECONDS_PER_HOUR, - seconds % SECONDS_PER_HOUR / SECONDS_PER_MINUTE, - seconds % SECONDS_PER_MINUTE - ); - let fraction = if micros == 0 { - String::new() - } else { - format!(".{micros:06}") - }; - format!("{year:04}-{month:02}-{day:02}T{clock}{fraction}+00:00") -} - -fn parse_timestamp(text: &str) -> Result { - if !text.is_ascii() { - return Err(Rejected); - } - let clock_start = ISO_DATE_LENGTH + 1; - let (date, time) = match text.len() { - ISO_DATE_LENGTH => (text, None), - length - if length > clock_start && matches!(text.as_bytes()[ISO_DATE_LENGTH], b'T' | b' ') => - { - (&text[..ISO_DATE_LENGTH], Some(&text[clock_start..])) - } - _ => return Err(Rejected), - }; - let days = date_days(date)?; - let (clock_micros, offset_seconds) = match time { - None => (0, None), - Some(clock) => clock_micros(clock)?, - }; - Ok(Timestamp { - days, - clock_micros, - offset_seconds, - }) -} - -fn date_days(date: &str) -> Result { - let bytes = date.as_bytes(); - let [year_end, month_end] = ISO_DATE_SEPARATOR_OFFSETS; - if bytes[year_end] != b'-' || bytes[month_end] != b'-' { - return Err(Rejected); - } - let year = number(&date[..year_end])?; - let month = number(&date[year_end + 1..month_end])?; - let day = number(&date[month_end + 1..])?; - if year == 0 || !(1..=12).contains(&month) || day == 0 || day > month_days(year, month) { - return Err(Rejected); - } - Ok(days_from_civil(year, month, day)) -} - -fn clock_micros(clock: &str) -> Result<(i128, Option), Rejected> { - let (clock, offset_seconds) = split_offset(clock)?; - let (hms, fraction) = clock - .split_once('.') - .map_or((clock, None), |(hms, fraction)| (hms, Some(fraction))); - let parts: Vec<&str> = hms.split(':').collect(); - if !CLOCK_PART_COUNTS.contains(&parts.len()) - || parts.iter().any(|part| part.len() != CLOCK_FIELD_LENGTH) - || (fraction.is_some() && parts.len() != CLOCK_PARTS_WITH_SECONDS) - { - return Err(Rejected); - } - let hour = number(parts[0])?; - let minute = number(parts[1])?; - let second = parts.get(2).map_or(Ok(0), |part| number(part))?; - if hour > MAX_HOUR || minute > MAX_MINUTE || second > MAX_MINUTE { - return Err(Rejected); - } - let fraction_micros = match fraction { - None => 0, - Some(digits) if FRACTION_DIGITS.contains(&digits.len()) => { - let scale = MICROSECOND_DIGITS - u32::try_from(digits.len()).map_err(|_| Rejected)?; - number(digits)? * 10_i128.pow(scale) +) -> Option { + if cursor_type == INTEGER_CURSOR { + if let Some(number) = value.integer() { + return Some(BoundKey::Integer(FiniteDecimal::integer(number))); } - Some(_) => return Err(Rejected), - }; - let seconds = hour * SECONDS_PER_HOUR + minute * SECONDS_PER_MINUTE + second; - Ok(( - seconds * MICROS_PER_SECOND + fraction_micros, - offset_seconds, - )) -} - -fn split_offset(clock: &str) -> Result<(&str, Option), Rejected> { - if let Some(clock) = clock.strip_suffix('Z') { - return Ok((clock, Some(0))); + return match decimal_text(config.python, &value.text()?) { + DecimalText::Finite(decimal) => Some(BoundKey::Integer(decimal)), + _ => None, + }; } - let Some(sign_index) = clock.rfind(['+', '-']) else { - return Ok((clock, None)); - }; - let offset = &clock[sign_index + 1..]; - let bytes = offset.as_bytes(); - if offset.len() != UTC_OFFSET_LENGTH || bytes[CLOCK_FIELD_LENGTH] != b':' { - return Err(Rejected); - } - let hours = number(&offset[..CLOCK_FIELD_LENGTH])?; - let minutes = number(&offset[CLOCK_FIELD_LENGTH + 1..])?; - if hours > MAX_HOUR || minutes > MAX_MINUTE { - return Err(Rejected); - } - let magnitude = hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE; - let sign = if clock.as_bytes()[sign_index] == b'-' { - -1 - } else { - 1 - }; - Ok((&clock[..sign_index], Some(sign * magnitude))) -} - -fn number(digits: &str) -> Result { - if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { - return Err(Rejected); - } - digits.parse().map_err(|_| Rejected) -} - -fn month_days(year: i128, month: i128) -> i128 { - match month { - 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29, - 2 => 28, - 4 | 6 | 9 | 11 => 30, - _ => 31, - } -} - -fn days_from_civil(year: i128, month: i128, day: i128) -> i128 { - let shifted_year = if month <= LAST_SHIFTED_MONTH { - year - 1 - } else { - year - }; - let era = shifted_year.div_euclid(400); - let year_of_era = shifted_year - era * 400; - let shifted_month = (month + 9) % 12; - let day_of_year = (153 * shifted_month + 2) / 5 + day - 1; - let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; - era * DAYS_PER_ERA + day_of_era - CIVIL_EPOCH_OFFSET_DAYS -} - -fn civil_from_days(days: i128) -> (i128, i128, i128) { - let shifted = days + CIVIL_EPOCH_OFFSET_DAYS; - let era = shifted.div_euclid(DAYS_PER_ERA); - let day_of_era = shifted - era * DAYS_PER_ERA; - let year_of_era = - (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; - let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); - let shifted_month = (5 * day_of_year + 2) / 153; - let day = day_of_year - (153 * shifted_month + 2) / 5 + 1; - let month = if shifted_month < MARCH_BASED_JANUARY { - shifted_month + 3 + let text: String = if value.is_date_like() { + value.python_str() } else { - shifted_month - 9 + value.text()? }; - let year = year_of_era + era * 400 + i128::from(month <= LAST_SHIFTED_MONTH); - (year, month, day) + datetime_fromisoformat(&text, config.python.minor_version()) + .ok() + .map(|parsed| timestamp_key(&parsed)) +} + +/// Return whether `start < end` as Python compares the two `Decimal` keys. +pub(crate) fn keys_ordered(start: &BoundKey, end: &BoundKey) -> bool { + match (start, end) { + (BoundKey::Integer(start), BoundKey::Integer(end)) => start.compare(end) == Ordering::Less, + (BoundKey::Timestamp(start), BoundKey::Timestamp(end)) => start < end, + _ => true, + } +} + +/// The UTC float timestamp Python compares, or the overflow of its shift to UTC. +fn timestamp_key(parsed: &IsoDateTime) -> BoundKey { + let utc_micros: i64 = local_micros(parsed) - parsed.offset_micros.unwrap_or(0); + let (first, end) = supported_micros(); + if utc_micros < first || utc_micros >= end { + return BoundKey::Overflow(utc_isoformat(parsed)); + } + let since_epoch: i64 = utc_micros - epoch_micros(); + let sign: &str = if since_epoch < 0 { "-" } else { "" }; + let magnitude: u64 = since_epoch.unsigned_abs(); + let seconds: u64 = magnitude / MICROS_PER_SECOND.unsigned_abs(); + let micros: u64 = magnitude % MICROS_PER_SECOND.unsigned_abs(); + BoundKey::Timestamp( + format!("{sign}{seconds}.{micros:06}") + .parse::() + .unwrap_or(f64::NAN), + ) } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/decimals.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/decimals.rs new file mode 100644 index 0000000000..f6768a9bde --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/decimals.rs @@ -0,0 +1,175 @@ +//! `decimal.Decimal(text)` as CPython's `_decimal` reads a string, and exact comparison. + +use std::cmp::Ordering; + +use sqlbuild_core::text::main::is_python_decimal::is_python_decimal; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; + +/// A finite decimal: its sign, its digits without leading zeros, and its exponent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct FiniteDecimal { + negative: bool, + digits: String, + exponent: i64, +} + +/// What `Decimal(text)` gives. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum DecimalText { + Finite(FiniteDecimal), + /// `Infinity`, `NaN` or `sNaN`, which no integer equals. + Special, + /// `InvalidOperation`: the text is no decimal. + Invalid, + /// A digit outside ASCII that Python would read; SQLBuild asks for ASCII digits. + NonAsciiDigit, +} + +const SPECIAL_NAMES: [&str; 4] = ["inf", "infinity", "nan", "snan"]; + +/// Read `text` as `Decimal(text)`: Python whitespace around it and every `_` are ignored. +pub(crate) fn decimal_text(python: PythonText, text: &str) -> DecimalText { + let trimmed: &str = text.trim_matches(is_python_space); + if trimmed + .chars() + .any(|character| !character.is_ascii() && is_python_decimal(python, character)) + { + return DecimalText::NonAsciiDigit; + } + let ascii: String = trimmed + .chars() + .filter(|character| *character != '_') + .collect(); + if !ascii.is_ascii() { + return DecimalText::Invalid; + } + parse_ascii(&ascii) +} + +fn parse_ascii(text: &str) -> DecimalText { + let unsigned: &str = text.strip_prefix(['+', '-']).unwrap_or(text); + let negative: bool = text.starts_with('-'); + let lowered: String = unsigned.to_ascii_lowercase(); + let special_payload = |name: &str| { + lowered + .strip_prefix(name) + .is_some_and(|payload| payload.bytes().all(|byte| byte.is_ascii_digit())) + }; + if SPECIAL_NAMES[..2].contains(&lowered.as_str()) || special_payload("snan") { + return DecimalText::Special; + } + if special_payload("nan") { + return DecimalText::Special; + } + let (mantissa, exponent_text) = match unsigned.find(['e', 'E']) { + Some(at) => (&unsigned[..at], Some(&unsigned[at + 1..])), + None => (unsigned, None), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + let all_digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit()); + if (whole.is_empty() && fraction.is_empty()) || !all_digits(whole) || !all_digits(fraction) { + return DecimalText::Invalid; + } + let exponent: i64 = match exponent_text { + None => 0, + Some(exponent) => { + let digits: &str = exponent.strip_prefix(['+', '-']).unwrap_or(exponent); + if digits.is_empty() || !all_digits(digits) { + return DecimalText::Invalid; + } + match exponent.parse::() { + Ok(value) => value, + Err(_) => return DecimalText::Invalid, + } + } + }; + let Some(exponent) = i64::try_from(fraction.len()) + .ok() + .and_then(|places| exponent.checked_sub(places)) + else { + return DecimalText::Invalid; + }; + let digits: String = format!("{whole}{fraction}") + .trim_start_matches('0') + .to_owned(); + DecimalText::Finite(FiniteDecimal { + negative: negative && !digits.is_empty(), + digits, + exponent, + }) +} + +impl FiniteDecimal { + /// The decimal of a Python `int`. + pub(crate) fn integer(value: i64) -> Self { + let digits: String = value + .unsigned_abs() + .to_string() + .trim_start_matches('0') + .to_owned(); + Self { + negative: value < 0, + digits, + exponent: 0, + } + } + + /// Whether `value == int(value)`. + pub(crate) fn is_integral(&self) -> bool { + self.exponent >= 0 + || usize::try_from(self.exponent.unsigned_abs()).is_ok_and(|places| { + self.digits.len() <= places && self.digits.bytes().all(|byte| byte == b'0') + || self.digits.len() > places + && self.digits[self.digits.len() - places..] + .bytes() + .all(|byte| byte == b'0') + }) + } + + /// Compare two decimals by value. + pub(crate) fn compare(&self, other: &Self) -> Ordering { + match (self.digits.is_empty(), other.digits.is_empty()) { + (true, true) => return Ordering::Equal, + (true, false) => { + return if other.negative { + Ordering::Greater + } else { + Ordering::Less + }; + } + (false, true) => { + return if self.negative { + Ordering::Less + } else { + Ordering::Greater + }; + } + (false, false) => {} + } + if self.negative != other.negative { + return if self.negative { + Ordering::Less + } else { + Ordering::Greater + }; + } + let magnitude: Ordering = self.magnitude_order(other); + if self.negative { + magnitude.reverse() + } else { + magnitude + } + } + + fn magnitude_order(&self, other: &Self) -> Ordering { + let adjusted = |decimal: &Self| { + i128::from(decimal.exponent) + i128::try_from(decimal.digits.len()).unwrap_or(0) + }; + adjusted(self).cmp(&adjusted(other)).then_with(|| { + let left: &str = self.digits.trim_end_matches('0'); + let right: &str = other.digits.trim_end_matches('0'); + left.cmp(right) + }) + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs index 46c7a50c5f..9a2e9237a5 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs @@ -1,7 +1,9 @@ -//! `Duration.parse` for ASCII text; other text is left to Python. +//! `Duration.parse`: `^(?:(\d+)y)?(?:(\d+)mo)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$`. + +use sqlbuild_core::text::main::is_python_decimal::is_python_decimal; +use sqlbuild_core::text::models::PythonText; use crate::model_validation::constants::DAY_UNIT_INDEX; -use crate::model_validation::models::Rejected; const MONTHS_PER_YEAR: u128 = 12; const UNIT_SECONDS: [u128; 4] = [86_400, 3_600, 60, 1]; @@ -18,6 +20,15 @@ pub(crate) struct Duration { amounts: [u128; 6], } +/// A duration Python reads but SQLBuild rejects. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DurationNumberError { + /// An amount uses digits outside ASCII. + NonAsciiDigits, + /// An amount does not fit in a signed 64-bit integer. + TooLarge, +} + impl Duration { /// Return whether only days are non-zero, like `units == {"d"}`. pub(crate) fn is_whole_days(&self) -> bool { @@ -33,42 +44,52 @@ impl Duration { } } -/// Return `Duration.parse(text)`: a duration, `None` when it does not parse, or a rejection. -pub(crate) fn parse_duration(text: &str) -> Result, Rejected> { - if !text.is_ascii() || text.contains('\n') { - return Err(Rejected); - } +/// Return `Duration.parse(text)`: a duration, `None` when it does not parse, or why SQLBuild +/// rejects a duration Python would read. +pub(crate) fn parse_duration( + python: PythonText, + text: &str, +) -> Result, DurationNumberError> { + let text: &str = text.strip_suffix('\n').unwrap_or(text); + let is_digit = |character: char| is_python_decimal(python, character); let mut amounts = [0_u128; 6]; let mut next_unit = 0; let mut rest = text; + let mut non_ascii = false; while !rest.is_empty() { - let digits = rest.bytes().take_while(u8::is_ascii_digit).count(); + let digits: usize = rest + .char_indices() + .find(|(_, character)| !is_digit(*character)) + .map_or(rest.len(), |(at, _)| at); if digits == 0 { return Ok(None); } - let amount: u128 = rest[..digits].parse().map_err(|_| Rejected)?; - rest = &rest[digits..]; - let Some(unit) = (next_unit..UNITS.len()).find(|index| rest.starts_with(UNITS[*index])) + let Some(unit) = + (next_unit..UNITS.len()).find(|index| rest[digits..].starts_with(UNITS[*index])) else { return Ok(None); }; - amounts[unit] = amount; - rest = &rest[UNITS[unit].len()..]; + non_ascii |= !rest[..digits].is_ascii(); + amounts[unit] = rest[..digits] + .parse::() + .ok() + .filter(|amount| i64::try_from(*amount).is_ok()) + .unwrap_or(u128::MAX); + rest = &rest[digits + UNITS[unit].len()..]; next_unit = unit + 1; } - let total_months = amounts[0] - .checked_mul(MONTHS_PER_YEAR) - .and_then(|months| months.checked_add(amounts[1])) - .ok_or(Rejected)?; + if non_ascii { + return Err(DurationNumberError::NonAsciiDigits); + } + if amounts.contains(&u128::MAX) { + return Err(DurationNumberError::TooLarge); + } + let total_months = amounts[0] * MONTHS_PER_YEAR + amounts[1]; let fixed_seconds = amounts[DAY_UNIT_INDEX..] .iter() .zip(UNIT_SECONDS) - .try_fold(0_u128, |total, (amount, seconds)| { - amount - .checked_mul(seconds) - .and_then(|value| total.checked_add(value)) - }) - .ok_or(Rejected)?; + .map(|(amount, seconds)| amount * seconds) + .sum(); if total_months == 0 && fixed_seconds == 0 { return Ok(None); } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs index 482751c1b6..02bf5cea11 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs @@ -2,15 +2,16 @@ use std::collections::HashSet; +use sqlbuild_core::text::main::python_strip::python_strip; + use crate::model_validation::_helpers::config::{ ConfigView, has_config_value, one_of, sorted_values, string_sequence, }; +use crate::model_validation::_helpers::config::{model_header_help, python_lower}; use crate::model_validation::_helpers::cursor_bounds::{ BoundKey, bound_key, check_bound, keys_ordered, }; -use crate::model_validation::_helpers::durations::parse_duration; use crate::model_validation::_helpers::microbatch::check_incremental_batching; -use crate::model_validation::_helpers::text::{model_header_help, python_lower, python_strip}; use crate::model_validation::constants::{ APPEND_STRATEGY, BOUNDED_REPLAY_PREFIX, CURSOR_GRAIN_BATCH_SIZES, CURSOR_INPUT_KEYS, CURSOR_POLICY_DISABLED, CURSOR_TYPES, DELETE_INSERT_STRATEGY, EFFECTIVE_BATCH_SIZE, @@ -19,7 +20,7 @@ use crate::model_validation::constants::{ REPLAY_ON_CHANGE_EXAMPLE, REPLAY_ON_CHANGE_VALID_VALUES, TABLE_MATERIALIZATION, TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, ZERO_DAY_DURATION, }; -use crate::model_validation::models::{ModelValidationFacts, Rejected, ValidationStop}; +use crate::model_validation::models::{ModelValidationFacts, ValidationStop}; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; @@ -133,7 +134,7 @@ fn check_core(config: &ConfigView<'_, N>, values: &IncrementalV (REPLAY_ON_CHANGE_KEY, REPLAY_ON_CHANGE_EXAMPLE), ] { if let Some(value) = config.get(key) - && let Some(problem) = change_policy_problem(key, value)? + && let Some(problem) = change_policy_problem(config, key, value)? { let help = model_header_help(&format!("use a valid {key}"), &format!("{key} {example}")); @@ -157,9 +158,10 @@ fn check_core(config: &ConfigView<'_, N>, values: &IncrementalV /// Return `change_policy_problem`: why a change-policy value is invalid, or `None`. fn change_policy_problem( + config: &ConfigView<'_, N>, key: &str, value: &N, -) -> Result, Rejected> { +) -> Result, ValidationStop> { let valid = if key == ON_SCHEMA_CHANGE_KEY { sorted_values(&ON_SCHEMA_CHANGE_POLICIES) } else { @@ -170,7 +172,7 @@ fn change_policy_problem( "{key} must be a string; valid values: {valid}" ))); } - let text = value.text().ok_or(Rejected)?; + let text: String = value.text().unwrap_or_default(); if key == ON_SCHEMA_CHANGE_KEY && one_of(&text, &ON_SCHEMA_CHANGE_POLICIES) { return Ok(None); } @@ -179,8 +181,11 @@ fn change_policy_problem( return Ok(None); } if let Some(rest) = text.strip_prefix(BOUNDED_REPLAY_PREFIX) { - let duration = python_strip(rest)?; - if parse_duration(duration)?.is_some() { + let duration = python_strip(rest); + if config + .duration_written_as(key, duration, REPLAY_ON_CHANGE_EXAMPLE)? + .is_some() + { return Ok(None); } return Ok(Some(format!( @@ -256,7 +261,9 @@ fn check_cursor_rules( { let start_key = ordering_key(config, start, cursor_type, "cursor_start")?; let end_key = ordering_key(config, end, cursor_type, "cursor_end")?; - if !keys_ordered(start_key, end_key, cursor_type)? { + if let (Some(start_key), Some(end_key)) = (start_key, end_key) + && !keys_ordered(&start_key, &end_key) + { return Err(config.error("cursor_start must be before exclusive cursor_end")); } } @@ -268,11 +275,10 @@ fn ordering_key( bound: &N, cursor_type: &str, key: &str, -) -> Result { - match bound_key(bound, cursor_type)? { - BoundKey::Key(value) => Ok(value), - BoundKey::Overflow(utc_value) => { - let text = bound.text().ok_or(Rejected)?; +) -> Result, ValidationStop> { + match bound_key(config, bound, cursor_type) { + Some(BoundKey::Overflow(utc_value)) => { + let text = bound.python_str(); let help = model_header_help( &format!("keep {key} within years 1-9999 in UTC"), &format!("{key} '{utc_value}'"), @@ -285,6 +291,7 @@ fn ordering_key( .with_help(help), )) } + bound_key => Ok(bound_key), } } @@ -319,7 +326,7 @@ fn check_cursor_safety( } for key in CURSOR_SAFETY_DURATION_KEYS { if let Some(value) = config.get(key) - && !safety_duration_check(value)? + && !safety_duration_check(config, key, value)? { return Err(config.error(format!("{key} must be a duration or 'disabled'"))); } @@ -328,28 +335,27 @@ fn check_cursor_safety( let Some(value) = config.get(key) else { continue; }; - let valid = match value.kind() { - NodeKind::Str => FUTURE_CURSOR_ACTIONS - .iter() - .any(|action| value.is_text(action)), - NodeKind::Int { .. } | NodeKind::Bool(_) => false, - _ => return Err(ValidationStop::Defer), - }; - if !valid { + if !FUTURE_CURSOR_ACTIONS + .iter() + .any(|action| value.is_text(action)) + { return Err(config.error(format!("{key} must be one of: cap, error"))); } } Ok(()) } -fn safety_duration_check(value: &N) -> Result { - if value.kind() != NodeKind::Str { +fn safety_duration_check( + config: &ConfigView<'_, N>, + key: &str, + value: &N, +) -> Result { + let Some(text) = value.text() else { return Ok(false); - } - let text = value.text().ok_or(Rejected)?; + }; Ok(text == CURSOR_POLICY_DISABLED || text == ZERO_DAY_DURATION - || parse_duration(&text)?.is_some()) + || config.duration(key, &text)?.is_some()) } fn check_write_strategy( @@ -368,22 +374,19 @@ fn check_write_strategy( if !is_non_empty_string_list(excluded) { return Err(config.error("merge_exclude_columns must be a list of non-empty strings")); } - let excluded = string_sequence(Some(excluded))?; + let excluded = string_sequence(Some(excluded)); if strategy != Some(MERGE_STRATEGY) { return Err(config.error("merge_exclude_columns requires incremental_strategy=merge")); } - let lowered = excluded - .iter() - .map(|column| python_lower(column)) - .collect::, Rejected>>()?; + let lowered: Vec = excluded.iter().map(|column| python_lower(column)).collect(); let distinct: HashSet<&String> = lowered.iter().collect(); if distinct.len() != lowered.len() { return Err(config.error("merge_exclude_columns contains duplicate columns")); } - let unique_columns = string_sequence(values.unique_key.as_ref())? + let unique_columns: HashSet = string_sequence(values.unique_key.as_ref()) .iter() .map(|column| python_lower(column)) - .collect::, Rejected>>()?; + .collect(); let overlap: Vec<&str> = excluded .iter() .zip(&lowered) @@ -425,13 +428,13 @@ fn check_contract_columns( require_declared( config, "unique_key", - &string_sequence(values.unique_key.as_ref())?, + &string_sequence(values.unique_key.as_ref()), &declared, )?; require_declared( config, "merge_exclude_columns", - &string_sequence(values.merge_exclude_columns.as_ref())?, + &string_sequence(values.merge_exclude_columns.as_ref()), &declared, ) } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/isoformat.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/isoformat.rs new file mode 100644 index 0000000000..6a7616449f --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/isoformat.rs @@ -0,0 +1,478 @@ +//! CPython's `datetime.fromisoformat`, following the C parser of each supported release. +//! +//! The C parser reads a NUL-terminated buffer; reads past the end of the text see that NUL. + +/// One parsed date and time, before CPython's range checks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct IsoDateTime { + pub(crate) year: i64, + pub(crate) month: i64, + pub(crate) day: i64, + pub(crate) hour: i64, + pub(crate) minute: i64, + pub(crate) second: i64, + pub(crate) microsecond: i64, + /// The UTC offset in microseconds, or `None` for a naive value. + pub(crate) offset_micros: Option, +} + +/// Why `fromisoformat` raises `ValueError`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum IsoError { + /// `Invalid isoformat string: `. + InvalidString, + /// Any other `ValueError` message. + Message(String), +} + +const MIN_YEAR: i64 = 1; +const MAX_YEAR: i64 = 9999; +const MICROS_PER_SECOND: i64 = 1_000_000; +const MICROS_PER_DAY: i64 = 86_400 * MICROS_PER_SECOND; +const DAYS_BEFORE_MONTH: [i64; 13] = [0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; +const FRACTION_CORRECTION: [i64; 5] = [100_000, 10_000, 1_000, 100, 10]; +const FRACTION_DIGITS: usize = 6; +const PYTHON_313: u8 = 13; +const PYTHON_314: u8 = 14; + +/// Parse `text` as `datetime.fromisoformat` does on CPython 3.`minor`. +pub(crate) fn datetime_fromisoformat(text: &str, minor: u8) -> Result { + let bytes: &[u8] = text.as_bytes(); + let length: usize = bytes.len(); + let separator: Option = isoformat_separator(bytes); + let (year, month, day) = + parse_date(bytes, separator.unwrap_or(usize::MAX)).map_err(|_| IsoError::InvalidString)?; + let mut time = TimeFields::default(); + let mut has_offset = false; + if let Some(separator) = separator.filter(|separator| length > *separator) { + let start: usize = separator + utf8_width(byte_at(bytes, separator)); + has_offset = parse_time(bytes, start, length, minor, &mut time) + .map_err(|_| IsoError::InvalidString)?; + } + let offset_micros: Option = has_offset + .then(|| time.offset_seconds * MICROS_PER_SECOND + time.offset_micros) + .map(|offset| checked_offset(offset, time.offset_seconds, minor)) + .transpose()?; + let mut parsed = IsoDateTime { + year, + month, + day, + hour: time.hour, + minute: time.minute, + second: time.second, + microsecond: time.microsecond, + offset_micros, + }; + if minor >= PYTHON_314 && parsed.hour == 24 && (1..=12).contains(&parsed.month) { + roll_iso_midnight(&mut parsed)?; + } + check_ranges(&parsed)?; + Ok(parsed) +} + +/// Microseconds since 0001-01-01 00:00 of the local wall time, ignoring any offset. +pub(crate) fn local_micros(parsed: &IsoDateTime) -> i64 { + let days: i64 = ymd_to_ordinal(parsed.year, parsed.month, parsed.day) - 1; + let seconds: i64 = parsed.hour * 3_600 + parsed.minute * 60 + parsed.second; + days * MICROS_PER_DAY + seconds * MICROS_PER_SECOND + parsed.microsecond +} + +/// The microseconds since 0001-01-01 of the earliest and the first excluded instant. +pub(crate) fn supported_micros() -> (i64, i64) { + ( + 0, + ymd_to_ordinal(MAX_YEAR + 1, 1, 1).saturating_sub(1) * MICROS_PER_DAY, + ) +} + +/// Microseconds from 0001-01-01 to the Unix epoch. +pub(crate) fn epoch_micros() -> i64 { + (ymd_to_ordinal(1970, 1, 1) - 1) * MICROS_PER_DAY +} + +/// `parsed.replace(tzinfo=UTC).isoformat()`. +pub(crate) fn utc_isoformat(parsed: &IsoDateTime) -> String { + let fraction: String = if parsed.microsecond == 0 { + String::new() + } else { + format!(".{:06}", parsed.microsecond) + }; + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}{fraction}+00:00", + parsed.year, parsed.month, parsed.day, parsed.hour, parsed.minute, parsed.second + ) +} + +#[derive(Default)] +struct TimeFields { + hour: i64, + minute: i64, + second: i64, + microsecond: i64, + offset_seconds: i64, + offset_micros: i64, +} + +fn byte_at(bytes: &[u8], index: usize) -> u8 { + bytes.get(index).copied().unwrap_or(0) +} + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +/// The UTF-8 length of the character starting with `lead`, as CPython skips the separator. +fn utf8_width(lead: u8) -> usize { + if lead & 0x80 == 0 { + 1 + } else { + match lead & 0xf0 { + 0xe0 => 3, + 0xf0 => 4, + _ => 2, + } + } +} + +/// `_find_isoformat_datetime_separator`; `None` stands for its -1. +fn isoformat_separator(bytes: &[u8]) -> Option { + let length: usize = bytes.len(); + if length == 7 { + return Some(7); + } + if byte_at(bytes, 4) == b'-' { + if byte_at(bytes, 5) != b'W' { + return Some(10); + } + if length < 8 { + return None; + } + if length > 8 && byte_at(bytes, 8) == b'-' { + if length == 9 { + return None; + } + if length > 10 && is_digit(byte_at(bytes, 10)) { + return Some(8); + } + return Some(10); + } + return Some(8); + } + if byte_at(bytes, 4) != b'W' { + return Some(8); + } + let digits_end: usize = (7..length) + .find(|index| !is_digit(byte_at(bytes, *index))) + .unwrap_or(length.max(7)); + if digits_end < 9 { + return Some(digits_end); + } + Some(if digits_end.is_multiple_of(2) { 7 } else { 8 }) +} + +/// `parse_digits`: read `count` digits at `start` into `value`; `None` at a non-digit. +fn parse_digits(bytes: &[u8], start: usize, count: usize, value: &mut i64) -> Option { + for index in start..start + count { + let byte: u8 = byte_at(bytes, index); + if !is_digit(byte) { + return None; + } + *value = *value * 10 + i64::from(byte - b'0'); + } + Some(start + count) +} + +/// `parse_isoformat_date`, reading the date in the first `length` bytes. +fn parse_date(bytes: &[u8], length: usize) -> Result<(i64, i64, i64), i32> { + let mut year: i64 = 0; + let mut position: usize = parse_digits(bytes, 0, 4, &mut year).ok_or(-1)?; + let uses_separator: bool = byte_at(bytes, position) == b'-'; + if uses_separator { + position += 1; + } + if byte_at(bytes, position) == b'W' { + let mut week: i64 = 0; + let mut weekday: i64 = 0; + position = parse_digits(bytes, position + 1, 2, &mut week).ok_or(-3)?; + if position < length { + if uses_separator { + if byte_at(bytes, position) != b'-' { + return Err(-2); + } + position += 1; + } + parse_digits(bytes, position, 1, &mut weekday).ok_or(-4)?; + } else { + weekday = 1; + } + return iso_to_ymd(year, week, weekday); + } + let mut month: i64 = 0; + let mut day: i64 = 0; + position = parse_digits(bytes, position, 2, &mut month).ok_or(-1)?; + if uses_separator { + if byte_at(bytes, position) != b'-' { + return Err(-2); + } + position += 1; + } + parse_digits(bytes, position, 2, &mut day).ok_or(-1)?; + Ok((year, month, day)) +} + +/// `parse_hh_mm_ss_ff` over `[start, end)`; `Ok(true)` when text follows the time. +fn parse_clock( + bytes: &[u8], + start: usize, + end: usize, + minor: u8, + fields: &mut [i64; 4], +) -> Result { + *fields = [0; 4]; + let mut position: usize = start; + let mut has_separator: bool = true; + let mut component: usize = 0; + while component < 3 { + position = parse_digits(bytes, position, 2, &mut fields[component]).ok_or(-3)?; + let character: u8 = byte_at(bytes, position); + position += 1; + if component == 0 { + has_separator = character == b':'; + } + let decimal_mark: bool = matches!(character, b'.' | b','); + if minor >= PYTHON_313 && decimal_mark { + if minor >= PYTHON_314 && component < 2 { + return Err(-3); + } + if position >= end { + return Err(-3); + } + break; + } + if position >= end { + return Ok(character != 0); + } + if has_separator && character == b':' { + if minor >= PYTHON_314 && component == 2 { + return Err(-4); + } + component += 1; + continue; + } + if decimal_mark { + break; + } + if !has_separator { + position -= 1; + } else { + return Err(-4); + } + component += 1; + } + let to_parse: usize = end.saturating_sub(position).min(FRACTION_DIGITS); + position = parse_digits(bytes, position, to_parse, &mut fields[3]).ok_or(-3)?; + if let Some(correction) = to_parse + .checked_sub(1) + .and_then(|at| FRACTION_CORRECTION.get(at)) + { + fields[3] *= correction; + } + while is_digit(byte_at(bytes, position)) { + position += 1; + } + Ok(byte_at(bytes, position) != 0) +} + +/// `parse_isoformat_time` over `[start, end)`; `Ok(true)` when it read a UTC offset. +fn parse_time( + bytes: &[u8], + start: usize, + end: usize, + minor: u8, + time: &mut TimeFields, +) -> Result { + let mut offset_at: usize = start; + loop { + if matches!(byte_at(bytes, offset_at), b'Z' | b'+' | b'-') { + break; + } + offset_at += 1; + if offset_at >= end { + break; + } + } + let mut clock: [i64; 4] = [0; 4]; + let trailing: bool = parse_clock(bytes, start, offset_at, minor, &mut clock)?; + [time.hour, time.minute, time.second, time.microsecond] = clock; + if offset_at == end { + return if trailing { Err(-5) } else { Ok(false) }; + } + if byte_at(bytes, offset_at) == b'Z' { + return if byte_at(bytes, offset_at + 1) == 0 { + Ok(true) + } else { + Err(-5) + }; + } + let sign: i64 = if byte_at(bytes, offset_at) == b'-' { + -1 + } else { + 1 + }; + let mut offset: [i64; 4] = [0; 4]; + let offset_trailing: bool = parse_clock(bytes, offset_at + 1, end, minor, &mut offset)?; + time.offset_seconds = sign * (offset[0] * 3_600 + offset[1] * 60 + offset[2]); + time.offset_micros = sign * offset[3]; + if offset_trailing { Err(-5) } else { Ok(true) } +} + +/// `tzinfo_from_isoformat_results`: `None` for UTC, else the checked offset. +fn checked_offset(offset: i64, offset_seconds: i64, minor: u8) -> Result { + let utc: bool = if minor >= PYTHON_313 { + offset == 0 + } else { + offset_seconds == 0 + }; + if utc { + return Ok(0); + } + if offset <= -MICROS_PER_DAY || offset >= MICROS_PER_DAY { + return Err(IsoError::Message(format!( + "offset must be a timedelta strictly between -timedelta(hours=24) and \ + timedelta(hours=24), not {}.", + timedelta_repr(offset) + ))); + } + Ok(offset) +} + +/// `repr(timedelta(microseconds=micros))`. +fn timedelta_repr(micros: i64) -> String { + let days: i64 = micros.div_euclid(MICROS_PER_DAY); + let rest: i64 = micros.rem_euclid(MICROS_PER_DAY); + let fields: Vec = [ + ("days", days), + ("seconds", rest / MICROS_PER_SECOND), + ("microseconds", rest % MICROS_PER_SECOND), + ] + .iter() + .filter(|(_, value)| *value != 0) + .map(|(name, value)| format!("{name}={value}")) + .collect(); + if fields.is_empty() { + return "datetime.timedelta(0)".to_owned(); + } + format!("datetime.timedelta({})", fields.join(", ")) +} + +/// Python 3.14's `24:00`: midnight of the next day, when the rest of the clock is zero. +fn roll_iso_midnight(parsed: &mut IsoDateTime) -> Result<(), IsoError> { + let month_days: i64 = days_in_month(parsed.year, parsed.month); + if parsed.day > month_days { + return Ok(()); + } + if parsed.minute != 0 || parsed.second != 0 || parsed.microsecond != 0 { + return Err(IsoError::Message( + "minute, second, and microsecond must be 0 when hour is 24".to_owned(), + )); + } + parsed.hour = 0; + parsed.day += 1; + if parsed.day > month_days { + parsed.day = 1; + parsed.month += 1; + if parsed.month > 12 { + parsed.month = 1; + parsed.year += 1; + } + } + Ok(()) +} + +/// `check_date_args` then `check_time_args`. +fn check_ranges(parsed: &IsoDateTime) -> Result<(), IsoError> { + let message: Option = if !(MIN_YEAR..=MAX_YEAR).contains(&parsed.year) { + Some(format!("year {} is out of range", parsed.year)) + } else if !(1..=12).contains(&parsed.month) { + Some("month must be in 1..12".to_owned()) + } else if parsed.day < 1 || parsed.day > days_in_month(parsed.year, parsed.month) { + Some("day is out of range for month".to_owned()) + } else if !(0..=23).contains(&parsed.hour) { + Some("hour must be in 0..23".to_owned()) + } else if !(0..=59).contains(&parsed.minute) { + Some("minute must be in 0..59".to_owned()) + } else if !(0..=59).contains(&parsed.second) { + Some("second must be in 0..59".to_owned()) + } else { + None + }; + message.map_or(Ok(()), |message| Err(IsoError::Message(message))) +} + +/// `iso_to_ymd`, with `parse_isoformat_date`'s codes for its failures. +fn iso_to_ymd(year: i64, week: i64, weekday: i64) -> Result<(i64, i64, i64), i32> { + if !(MIN_YEAR..=MAX_YEAR).contains(&year) { + return Err(-7); + } + if week <= 0 || week >= 53 { + let first_weekday: i64 = (ymd_to_ordinal(year, 1, 1) + 6) % 7; + let long_year: bool = first_weekday == 3 || (first_weekday == 2 && is_leap(year)); + if week != 53 || !long_year { + return Err(-5); + } + } + if weekday <= 0 || weekday >= 8 { + return Err(-6); + } + let first_day: i64 = ymd_to_ordinal(year, 1, 1); + let first_weekday: i64 = (first_day + 6) % 7; + let week1_monday: i64 = first_day - first_weekday + if first_weekday > 3 { 7 } else { 0 }; + Ok(ordinal_to_ymd(week1_monday + (week - 1) * 7 + weekday - 1)) +} + +fn is_leap(year: i64) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in_month(year: i64, month: i64) -> i64 { + match month { + 2 if is_leap(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +fn ymd_to_ordinal(year: i64, month: i64, day: i64) -> i64 { + let before_year: i64 = year - 1; + let days_before_year: i64 = + before_year * 365 + before_year / 4 - before_year / 100 + before_year / 400; + let index: usize = usize::try_from(month.clamp(1, 12)).unwrap_or(1); + days_before_year + DAYS_BEFORE_MONTH[index] + i64::from(month > 2 && is_leap(year)) + day +} + +/// `ord_to_ymd`. +fn ordinal_to_ymd(ordinal: i64) -> (i64, i64, i64) { + let mut days: i64 = ordinal - 1; + let four_centuries: i64 = days / 146_097; + days %= 146_097; + let centuries: i64 = days / 36_524; + days %= 36_524; + let four_years: i64 = days / 1_461; + days %= 1_461; + let years: i64 = days / 365; + days %= 365; + let year: i64 = four_centuries * 400 + 1 + centuries * 100 + four_years * 4 + years; + if years == 4 || centuries == 4 { + return (year - 1, 12, 31); + } + let leap: bool = years == 3 && (four_years != 24 || centuries == 3); + let mut month: i64 = (days + 50) >> 5; + let month_index: usize = usize::try_from(month).unwrap_or(1); + let mut preceding: i64 = DAYS_BEFORE_MONTH[month_index] + i64::from(month > 2 && leap); + if preceding > days { + month -= 1; + preceding -= days_in_month(year, month); + } + (year, month, days - preceding + 1) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index e76ea01eae..b15366a917 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -2,11 +2,14 @@ use std::collections::BTreeSet; +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; + +use crate::model_validation::_helpers::config::python_lower; use crate::model_validation::_helpers::config::{ - ConfigView, one_of, python_repr, python_str, sorted_values, text_if_string, + ConfigView, one_of, sorted_values, text_if_string, }; -use crate::model_validation::_helpers::durations::parse_duration; -use crate::model_validation::_helpers::text::{python_lower, python_strip}; use crate::model_validation::constants::{ BUILTIN_MATERIALIZATIONS, CONTRACT_POLICIES, CUSTOM_MATERIALIZATION_DISALLOWED_KEYS, HISTORY_MATERIALIZATIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_ONLY_KEYS, @@ -14,7 +17,7 @@ use crate::model_validation::constants::{ TABLE_BACKED_MATERIALIZATIONS, VIEW_MATERIALIZATION, }; use crate::model_validation::models::{ - ModelValidationFacts, ProjectValidationFacts, Rejected, ValidationStop, + ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; @@ -62,7 +65,7 @@ pub(crate) fn check_contract(config: &ConfigView<'_, N>) -> Che { return Ok(()); } - let contract = contract.text().ok_or(Rejected)?; + let contract: String = contract.text().unwrap_or_default(); Err(config.error(format!( "unknown contract '{contract}'; valid values: {}", sorted_values(&CONTRACT_POLICIES) @@ -173,14 +176,13 @@ pub(crate) fn check_migration( let Some(migrate_from) = migrate_from else { return Err(config.error("migrate_force requires migrate_from")); }; - let origin = match text_if_string(migrate_from)? { - Some(text) => python_strip(&text)?.to_owned(), - None => String::new(), - }; + let origin: String = text_if_string(migrate_from) + .map(|text| python_strip(&text).to_owned()) + .unwrap_or_default(); if origin.is_empty() { return Err(config.error("migrate_from must be a model name or a qualified relation")); } - if python_lower(&origin)? == python_lower(facts.model_name)? { + if python_lower(&origin) == python_lower(facts.model_name) { return Err(config.error("migrate_from cannot name the model itself")); } match migrate_force.map(AuthoredNode::kind) { @@ -196,12 +198,14 @@ fn check_old_name_view(config: &ConfigView<'_, N>) -> Check { if value.kind() == NodeKind::Bool(false) { return Ok(()); } - if let Some(text) = text_if_string(value)? - && parse_duration(python_strip(&text)?)?.is_some() + if let Some(text) = text_if_string(value) + && config + .duration(OLD_NAME_VIEW_KEY, python_strip(&text))? + .is_some() { return Ok(()); } - let shown = python_repr(value)?; + let shown: String = value.python_repr(); Err(ValidationStop::Error( config .config_error(format!( @@ -221,7 +225,7 @@ pub(crate) fn check_placeholders( let custom = materialized .as_ref() .is_some_and(|value| project.custom_materializations.contains(value)); - let used = placeholder_names(facts.query_sql)?; + let used = placeholder_names(project.python, facts.query_sql); let declared = config.get("placeholders"); if !custom && !used.is_empty() { return Err(config.error("@@@placeholders are only allowed on custom materializations")); @@ -236,8 +240,8 @@ pub(crate) fn check_placeholders( Some(value) if value.kind() == NodeKind::Map => value .entries() .iter() - .map(|(key, _)| python_str(key)) - .collect::, Rejected>>()?, + .map(|(key, _)| key.python_str()) + .collect::>(), _ => BTreeSet::new(), }; let missing: Vec<&str> = used @@ -263,23 +267,16 @@ pub(crate) fn check_placeholders( Ok(()) } -/// Return the names `re.findall(r"@@@(\w+)", sql)` finds, rejecting non-ASCII word candidates. -fn placeholder_names(sql: &str) -> Result, Rejected> { +/// Return the names `re.findall(r"@@@(\w+)", sql)` finds; `\w` is Python's word class. +fn placeholder_names(python: PythonText, sql: &str) -> BTreeSet { let mut names: BTreeSet = BTreeSet::new(); let mut rest = sql; while let Some(start) = rest.find(PLACEHOLDER_SIGIL) { let after = &rest[start + PLACEHOLDER_SIGIL.len()..]; - let length = after - .bytes() - .take_while(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') - .count(); - if after[length..] - .chars() - .next() - .is_some_and(|next| !next.is_ascii()) - { - return Err(Rejected); - } + let length: usize = after + .char_indices() + .find(|(_, character)| !is_python_word(python, *character)) + .map_or(after.len(), |(at, _)| at); if length == 0 { rest = &rest[start + 1..]; continue; @@ -287,5 +284,5 @@ fn placeholder_names(sql: &str) -> Result, Rejected> { names.insert(after[..length].to_owned()); rest = &after[length..]; } - Ok(names) + names } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs index 9537395733..761aa37d06 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs @@ -3,9 +3,8 @@ use std::collections::HashSet; use crate::model_validation::_helpers::config::{ - ConfigView, non_blank_string_check, one_of, python_str, sorted_values, + ConfigView, non_blank_string_check, one_of, sorted_values, }; -use crate::model_validation::_helpers::durations::parse_duration; use crate::model_validation::_helpers::incremental::IncrementalValues; use crate::model_validation::constants::{ CAP_FROM_START_ACTION, CURSOR_INPUT_ROLES, DELETE_INSERT_STRATEGY, EFFECTIVE_BATCH_SIZE, @@ -14,7 +13,7 @@ use crate::model_validation::constants::{ ROLLING_WINDOW_STRATEGY, TIMESTAMP_CURSOR, UNACCOUNTED_PARTITION_POLICIES, WATERMARK_BLOCK_KEYS, WATERMARK_MODES, WATERMARK_ROLE, WATERMARK_STRATEGY, }; -use crate::model_validation::models::{ModelValidationFacts, Rejected, ValidationStop}; +use crate::model_validation::models::{ModelValidationFacts, ValidationStop}; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; @@ -67,7 +66,9 @@ fn check_state_config( if let Some(concurrency) = config.get("batch_concurrency") { let concurrent = match (concurrency.kind(), concurrency.integer()) { (NodeKind::Int { negative: false }, Some(value)) if value > 0 => value > 1, - (NodeKind::Int { negative: false }, None) => true, + (NodeKind::Int { negative: false }, None) => { + return Err(config.integer_too_large("batch_concurrency", concurrency)); + } _ => return Err(config.error("batch_concurrency must be a positive integer")), }; if concurrent && !microbatch { @@ -165,6 +166,9 @@ fn check_cursor_inputs( ); } if let Some(limit) = &values.max_microbatches { + if is_big_integer(limit) { + return Err(config.integer_too_large("max_microbatches", limit)); + } if !is_positive_integer(limit) { return Err(config.error("max_microbatches must be a positive integer")); } @@ -183,11 +187,15 @@ fn check_cursor_inputs( } fn is_positive_integer(value: &N) -> bool { - match (value.kind(), value.integer()) { - (NodeKind::Int { negative: false }, Some(number)) => number >= 1, - (NodeKind::Int { negative: false }, None) => true, - _ => false, - } + matches!( + (value.kind(), value.integer()), + (NodeKind::Int { negative: false }, Some(number)) if number >= 1 + ) +} + +/// Whether the value is an integer beyond a signed 64-bit integer. +fn is_big_integer(value: &N) -> bool { + matches!(value.kind(), NodeKind::Int { .. }) && value.integer().is_none() } fn expected_inputs(input_names: &HashSet<&str>) -> String { @@ -209,18 +217,18 @@ fn check_input_map( return Err(config.error("cursor_inputs must not be empty")); } for (relation, column) in &entries { - if !non_blank_string_check(relation)? { + if !non_blank_string_check(relation) { return Err(config.error("cursor_inputs relation names must be non-empty strings")); } - if !non_blank_string_check(column)? { - let relation = relation.text().ok_or(Rejected)?; + if !non_blank_string_check(column) { + let relation: String = relation.text().unwrap_or_default(); return Err(config.error(format!( "cursor_inputs column for relation '{relation}' must be a non-empty string" ))); } } for (relation, _) in &entries { - let relation = relation.text().ok_or(Rejected)?; + let relation: String = relation.text().unwrap_or_default(); if !input_names.contains(relation.as_str()) { return Err(config.error(format!( "cursor_inputs references unknown input '{relation}'; expected one of: {}", @@ -242,28 +250,30 @@ fn check_watermark_inputs( } let mut has_watermark = false; for (relation, block) in &entries { - if !non_blank_string_check(relation)? || block.kind() != NodeKind::Map { - let relation = python_str(relation)?; + if !non_blank_string_check(relation) || block.kind() != NodeKind::Map { + let relation: String = relation.python_str(); return Err(config.error(format!( "watermark cursor_inputs relation '{relation}' must use \ (column ..., roles [...])" ))); } - let relation = relation.text().ok_or(Rejected)?; + let relation: String = relation.text().unwrap_or_default(); let block = block.entries(); - if !has_exact_keys(&block, &WATERMARK_BLOCK_KEYS) { + let (true, Some(column), Some(roles)) = ( + has_exact_keys(&block, &WATERMARK_BLOCK_KEYS), + block_value(&block, COLUMN_KEY), + block_value(&block, ROLES_KEY), + ) else { return Err(config.error(format!( "cursor_inputs relation '{relation}' requires exactly column and roles" ))); - } - let column = block_value(&block, COLUMN_KEY).ok_or(Rejected)?; - if !non_blank_string_check(column)? { + }; + if !non_blank_string_check(column) { return Err(config.error(format!( "cursor_inputs column for relation '{relation}' must be a non-empty string" ))); } - let roles = block_value(&block, ROLES_KEY).ok_or(Rejected)?; - let Some(roles) = valid_roles(roles)? else { + let Some(roles) = valid_roles(roles) else { return Err(config.error(format!( "cursor_inputs roles for relation '{relation}' must be a non-empty list \ containing only filter and/or watermark" @@ -311,27 +321,24 @@ fn block_value<'b, N: AuthoredNode>(block: &'b [(N, N)], key: &str) -> Option<&' .map(|(_, value)| value) } -/// Return a non-empty list of valid roles, `None` when Python rejects it, or defer. -fn valid_roles(roles: &N) -> Result>, Rejected> { +/// Return a non-empty list of valid roles, or `None` when the roles are invalid. +fn valid_roles(roles: &N) -> Option> { if roles.kind() != NodeKind::List { - return Ok(None); + return None; } let items = roles.items(); if items.is_empty() { - return Ok(None); - } - let mut valid: Vec<&'static str> = Vec::with_capacity(items.len()); - for item in &items { - match item.kind() { - NodeKind::Str => match CURSOR_INPUT_ROLES.iter().find(|role| item.is_text(role)) { - Some(role) => valid.push(role), - None => return Ok(None), - }, - NodeKind::Int { .. } | NodeKind::Bool(_) | NodeKind::Null => return Ok(None), - _ => return Err(Rejected), - } + return None; } - Ok(Some(valid)) + items + .iter() + .map(|item| { + CURSOR_INPUT_ROLES + .iter() + .copied() + .find(|role| item.is_text(role)) + }) + .collect() } /// The resolved batch limit and its action, as `_validate_model_microbatch_limit` returns them. @@ -359,15 +366,21 @@ fn microbatch_limit( return Err(config.error("use either max_microbatches or microbatch_limit, not both")); } let block = limit.entries(); - if limit.kind() != NodeKind::Map || !has_exact_keys(&block, &MICROBATCH_LIMIT_KEYS) { + let [max_key, action_key] = MICROBATCH_LIMIT_KEYS; + let (true, true, Some(max_batches), Some(action)) = ( + limit.kind() == NodeKind::Map, + has_exact_keys(&block, &MICROBATCH_LIMIT_KEYS), + block_value(&block, max_key), + block_value(&block, action_key), + ) else { return Err(config.error("microbatch_limit requires exactly max_batches and action")); + }; + if is_big_integer(max_batches) { + return Err(config.integer_too_large("microbatch_limit max_batches", max_batches)); } - let [max_key, action_key] = MICROBATCH_LIMIT_KEYS; - let max_batches = block_value(&block, max_key).ok_or(Rejected)?; if !is_positive_integer(max_batches) { return Err(config.error("microbatch_limit max_batches must be a positive integer")); } - let action = block_value(&block, action_key).ok_or(Rejected)?; let Some(action) = MICROBATCH_LIMIT_ACTIONS .iter() .find(|valid| action.is_text(valid)) @@ -406,8 +419,10 @@ fn check_static_watermark_limit( let (Some(lookback), Some(batch_size)) = (lookback, batch_size) else { return Ok(()); }; - let (Some(lookback), Some(batch)) = (parse_duration(lookback)?, parse_duration(batch_size)?) - else { + let (Some(lookback), Some(batch)) = ( + config.duration("lookback", lookback)?, + config.duration("batch_size", batch_size)?, + ) else { return Ok(()); }; let lookback_batches = if lookback.total_months == 0 && batch.total_months == 0 { @@ -432,8 +447,7 @@ fn check_static_watermark_limit( let Some(max_batches) = limit.max_batches else { return Ok(()); }; - let max = u128::try_from(max_batches).map_err(|_| Rejected)?; - if max < required { + if u128::try_from(max_batches).is_ok_and(|max| max < required) { Err(config.error(format!( "max_microbatches {max_batches} is below the ordinary lookback requirement of \ {required} batches" diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs index bb9420ceec..abcc6794ed 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs @@ -1,9 +1,10 @@ pub(crate) mod config; pub(crate) mod cursor_bounds; +pub(crate) mod decimals; pub(crate) mod durations; pub(crate) mod incremental; +pub(crate) mod isoformat; pub(crate) mod materialization; pub(crate) mod microbatch; pub(crate) mod references; pub(crate) mod snapshot; -pub(crate) mod text; diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs index bb8967c152..d8af8c5de2 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs @@ -2,20 +2,26 @@ use crate::errors::ConfigError; use crate::model_validation::constants::{ - REF_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, UDF_KIND, + DBT_REF_KIND, REF_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, UDF_KIND, }; use crate::model_validation::models::{ ModelReference, ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; use crate::model_validation::types::Check; -/// Check each reference in order; external dbt references and unknown kinds defer to Python. +/// Check each reference in order; a dbt reference stops where its resolver rejected it. pub(crate) fn check_references( facts: &ModelValidationFacts<'_>, project: &ProjectValidationFacts, ) -> Check { - for reference in facts.references { - if let Some(problem) = reference_problem(reference, project)? { + for (index, reference) in facts.references.iter().enumerate() { + if reference.kind == DBT_REF_KIND { + if reference.externally_rejected { + return Err(ValidationStop::External(index)); + } + continue; + } + if let Some(problem) = reference_problem(reference, project) { return Err(ValidationStop::Error(ConfigError::compile(format!( "Model file {} {problem}", facts.relative_path @@ -28,13 +34,13 @@ pub(crate) fn check_references( fn reference_problem( reference: &ModelReference, project: &ProjectValidationFacts, -) -> Result, ValidationStop> { +) -> Option { let name = &reference.name; let model = project.models.contains(name); let seed = project.seeds.contains(name); let function = project.functions.contains(name); let table_function = project.table_functions.contains(name); - Ok(match reference.kind.as_str() { + match reference.kind.as_str() { REF_KIND if !model && seed => Some(format!( "references seed '{name}' with __ref(...). Use {} for seed references; __ref only \ resolves models.", @@ -60,9 +66,8 @@ fn reference_problem( TABLE_FUNCTION_KIND if !table_function => Some(format!( "references scalar function '{name}' with __table_fn(); use __udf() for scalar UDFs" )), - REF_KIND | SEED_KIND | SOURCE_KIND | UDF_KIND | TABLE_FUNCTION_KIND => None, - _ => return Err(ValidationStop::Defer), - }) + _ => None, + } } /// Return `SqlReferenceKind.example_call(name, quote='"')`. diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs index 6c5c302bc2..0d4a0318cd 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs @@ -78,7 +78,7 @@ pub(crate) fn check_snapshot( check_history(config, &values)?; check_policies(config, &values)?; if let (Some(valid_from), Some(valid_to)) = (&values.valid_from_column, &values.valid_to_column) - && same_lowered(valid_from, valid_to)? + && same_lowered(valid_from, valid_to) { return Err(config.error("valid_from_column and valid_to_column must differ")); } @@ -215,7 +215,7 @@ fn check_contract_columns( require_declared( config, "unique_key", - &string_sequence(values.unique_key.as_ref())?, + &string_sequence(values.unique_key.as_ref()), &declared, )?; if let Some(updated_at) = &values.updated_at { @@ -240,7 +240,7 @@ fn check_contract_columns( require_declared( config, "check_columns", - &string_sequence(values.check_columns.as_ref())?, + &string_sequence(values.check_columns.as_ref()), &declared, ) } diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs deleted file mode 100644 index 67bca29187..0000000000 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! ASCII text operations that match Python's `str` methods; other text is left to Python. - -use crate::model_validation::constants::SETTING_SNIPPET_INDENT; -use crate::model_validation::models::Rejected; - -/// Return Python's `text.strip()` for ASCII text, or reject other text. -pub(crate) fn python_strip(text: &str) -> Result<&str, Rejected> { - if text.is_ascii() { - Ok(text.trim_matches(|character: char| is_python_space(character as u8))) - } else { - Err(Rejected) - } -} - -/// Return Python's `text.lower()` for ASCII text, or reject other text. -pub(crate) fn python_lower(text: &str) -> Result { - if text.is_ascii() { - Ok(text.to_ascii_lowercase()) - } else { - Err(Rejected) - } -} - -fn is_python_space(byte: u8) -> bool { - matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') -} - -/// Return `model_header_help`: the purpose, then the exact MODEL header entry to add. -pub(crate) fn model_header_help(purpose: &str, entry: &str) -> String { - let indent = SETTING_SNIPPET_INDENT; - format!( - "{purpose}, add this to the MODEL header:\n{indent}MODEL (\n{indent} {entry},\n\ - {indent} ...\n{indent});" - ) -} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index fc5140ce97..f1bb436fff 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -195,6 +195,8 @@ pub const SOURCE_KIND: &str = "source"; pub const UDF_KIND: &str = "udf"; /// A table function reference. pub const TABLE_FUNCTION_KIND: &str = "table_fn"; +/// The `__dbt_ref` reference kind. +pub const DBT_REF_KIND: &str = "dbt_ref"; /// The `inherit` storage policy value. pub const INHERIT_POLICY: &str = "inherit"; @@ -205,29 +207,5 @@ pub const PERMANENT_TABLE_TYPE: &str = "permanent"; /// The transient table type. pub const TRANSIENT_TABLE_TYPE: &str = "transient"; -/// The length of an ISO `YYYY-MM-DD` date. -pub const ISO_DATE_LENGTH: usize = 10; -/// Byte offsets of the separators in an ISO date. -pub const ISO_DATE_SEPARATOR_OFFSETS: [usize; 2] = [4, 7]; -/// The length of a two-digit clock field. -pub const CLOCK_FIELD_LENGTH: usize = 2; -/// Clock part counts `HH:MM` and `HH:MM:SS`. -pub const CLOCK_PART_COUNTS: [usize; 2] = [2, 3]; -/// The clock part count that may carry fractional seconds. -pub const CLOCK_PARTS_WITH_SECONDS: usize = 3; -/// The latest hour of a day. -pub const MAX_HOUR: i128 = 23; -/// The latest minute of an hour, and second of a minute. -pub const MAX_MINUTE: i128 = 59; -/// The length of a `HH:MM` UTC offset. -pub const UTC_OFFSET_LENGTH: usize = 5; -/// The first year Python's `datetime` represents. -pub const MIN_YEAR: i128 = 1; -/// The last year Python's `datetime` represents. -pub const MAX_YEAR: i128 = 9999; -/// Prefixes of the special values `Decimal` parses, after any sign. -pub const DECIMAL_SPECIAL_PREFIXES: [&str; 3] = ["inf", "nan", "snan"]; -/// The last month whose civil-day computation shifts the year back. -pub const LAST_SHIFTED_MONTH: i128 = 2; /// The position of days among duration units. pub const DAY_UNIT_INDEX: usize = 2; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs index 0e18dcfe87..106d25d285 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs @@ -1,3 +1,4 @@ pub mod retention_override; pub mod table_type_override; pub mod validate_model_config; +pub mod validate_model_references; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs index 42c65c51d7..f0eeb65994 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs @@ -1,37 +1,58 @@ //! Read a MODEL header's `time_travel_retention` as `resolve_time_travel_retention` does. +use sqlbuild_core::text::models::PythonText; + use crate::errors::ConfigError; +use crate::model_validation::_helpers::config::model_header_help; use crate::model_validation::_helpers::config::text_if_string; -use crate::model_validation::_helpers::durations::parse_duration; +use crate::model_validation::_helpers::durations::{DurationNumberError, parse_duration}; use crate::model_validation::constants::{DISABLED_RETENTION, INHERIT_POLICY, ZERO_DAY_DURATION}; -use crate::model_validation::models::{RetentionOverride, ValidationStop}; +use crate::model_validation::models::RetentionOverride; use crate::types::{AuthoredNode, NodeKind}; -/// Return the header retention override, the error Python raises, or a deferral. +const RETENTION_KEY: &str = "time_travel_retention"; + +/// Return the header retention override, or the error it raises. pub fn retention_override( + python: PythonText, value: Option<&N>, model_name: &str, -) -> Result { - let invalid = |expected: &str| { - ValidationStop::Error(ConfigError::compile(format!( - "model '{model_name}': time_travel_retention must be a whole-day string like {expected}" - ))) +) -> Result { + let error = |text: String| { + ConfigError::compile(format!("model '{model_name}': {RETENTION_KEY} {text}")) }; + let invalid = |expected: &str| error(format!("must be a whole-day string like {expected}")); let Some(value) = value.filter(|node| node.kind() != NodeKind::Null) else { return Ok(RetentionOverride::Inherit); }; - let Some(text) = text_if_string(value)? else { + let Some(text) = text_if_string(value) else { return Err(invalid("'7d'")); }; match text.as_str() { INHERIT_POLICY => Ok(RetentionOverride::Inherit), DISABLED_RETENTION => Ok(RetentionOverride::Unmanaged), ZERO_DAY_DURATION => Ok(RetentionOverride::Days(0)), - _ => match parse_duration(&text)? { - Some(duration) if duration.is_whole_days() => u64::try_from(duration.days()) + _ => match parse_duration(python, &text) { + Ok(Some(duration)) if duration.is_whole_days() => u64::try_from(duration.days()) .map(RetentionOverride::Days) - .map_err(|_| ValidationStop::Defer), - _ => Err(invalid("'7d', 'inherit', or 'disabled'")), + .map_err(|_| invalid("'7d', 'inherit', or 'disabled'")), + Ok(_) => Err(invalid("'7d', 'inherit', or 'disabled'")), + Err(number_error) => { + let (problem, purpose) = match number_error { + DurationNumberError::NonAsciiDigits => ( + "uses digits outside ASCII", + "write time_travel_retention with ASCII digits 0-9", + ), + DurationNumberError::TooLarge => ( + "has a number larger than a 64-bit integer", + "use a time_travel_retention whose days fit in 64 bits", + ), + }; + Err(ConfigError::compile(format!( + "model '{model_name}': {RETENTION_KEY} '{text}' {problem}" + )) + .with_help(model_header_help(purpose, &format!("{RETENTION_KEY} '7d'")))) + } }, } } diff --git a/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs index c449a5aa99..c55f89715b 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs @@ -5,23 +5,23 @@ use crate::model_validation::_helpers::config::text_if_string; use crate::model_validation::constants::{ INHERIT_POLICY, PERMANENT_TABLE_TYPE, TRANSIENT_TABLE_TYPE, }; -use crate::model_validation::models::{TableTypeOverride, ValidationStop}; +use crate::model_validation::models::TableTypeOverride; use crate::types::{AuthoredNode, NodeKind}; -/// Return the header table-type override, the error Python raises, or a deferral. +/// Return the header table-type override, or the error it raises. pub fn table_type_override( value: Option<&N>, model_name: &str, -) -> Result { +) -> Result { let Some(value) = value.filter(|node| node.kind() != NodeKind::Null) else { return Ok(TableTypeOverride::Inherit); }; - match text_if_string(value)?.as_deref() { + match text_if_string(value).as_deref() { Some(INHERIT_POLICY) => Ok(TableTypeOverride::Inherit), Some(PERMANENT_TABLE_TYPE) => Ok(TableTypeOverride::Permanent), Some(TRANSIENT_TABLE_TYPE) => Ok(TableTypeOverride::Transient), - _ => Err(ValidationStop::Error(ConfigError::compile(format!( + _ => Err(ConfigError::compile(format!( "model '{model_name}': table_type must be permanent, transient, or inherit" - )))), + ))), } } diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index 7c1d046875..a0826cecde 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -13,13 +13,13 @@ use crate::model_validation::models::{ }; use crate::types::AuthoredNode; -/// Run the Python validators' rules in their order; defer where only Python can decide. +/// Run the model validators in order, stopping at the first problem. pub fn validate_model_config( entries: Vec<(N, N)>, project: &ProjectValidationFacts, facts: &ModelValidationFacts<'_>, ) -> Result<(), ValidationStop> { - let config = ConfigView::new(entries, facts.model_name); + let config = ConfigView::new(entries, facts.model_name, project.python); check_references(facts, project)?; check_incremental(&config, facts)?; check_project_capability(&config, project)?; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_references.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_references.rs new file mode 100644 index 0000000000..800da1dba3 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_references.rs @@ -0,0 +1,14 @@ +//! Validate one model's references on their own, as its hook references are checked. + +use crate::model_validation::_helpers::references::check_references; +use crate::model_validation::models::{ + ModelValidationFacts, ProjectValidationFacts, ValidationStop, +}; + +/// Check each reference in order, stopping at the first unknown or misused one. +pub fn validate_model_references( + project: &ProjectValidationFacts, + facts: &ModelValidationFacts<'_>, +) -> Result<(), ValidationStop> { + check_references(facts, project) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/models.rs b/crates/sqlbuild-model-config/src/model_validation/models.rs index 9a7ed254c7..03ab6979e0 100644 --- a/crates/sqlbuild-model-config/src/model_validation/models.rs +++ b/crates/sqlbuild-model-config/src/model_validation/models.rs @@ -2,11 +2,15 @@ use std::collections::HashSet; +use sqlbuild_core::text::models::PythonText; + use crate::errors::ConfigError; /// Project-wide facts every model's validation reads. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct ProjectValidationFacts { + /// The Python string and library semantics the validators follow. + pub python: PythonText, /// Discovered custom materialization names. pub custom_materializations: HashSet, /// Whether the project enables concurrent microbatches. @@ -28,6 +32,8 @@ pub struct ProjectValidationFacts { pub struct ModelReference { pub kind: String, pub name: String, + /// For a dbt reference, whether the external resolver rejects it. + pub externally_rejected: bool, } /// One model's facts beyond its config values. @@ -48,23 +54,13 @@ pub struct ModelValidationFacts<'a> { pub table_type_declared: bool, } -/// Why native validation cannot decide a value: Python must run and decide. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Rejected; - -/// Why native validation stops: Python must decide, or the exact error the validators raise. +/// Why validation stops at the first problem. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ValidationStop { - /// The config holds a value only Python can judge; the Python validators run. - Defer, - /// The first error the Python validators raise. + /// The first config error. Error(ConfigError), -} - -impl From for ValidationStop { - fn from(_: Rejected) -> Self { - Self::Defer - } + /// The reference at this index names a dbt model the external resolver rejected. + External(usize), } /// The header's `time_travel_retention`, applied over the inherited policy. diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs index 4f2b5c2fb6..c12612be88 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -4,6 +4,13 @@ use crate::model_validation::models::{ ModelReference, ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; use crate::tests::test_types::Value; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; + +/// The string semantics of Python 3.12. +pub(super) fn python_312() -> PythonText { + python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") +} /// The validator error `model 'orders_daily': `. pub(super) fn validator_error(text: &str) -> ValidationStop { @@ -12,7 +19,8 @@ pub(super) fn validator_error(text: &str) -> ValidationStop { ))) } -/// Validate `config` for `orders_daily`: `accepted`, `deferred`, or the error message. +/// Validate `config` for `orders_daily`: `accepted`, `external `, or the error message; +/// a `kind!:name` reference is a dbt reference its resolver rejects. pub(super) fn validation_outcome( config: Vec<(&'static str, Value)>, references: &[&str], @@ -22,8 +30,9 @@ pub(super) fn validation_outcome( .iter() .filter_map(|reference| reference.split_once(':')) .map(|(kind, name)| ModelReference { - kind: kind.to_owned(), + kind: kind.trim_end_matches('!').to_owned(), name: name.to_owned(), + externally_rejected: kind.ends_with('!'), }) .collect(); let facts = ModelValidationFacts { @@ -44,10 +53,10 @@ pub(super) fn validation_outcome( } fn stop_text(stop: ValidationStop) -> String { - let ValidationStop::Error(error) = stop else { - return "deferred".to_owned(); - }; - error.message + match stop { + ValidationStop::Error(error) => error.message, + ValidationStop::External(index) => format!("external {index}"), + } } /// A list of strings. @@ -74,6 +83,7 @@ pub(super) fn map(entries: Vec<(&'static str, Value)>) -> Value { pub(super) fn project() -> ProjectValidationFacts { let names = |values: &[&str]| values.iter().map(|value| (*value).to_owned()).collect(); ProjectValidationFacts { + python: python_312(), custom_materializations: names(&["ledger"]), microbatch_concurrency: false, models: names(&["orders", "customers"]), diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs index c4a076a42d..1e37fe78a3 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs @@ -1,7 +1,7 @@ use crate::model_validation::main::retention_override::retention_override; use crate::model_validation::main::table_type_override::table_type_override; -use crate::model_validation::models::{RetentionOverride, TableTypeOverride}; -use crate::model_validation::tests::helpers::validator_error; +use crate::model_validation::models::{RetentionOverride, TableTypeOverride, ValidationStop}; +use crate::model_validation::tests::helpers::{python_312, validator_error}; use crate::model_validation::tests::test_types::{ RetentionOverrideTestCase, TableTypeOverrideTestCase, }; @@ -53,7 +53,8 @@ fn given_header_retention_values_when_reading_then_python_policies_result() { ]; for test_case in test_cases { - let resolved = retention_override(test_case.value.as_ref(), "orders_daily"); + let resolved = retention_override(python_312(), test_case.value.as_ref(), "orders_daily") + .map_err(ValidationStop::Error); assert_eq!( resolved, test_case.expected_override, @@ -96,7 +97,8 @@ fn given_header_table_types_when_reading_then_python_types_result() { ]; for test_case in test_cases { - let resolved = table_type_override(test_case.value.as_ref(), "orders_daily"); + let resolved = table_type_override(test_case.value.as_ref(), "orders_daily") + .map_err(ValidationStop::Error); assert_eq!( resolved, test_case.expected_override, diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index cfe5224f06..9de1513ba1 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -34,11 +34,18 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( expected_outcome: "Model file models/marts/orders_daily.sql references table function 'order_lines' with __udf(); use __table_fn() in SQL contexts that support table-valued functions", }, ValidateTestCase { - description: "a dbt reference always defers to Python", + description: "a dbt reference its resolver accepts", config: vec![], references: &["dbt_ref:orders"], query_sql: "select 1", - expected_outcome: "deferred", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a rejected dbt reference stops at its index, after earlier references", + config: vec![], + references: &["ref:orders", "dbt_ref!:invoices", "ref:missing"], + query_sql: "select 1", + expected_outcome: "external 1", }, ValidateTestCase { description: "an append incremental with ordered ISO bounds", @@ -92,11 +99,150 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( expected_outcome: "accepted", }, ValidateTestCase { - description: "an unusual ISO timestamp defers", - config: incremental(vec![("cursor_start", Value::Str("2024-W01-1"))]), + description: "an ISO week date, a basic date and a comma fraction are timestamps", + config: incremental(vec![ + ("cursor_start", Value::Str("2024-W01-1")), + ("cursor_end", Value::Str("20240102T000000,5")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a date object is a timestamp bound and orders by its text", + config: incremental(vec![ + ("cursor_start", Value::Date("2024-03-01")), + ("cursor_end", Value::Str("2024-02-01")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start must be before exclusive cursor_end", + }, + ValidateTestCase { + description: "an unparsable timestamp names Python's repr", + config: incremental(vec![("cursor_start", Value::Str("2024-01-01X1"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start value '2024-01-01X1' is not a valid ISO timestamp: Invalid isoformat string: '2024-01-01X1'", + }, + ValidateTestCase { + description: "a day past the month's end gives Python's range error", + config: incremental(vec![("cursor_start", Value::Str("2023-02-29T10:00"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start value '2023-02-29T10:00' is not a valid ISO timestamp: day is out of range for month", + }, + ValidateTestCase { + description: "an offset of a day or more gives Python's timezone error", + config: incremental(vec![("cursor_start", Value::Str("2024-01-01T00:00+24:00"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start value '2024-01-01T00:00+24:00' is not a valid ISO timestamp: offset must be a timedelta strictly between -timedelta(hours=24) and timedelta(hours=24), not datetime.timedelta(days=1).", + }, + ValidateTestCase { + description: "bounds under a second apart order by their float timestamps", + config: incremental(vec![ + ("cursor_start", Value::Str("2024-01-01T00:00:00.000001")), + ("cursor_end", Value::Str("2024-01-01T00:00:00.000002")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "an integer cursor reads Decimal text with underscores and exponents", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("order_id")), + ("cursor_type", Value::Str("integer")), + ("cursor_start", Value::Str(" 1_000 ")), + ("cursor_end", Value::Str("1.5e3")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "integer bounds compare as decimals beyond 64 bits", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("order_id")), + ("cursor_type", Value::Str("integer")), + ("cursor_start", Value::Str("100000000000000000000000")), + ("cursor_end", Value::Str("99999999999999999999999")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start must be before exclusive cursor_end", + }, + ValidateTestCase { + description: "an integer cursor value of NaN is not a valid integer", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("order_id")), + ("cursor_type", Value::Str("integer")), + ("cursor_start", Value::Str("NaN")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start value 'NaN' is not a valid integer", + }, + ValidateTestCase { + description: "an integer cursor value in non-ASCII digits is rejected", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("order_id")), + ("cursor_type", Value::Str("integer")), + ("cursor_start", Value::Str("\u{663}")), + ], references: &["ref:orders"], query_sql: "select 1", - expected_outcome: "deferred", + expected_outcome: "model 'orders_daily': cursor_start value '\u{663}' uses digits outside ASCII", + }, + ValidateTestCase { + description: "an integer cursor value beyond 64 bits is rejected", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("order_id")), + ("cursor_type", Value::Str("integer")), + ("cursor_start", Value::BigInt("9223372036854775808")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start 9223372036854775808 is larger than a 64-bit integer", + }, + ValidateTestCase { + description: "a list cursor action is not an action", + config: incremental(vec![("cursor_future_action", strings(&["cap"]))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_future_action must be one of: cap, error", + }, + ValidateTestCase { + description: "a duration in non-ASCII digits is rejected", + config: incremental(vec![("cursor_start_max_ahead", Value::Str("\u{663}d"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor_start_max_ahead '\u{663}d' uses digits outside ASCII", + }, + ValidateTestCase { + description: "a duration with a trailing newline parses as Python's $ allows", + config: incremental(vec![("cursor_start_max_ahead", Value::Str("2d\n"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a placeholder name with Unicode word characters", + config: vec![("materialized", Value::Str("table"))], + references: &["ref:orders"], + query_sql: "select @@@gr\u{f6}\u{df}e", + expected_outcome: "model 'orders_daily': @@@placeholders are only allowed on custom materializations", }, ValidateTestCase { description: "a timestamp cursor without a grain", diff --git a/crates/sqlbuild-model-config/src/model_validation/types.rs b/crates/sqlbuild-model-config/src/model_validation/types.rs index df97c761d4..f88bdef04f 100644 --- a/crates/sqlbuild-model-config/src/model_validation/types.rs +++ b/crates/sqlbuild-model-config/src/model_validation/types.rs @@ -2,5 +2,5 @@ use crate::model_validation::models::ValidationStop; -/// Native validation's outcome: accepted, the first Python error, or a deferral to Python. +/// Validation's outcome: accepted, or the first problem. pub type Check = Result<(), ValidationStop>; diff --git a/crates/sqlbuild-model-config/src/tests/test_types.rs b/crates/sqlbuild-model-config/src/tests/test_types.rs index ba3ab491e4..42112eabf3 100644 --- a/crates/sqlbuild-model-config/src/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/tests/test_types.rs @@ -11,6 +11,10 @@ pub(crate) enum Value { Map(Vec<(Value, Value)>), Tuple(Vec), Float, + /// A `datetime.date` or `datetime.datetime`, by its `str()`. + Date(&'static str), + /// An integer beyond `i64`, by its decimal digits. + BigInt(&'static str), } impl AuthoredNode for Value { @@ -25,7 +29,10 @@ impl AuthoredNode for Value { Self::List(_) => NodeKind::List, Self::Map(_) => NodeKind::Map, Self::Tuple(_) => NodeKind::Tuple, - Self::Float => NodeKind::Other, + Self::Float | Self::Date(_) => NodeKind::Other, + Self::BigInt(digits) => NodeKind::Int { + negative: digits.starts_with('-'), + }, } } @@ -67,4 +74,45 @@ impl AuthoredNode for Value { _ => Vec::new(), } } + + fn python_str(&self) -> String { + match self { + Self::Null => "None".to_owned(), + Self::Bool(flag) => if *flag { "True" } else { "False" }.to_owned(), + Self::Int(value) => value.to_string(), + Self::Str(text) | Self::Date(text) | Self::BigInt(text) => (*text).to_owned(), + Self::Float => "0.5".to_owned(), + Self::List(_) | Self::Map(_) | Self::Tuple(_) => self.python_repr(), + } + } + + fn python_repr(&self) -> String { + match self { + Self::Str(text) => format!("'{text}'"), + Self::Date(text) => format!("datetime.date({text})"), + Self::List(items) => format!("[{}]", reprs(items)), + Self::Tuple(items) => format!("({})", reprs(items)), + Self::Map(entries) => format!( + "{{{}}}", + entries + .iter() + .map(|(key, value)| format!("{}: {}", key.python_repr(), value.python_repr())) + .collect::>() + .join(", ") + ), + _ => self.python_str(), + } + } + + fn is_date_like(&self) -> bool { + matches!(self, Self::Date(_)) + } +} + +fn reprs(items: &[Value]) -> String { + items + .iter() + .map(AuthoredNode::python_repr) + .collect::>() + .join(", ") } diff --git a/crates/sqlbuild-model-config/src/types.rs b/crates/sqlbuild-model-config/src/types.rs index bc9fbe7318..b0d66cde68 100644 --- a/crates/sqlbuild-model-config/src/types.rs +++ b/crates/sqlbuild-model-config/src/types.rs @@ -23,12 +23,18 @@ pub trait AuthoredNode: Clone { fn kind(&self) -> NodeKind; /// Return an integer that is not a boolean and fits in `i64`, or `None`. fn integer(&self) -> Option; - /// Return the text of a string value, or `None` when it is not readable UTF-8 text. + /// Return the text of a string value, or `None` for another value. fn text(&self) -> Option; /// Return whether this is a string value equal to `text`. fn is_text(&self, text: &str) -> bool; - /// Apply `read` to a string value's text, or return `None` when it is not readable UTF-8. + /// Apply `read` to a string value's text, or return `None` for another value. fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option; + /// Return Python's `str(value)`. + fn python_str(&self) -> String; + /// Return Python's `repr(value)`. + fn python_repr(&self) -> String; + /// Return whether the value is a `datetime.date` or `datetime.datetime`. + fn is_date_like(&self) -> bool; /// Return a list's or tuple's items in order. fn items(&self) -> Vec; /// Return a mapping's entries in insertion order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs index ebec35d8f1..7ae947f44c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs @@ -50,14 +50,12 @@ impl AuthoredNode for PyNode<'_> { self.with_text(str::to_owned) } + /// Read a string's text; lone surrogates, rejected where text enters a compile, read as U+FFFD. fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { - if let Ok(value) = self.0.downcast::() - && let Ok(text) = value.to_str() - { - Some(read(text)) - } else { - None - } + self.0 + .downcast::() + .ok() + .map(|value| read(&value.to_string_lossy())) } fn is_text(&self, text: &str) -> bool { @@ -81,6 +79,28 @@ impl AuthoredNode for PyNode<'_> { Vec::new() } } + + fn python_str(&self) -> String { + self.0 + .str() + .map(|text| text.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + fn python_repr(&self) -> String { + self.0 + .repr() + .map(|text| text.to_string_lossy().into_owned()) + .unwrap_or_default() + } + + fn is_date_like(&self) -> bool { + let py = self.0.py(); + py.import("datetime") + .and_then(|module| module.getattr("date")) + .and_then(|date| self.0.is_instance(&date)) + .unwrap_or(false) + } } fn entry_nodes<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs deleted file mode 100644 index 8324088f1e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_presence.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Recursive template and macro presence scans over authored Python config values. - -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_model_config::config_presence::main::contains_macro_call::contains_macro_call; -use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::models::Presence; - -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; - -fn answer(presence: Presence) -> Option { - match presence { - Presence::Present => Some(true), - Presence::Absent => Some(false), - Presence::Deferred => None, - } -} - -/// Return whether any nested string holds `${`, or `None` where Python must scan. -#[pyfunction] -fn config_contains_template(value: Bound<'_, PyAny>) -> Option { - answer(contains_template(&PyNode(value))) -} - -/// Return whether any nested string holds a macro call, or `None` where Python must scan. -#[pyfunction] -fn config_contains_macro_call(value: Bound<'_, PyAny>) -> Option { - answer(contains_macro_call(&PyNode(value))) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(config_contains_template, module)?)?; - module.add_function(wrap_pyfunction!(config_contains_macro_call, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 8644d5c08b..72e2c180e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod authored_nodes; pub(crate) mod config_errors; -pub(crate) mod config_presence; pub(crate) mod config_templates; pub(crate) mod errors; pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index a1101192f2..7f075e6ee4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -1,19 +1,21 @@ //! Effective model config layered and templated natively, as `build_model_config` builds it. +use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, }; use pyo3::{IntoPyObject, PyErr, pyclass, pymethods}; +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; use sqlbuild_model_config::config_presence::main::contains_template::contains_template; use sqlbuild_model_config::config_presence::main::first_macro_path::first_macro_path; -use sqlbuild_model_config::config_presence::models::{MacroPath, Presence}; +use sqlbuild_model_config::config_presence::models::MacroPath; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::model_validation::main::retention_override::retention_override; use sqlbuild_model_config::model_validation::main::table_type_override::table_type_override; -use sqlbuild_model_config::model_validation::models::{ - RetentionOverride, TableTypeOverride, ValidationStop, -}; +use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; use sqlbuild_model_config::templates::models::TemplateOptions; @@ -56,9 +58,8 @@ const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; -/// Why the native build stops: a deferral to Python, the exact error, or a Python error. +/// Why the native build stops: the config error, or a Python error. enum Halt { - Defer, Config(ConfigError), Error(PyErr), } @@ -69,12 +70,9 @@ impl From for Halt { } } -impl From for Halt { - fn from(stop: ValidationStop) -> Self { - match stop { - ValidationStop::Defer => Self::Defer, - ValidationStop::Error(error) => Self::Config(error), - } +impl From for Halt { + fn from(error: ConfigError) -> Self { + Self::Config(error) } } @@ -97,6 +95,7 @@ pub(crate) struct NativeModelConfigBuilder { target_name: Option>, run_id: Py, target_namespace: Option<(Py, Py)>, + python: PythonText, } /// One model's built config before Python wraps it in `CompileModelConfig`. @@ -119,8 +118,16 @@ impl NativeModelConfigBuilder { sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), run: (Bound<'_, PyAny>, Bound<'_, PyAny>), target_namespace: Option<(Bound<'_, PyAny>, Bound<'_, PyAny>)>, + python: ((u8, u8), String), ) -> PyResult { let (defaults, path_defaults, hook_entry_types) = layers; + let (python_version, unicode_version) = python; + let python = python_text(python_version, &unicode_version).ok_or_else(|| { + PyValueError::new_err(format!( + "no Python string semantics for Python {}.{} with Unicode {unicode_version}", + python_version.0, python_version.1 + )) + })?; let path_keys = path_defaults .keys() .iter() @@ -138,6 +145,7 @@ impl NativeModelConfigBuilder { run_id: run_id.unbind(), target_namespace: target_namespace .map(|(database, schema)| (database.unbind(), schema.unbind())), + python, }) } @@ -151,19 +159,18 @@ impl NativeModelConfigBuilder { }) } - /// Return the built config parts, the first error, or `None` when Python must build it. + /// Return the built config parts or the first config error. fn build<'py>( &self, py: Python<'py>, header: Bound<'py, PyDict>, matched_path_default: Option<&str>, model_name: &str, - ) -> PyResult>> { + ) -> PyResult> { compiler_guard( || match self.build_config(py, &header, matched_path_default, model_name) { - Ok(built) => built_tuple(py, built).map(|built| Some(built.into_any().unbind())), - Err(Halt::Defer) => Ok(None), - Err(Halt::Config(error)) => Ok(Some(native_config_error(py, error)?.into_any())), + Ok(built) => built_tuple(py, built).map(|built| built.into_any().unbind()), + Err(Halt::Config(error)) => Ok(native_config_error(py, error)?.into_any()), Err(Halt::Error(error)) => Err(error), }, ) @@ -195,7 +202,7 @@ impl NativeModelConfigBuilder { layered.del_item(key)?; } } - let has_templates = present(contains_template(&PyNode(layered.clone().into_any())))?; + let has_templates = contains_template(&PyNode(layered.clone().into_any())); let mut expansion = Expansion { builder: self, py, @@ -240,8 +247,7 @@ impl NativeModelConfigBuilder { .iter() .map(|key| entry_has_template(&namespaced, key)) .collect::>>()? - .into_iter() - .any(|found| found); + .contains(&true); let resolved = if needs_target_templates { let empty = PyDict::new(py); expansion.expand( @@ -258,6 +264,7 @@ impl NativeModelConfigBuilder { } let values = storage_free_values(py, &resolved, header)?; let retention = retention_override( + self.python, header.get_item(RETENTION_KEY)?.map(PyNode).as_ref(), model_name, )?; @@ -283,15 +290,15 @@ impl NativeModelConfigBuilder { .path_defaults .bind(py) .get_item(key)? - .ok_or(Halt::Defer)?; - values.downcast_into::().map_err(|_| Halt::Defer) + .ok_or_else(|| PyKeyError::new_err(key.to_owned()))?; + Ok(values.downcast_into::().map_err(PyErr::from)?) } /// Return a fresh `project_defaults_to_mapping` result, with its own `tags` list. fn project_defaults<'py>(&self, py: Python<'py>) -> Built> { let defaults = self.defaults.bind(py).copy()?; if let Some(tags) = defaults.get_item(TAGS_KEY)? { - let tags = tags.downcast_into::().map_err(|_| Halt::Defer)?; + let tags = tags.downcast_into::().map_err(PyErr::from)?; defaults.set_item(TAGS_KEY, PyList::new(py, tags.iter())?)?; } Ok(defaults) @@ -349,7 +356,7 @@ impl<'py> Expansion<'_, 'py> { scope: (bool, &str), ) -> Built> { let result = self.expand_value(values.as_any(), variables, context, scope)?; - result.downcast_into::().map_err(|_| Halt::Defer) + Ok(result.downcast_into::().map_err(PyErr::from)?) } fn expand_value( @@ -546,14 +553,14 @@ fn merged_strings(base: &Bound<'_, PyAny>, overlay: &Bound<'_, PyAny>) -> Built< Ok(merged) } -/// Return `_as_string_list(value)` when every item is a string. +/// Return `_as_string_list(value)`: `str()` of each list or tuple item. fn string_items(value: &Bound<'_, PyAny>) -> Built> { let Some(items) = sequence_items(value) else { return Ok(Vec::new()); }; items .iter() - .map(|item| PyNode(item.clone()).text().ok_or(Halt::Defer)) + .map(|item| Ok(item.str()?.to_string_lossy().into_owned())) .collect() } @@ -609,7 +616,11 @@ fn string_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built) -> Built { - PyNode(value.clone()).text().ok_or(Halt::Defer) + Ok(value + .downcast::() + .map_err(PyErr::from)? + .to_string_lossy() + .into_owned()) } fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option>> { @@ -622,19 +633,10 @@ fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option Built { - match presence { - Presence::Present => Ok(true), - Presence::Absent => Ok(false), - Presence::Deferred => Err(Halt::Defer), - } -} - fn entry_has_template(values: &Bound<'_, PyDict>, key: &str) -> Built { - match values.get_item(key)? { - Some(value) => present(contains_template(&PyNode(value))), - None => Ok(false), - } + Ok(values + .get_item(key)? + .is_some_and(|value| contains_template(&PyNode(value)))) } /// Check header tags as `_validate_model_header_tags` does. @@ -658,11 +660,6 @@ fn storage_free_values<'py>( resolved: &Bound<'py, PyDict>, header: &Bound<'py, PyDict>, ) -> Built> { - if let Some(materialized) = set_entry(resolved, MATERIALIZED_KEY)? - && !materialized.is_instance_of::() - { - return Err(Halt::Defer); - } let values = PyDict::new(py); for (key, value) in resolved.iter() { let storage = [RETENTION_KEY, TABLE_TYPE_KEY] @@ -685,7 +682,6 @@ fn storage_free_values<'py>( fn check_no_config_macros(values: &Bound<'_, PyDict>) -> Built<()> { match first_macro_path(&PyNode(values.clone().into_any()), &HOOK_KEYS) { MacroPath::Absent => Ok(()), - MacroPath::Deferred => Err(Halt::Defer), MacroPath::Found(path) => Err(Halt::Config(ConfigError::compile(format!( "model config field '{}' does not allow macros", path.join(".") @@ -698,11 +694,11 @@ fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { .keys() .iter() .map(|key| { - if key.is_exact_instance_of::() { - key.extract::().map_err(Halt::from) - } else { - Err(Halt::Defer) - } + Ok(key + .downcast::() + .map_err(PyErr::from)? + .to_string_lossy() + .into_owned()) }) .collect::>>()?; keys.sort(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 6e40b3e283..e4618d4617 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -2,10 +2,13 @@ use std::collections::HashSet; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pymethods}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{IntoPyObject, pyclass, pymethods}; +use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_model_config::model_validation::main::validate_model_config::validate_model_config; +use sqlbuild_model_config::model_validation::main::validate_model_references::validate_model_references; use sqlbuild_model_config::model_validation::models::{ ModelReference, ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -23,8 +26,9 @@ type ResourceNames = ( HashSet, ); -/// References, declared schema columns, unmanaged retention and declared table type. -type ModelFacts<'py> = (Bound<'py, PyTuple>, Option>, bool, bool); +/// References as `(kind, name, externally rejected)`, declared schema columns, unmanaged +/// retention and declared table type. +type ModelFacts = (Vec<(String, String, bool)>, Option>, bool, bool); /// The project facts every model's validation reads, captured once per compile. #[pyclass(module = "sqlbuild._native", frozen)] @@ -39,10 +43,19 @@ impl NativeModelValidator { names: ResourceNames, custom_materializations: HashSet, microbatch_concurrency: bool, - ) -> Self { + python: (u8, u8), + unicode_version: &str, + ) -> PyResult { let (models, seeds, sources, functions, table_functions) = names; - Self { + let python = python_text(python, unicode_version).ok_or_else(|| { + PyValueError::new_err(format!( + "no Python string semantics for Python {}.{} with Unicode {unicode_version}", + python.0, python.1 + )) + })?; + Ok(Self { project: ProjectValidationFacts { + python, custom_materializations, microbatch_concurrency, models, @@ -51,29 +64,29 @@ impl NativeModelValidator { functions, table_functions, }, - } + }) } - /// Return `True` when the validators accept, the first error, or `False` to run Python's. + /// Return `None` when the validators accept, the first error, or the index of the first + /// reference whose external resolver rejected it. fn validate( &self, py: Python<'_>, values: Bound<'_, PyDict>, model: (String, String, String), - facts: ModelFacts<'_>, + facts: ModelFacts, ) -> PyResult> { compiler_guard(|| { let (model_name, query_sql, relative_path) = model; let (references, declared_columns, retention_unmanaged, table_type_declared) = facts; - let references = references - .iter() - .map(|reference| { - Ok(ModelReference { - kind: reference.getattr("ref_kind")?.extract()?, - name: reference.getattr("ref_name")?.extract()?, - }) + let references: Vec = references + .into_iter() + .map(|(kind, name, externally_rejected)| ModelReference { + kind, + name, + externally_rejected, }) - .collect::>>()?; + .collect(); let entries: Vec<(PyNode<'_>, PyNode<'_>)> = values .iter() .map(|(key, value)| (PyNode(key), PyNode(value))) @@ -87,19 +100,54 @@ impl NativeModelValidator { retention_unmanaged, table_type_declared, }; - Ok( - match validate_model_config(entries, &self.project, &facts) { - Ok(()) => PyBool::new(py, true).to_owned().into_any().unbind(), - Err(ValidationStop::Defer) => { - PyBool::new(py, false).to_owned().into_any().unbind() - } - Err(ValidationStop::Error(error)) => native_config_error(py, error)?.into_any(), - }, - ) + match validate_model_config(entries, &self.project, &facts) { + Ok(()) => Ok(py.None()), + Err(stop) => stop_object(py, stop), + } + }) + } + + /// Check references alone: `None`, the first error, or a rejected dbt reference's index. + fn references( + &self, + py: Python<'_>, + model: (String, String), + references: Vec<(String, String, bool)>, + ) -> PyResult> { + compiler_guard(|| { + let (model_name, relative_path) = model; + let references: Vec = references + .into_iter() + .map(|(kind, name, externally_rejected)| ModelReference { + kind, + name, + externally_rejected, + }) + .collect(); + let facts = ModelValidationFacts { + model_name: &model_name, + relative_path: &relative_path, + references: &references, + declared_columns: None, + query_sql: "", + retention_unmanaged: false, + table_type_declared: false, + }; + match validate_model_references(&self.project, &facts) { + Ok(()) => Ok(py.None()), + Err(stop) => stop_object(py, stop), + } }) } } +fn stop_object(py: Python<'_>, stop: ValidationStop) -> PyResult> { + match stop { + ValidationStop::External(index) => Ok(index.into_pyobject(py)?.into_any().unbind()), + ValidationStop::Error(error) => Ok(native_config_error(py, error)?.into_any()), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 936ccaa273..4414cbafb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,14 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - config_errors, config_presence, config_templates, header_metadata, model_config_builder, - model_validator, + config_errors, config_templates, header_metadata, model_config_builder, model_validator, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { config_errors::register(module)?; header_metadata::register(module)?; - config_presence::register(module)?; config_templates::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 70621c736b..3c0c3fa5e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -11,7 +11,6 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; - const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 755ea5799a..a624e65ed6 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -4,7 +4,6 @@ use crate::engine::tests::test_types::{ use crate::rules::_helpers::evaluation::normalize_rules_sql; use crate::rules::_helpers::numeric_decisions::compact_sql; use crate::sql_lint::_helpers::preparation::prepare; -use sqlbuild_sqltext::compiler::main::sql_references::extract; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; @@ -25,7 +24,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=aandb", expected_snowflake_compact: "a=aandb", expected_lint_site: Some("@m(a x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "single quote containing open paren", @@ -38,7 +36,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='a(b'andb", expected_snowflake_compact: "a='a(b'andb", expected_lint_site: Some("@m('a(b' x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "single quote containing close paren", @@ -51,7 +48,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='a)b'andb", expected_snowflake_compact: "a='a)b'andb", expected_lint_site: Some("@m('a)b' x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "doubled single quote", @@ -64,7 +60,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='it''s )'andb", expected_snowflake_compact: "a='it''s )'andb", expected_lint_site: Some("@m('it''s )' x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "backslash escaped quote before close paren", @@ -77,7 +72,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='a\\')' and b", expected_snowflake_compact: "a='a\\')'andb", expected_lint_site: Some("@m('a\\')' x)"), - expected_reference_fast_path: false, }, SqlScannerTestCase { description: "backslash escaped quote before open paren", @@ -90,7 +84,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='a\\'(' and b", expected_snowflake_compact: "a='a\\'('andb", expected_lint_site: Some("@m('a\\'(' x)"), - expected_reference_fast_path: false, }, SqlScannerTestCase { description: "double quote containing close paren", @@ -103,7 +96,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=\"a)b\"andb", expected_snowflake_compact: "a=\"a)b\"andb", expected_lint_site: Some("@m(\"a)b\" x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "backtick containing close paren", @@ -116,7 +108,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=`a)b`andb", expected_snowflake_compact: "a=`a)b`andb", expected_lint_site: Some("@m(`a)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "doubled backtick", @@ -129,7 +120,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=`a``)b`andb", expected_snowflake_compact: "a=`a``)b`andb", expected_lint_site: Some("@m(`a``)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "line comment", @@ -142,7 +132,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=andb", expected_snowflake_compact: "a=andb", expected_lint_site: Some("@m(-- )\n x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "block comment", @@ -155,7 +144,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=andb", expected_snowflake_compact: "a=andb", expected_lint_site: Some("@m(/* ) */ x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "nested block comment", @@ -168,7 +156,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=)*/andb", expected_snowflake_compact: "a=)*/andb", expected_lint_site: Some("@m(/* /* ) */ )"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "dollar quote", @@ -181,7 +168,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=$$)$$andb", expected_snowflake_compact: "a=$$)$$andb", expected_lint_site: Some("@m($$ ) $$ x)"), - expected_reference_fast_path: true, }, SqlScannerTestCase { description: "unterminated quote", @@ -194,7 +180,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='abc and b", expected_snowflake_compact: "a='abc and b", expected_lint_site: Some(""), - expected_reference_fast_path: false, }, SqlScannerTestCase { description: "unterminated block comment", @@ -207,7 +192,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a=", expected_snowflake_compact: "a=", expected_lint_site: Some(""), - expected_reference_fast_path: false, }, SqlScannerTestCase { description: "multibyte utf8", @@ -220,7 +204,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_compact: "a='é)ü'ñandb", expected_snowflake_compact: "a='é)ü'ñandb", expected_lint_site: None, - expected_reference_fast_path: true, }, ]; @@ -298,12 +281,6 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann "SQL lint macro site: {}", test_case.description ); - assert_eq!( - extract(&format!("SELECT {fragment} __ref(\"b\")")).is_some(), - test_case.expected_reference_fast_path, - "SQL reference fast path: {}", - test_case.description - ); } Ok(()) } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 8f825fa48a..c3476b1906 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -208,7 +208,6 @@ pub(crate) struct SqlScannerTestCase { pub(crate) expected_snowflake_compact: &'static str, /// `None` when SQL lint declines non-ASCII input; `Some("")` when no macro site closes. pub(crate) expected_lint_site: Option<&'static str>, - pub(crate) expected_reference_fast_path: bool, } pub(crate) struct IncrementalRulesTestCase { diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index a54814abb6..213734a8e9 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -874,4 +874,40 @@ def engine_error_cases() -> tuple[FailureCase, ...]: "ISO timestamp: hour must be in 0..23" ), ), + failure_case( + name="engine-error-non-ascii-replay-duration", + expected_code="P001", + files={ + FAILURE_MART_PATH: _MART_HEADER_START + + " materialized incremental,\n incremental_strategy append,\n" + + ' replay_on_change "bounded-\u0661d",\n);\n\n' + + 'SELECT customer_id\nFROM __ref("stg_orders")\n' + }, + expected_message=( + "model 'customer_totals': replay_on_change '\u0661d' uses digits outside ASCII" + ), + expected_help=( + "write replay_on_change with ASCII digits 0-9, add this to the MODEL header:\n" + " MODEL (\n replay_on_change 'bounded-14d',\n" + " ...\n );" + ), + ), + failure_case( + name="engine-error-oversized-retention", + expected_code="P001", + files={ + FAILURE_MART_PATH: _MART_HEADER_START + + ' materialized table,\n time_travel_retention "99999999999999999999d",\n);\n\n' + + 'SELECT customer_id\nFROM __ref("stg_orders")\n' + }, + expected_message=( + "model 'customer_totals': time_travel_retention '99999999999999999999d' has a " + "number larger than a 64-bit integer" + ), + expected_help=( + "use a time_travel_retention whose days fit in 64 bits, add this to the MODEL " + "header:\n MODEL (\n time_travel_retention '7d',\n" + " ...\n );" + ), + ), ) diff --git a/scripts/compiler_differential/native_fallbacks.toml b/scripts/compiler_differential/native_fallbacks.toml index 37349b0142..a162019355 100644 --- a/scripts/compiler_differential/native_fallbacks.toml +++ b/scripts/compiler_differential/native_fallbacks.toml @@ -88,7 +88,7 @@ engine = "native" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 234, fixture = 12, seed = 44 } +counts = { example = 12, failure = 236, fixture = 12, seed = 44 } [[entry]] engine = "native" @@ -102,7 +102,7 @@ engine = "native" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 301, fixture = 12, seed = 50 } +counts = { example = 12, failure = 303, fixture = 12, seed = 50 } [[entry]] engine = "native" @@ -158,14 +158,14 @@ engine = "native" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 311, fixture = 64, seed = 890 } +counts = { example = 56, failure = 313, fixture = 64, seed = 890 } [[entry]] engine = "native" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 249, fixture = 15, seed = 121 } +counts = { example = 29, failure = 251, fixture = 15, seed = 121 } [[entry]] engine = "native" @@ -179,21 +179,14 @@ engine = "native" stage = "model_config" site = "model_config.native" kind = "validations" -counts = { example = 56, failure = 372, fixture = 60, seed = 1091 } - -[[entry]] -engine = "native" -stage = "model_config" -site = "model_config.validators" -kind = "deferred" -counts = { failure = 2, fixture = 4, seed = 3 } +counts = { example = 56, failure = 397, fixture = 64, seed = 1100 } [[entry]] engine = "native" stage = "model_loop" site = "model_loop.cursor_intrinsic_validation" kind = "deferred" -counts = { example = 56, failure = 402, fixture = 64, seed = 1100 } +counts = { example = 56, failure = 403, fixture = 64, seed = 1100 } [[entry]] engine = "native" @@ -207,21 +200,21 @@ engine = "native" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 423, fixture = 64, seed = 1103 } [[entry]] engine = "native" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 477, fixture = 72, seed = 1163 } [[entry]] engine = "native" stage = "reference_extraction" site = "reference_extraction.native" kind = "reference_scans" -counts = { example = 250, failure = 463, fixture = 249, seed = 1523 } +counts = { example = 250, failure = 464, fixture = 249, seed = 1523 } [[entry]] engine = "native-preview" @@ -319,7 +312,7 @@ engine = "native-preview" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 234, fixture = 12, seed = 44 } +counts = { example = 12, failure = 236, fixture = 12, seed = 44 } [[entry]] engine = "native-preview" @@ -333,7 +326,7 @@ engine = "native-preview" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 301, fixture = 12, seed = 50 } +counts = { example = 12, failure = 303, fixture = 12, seed = 50 } [[entry]] engine = "native-preview" @@ -431,14 +424,14 @@ engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "config_builds" -counts = { example = 56, failure = 311, fixture = 64, seed = 890 } +counts = { example = 56, failure = 313, fixture = 64, seed = 890 } [[entry]] engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 249, fixture = 15, seed = 121 } +counts = { example = 29, failure = 251, fixture = 15, seed = 121 } [[entry]] engine = "native-preview" @@ -452,21 +445,14 @@ engine = "native-preview" stage = "model_config" site = "model_config.native" kind = "validations" -counts = { example = 56, failure = 372, fixture = 60, seed = 1091 } - -[[entry]] -engine = "native-preview" -stage = "model_config" -site = "model_config.validators" -kind = "deferred" -counts = { failure = 2, fixture = 4, seed = 3 } +counts = { example = 56, failure = 397, fixture = 64, seed = 1100 } [[entry]] engine = "native-preview" stage = "model_loop" site = "model_loop.cursor_intrinsic_validation" kind = "deferred" -counts = { example = 56, failure = 402, fixture = 64, seed = 1100 } +counts = { example = 56, failure = 403, fixture = 64, seed = 1100 } [[entry]] engine = "native-preview" @@ -480,14 +466,14 @@ engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 423, fixture = 64, seed = 1103 } [[entry]] engine = "native-preview" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 477, fixture = 72, seed = 1163 } [[entry]] engine = "native-preview" @@ -501,7 +487,7 @@ engine = "native-preview" stage = "reference_extraction" site = "reference_extraction.native" kind = "reference_scans" -counts = { example = 250, failure = 463, fixture = 249, seed = 1523 } +counts = { example = 250, failure = 464, fixture = 249, seed = 1523 } [[entry]] engine = "native-preview" @@ -550,7 +536,7 @@ engine = "python" stage = "declaration_scopes" site = "declaration_scopes.native" kind = "scope_indexes" -counts = { example = 12, failure = 234, fixture = 12, seed = 44 } +counts = { example = 12, failure = 236, fixture = 12, seed = 44 } [[entry]] engine = "python" @@ -564,7 +550,7 @@ engine = "python" stage = "discovery" site = "discovery.native" kind = "model_file_listings" -counts = { example = 12, failure = 301, fixture = 12, seed = 50 } +counts = { example = 12, failure = 303, fixture = 12, seed = 50 } [[entry]] engine = "python" @@ -573,30 +559,65 @@ site = "macro_calls.native" kind = "parsed_arguments" counts = { example = 32, failure = 16, fixture = 28, seed = 840 } +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.header_metadata" +kind = "stale" +counts = { seed = 19 } + +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.header_metadata" +kind = "unsupported" +counts = { failure = 1 } + +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.native" +kind = "config_builds" +counts = { example = 56, failure = 313, fixture = 64, seed = 890 } + [[entry]] engine = "python" stage = "model_config" site = "model_config.native" kind = "effective_vars" -counts = { example = 29, failure = 249, fixture = 15, seed = 121 } +counts = { example = 29, failure = 251, fixture = 15, seed = 121 } [[entry]] engine = "python" stage = "model_config" site = "model_config.native" kind = "template_expansions" -counts = { example = 286, failure = 1597, fixture = 333, seed = 2503 } +counts = { example = 230, failure = 1588, fixture = 333, seed = 1524 } + +[[entry]] +engine = "python" +stage = "model_config" +site = "model_config.native" +kind = "validations" +counts = { example = 56, failure = 397, fixture = 64, seed = 1100 } [[entry]] engine = "python" stage = "model_loop" site = "model_loop.native" kind = "declaration_expansions" -counts = { example = 56, failure = 422, fixture = 64, seed = 1103 } +counts = { example = 56, failure = 423, fixture = 64, seed = 1103 } [[entry]] engine = "python" stage = "model_loop" site = "model_loop.native" kind = "variable_substitutions" -counts = { example = 56, failure = 473, fixture = 72, seed = 1163 } +counts = { example = 56, failure = 477, fixture = 72, seed = 1163 } + +[[entry]] +engine = "python" +stage = "reference_extraction" +site = "reference_extraction.native" +kind = "reference_scans" +counts = { example = 250, failure = 463, fixture = 249, seed = 1523 } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index aa85d3549e..6e4168e86d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -175,8 +175,6 @@ def parse_model_header_metadata( ) -> list[ tuple[tuple[Any, ...] | NativeConfigError, tuple[Any, ...] | NativeConfigError | None] | str ]: ... -def config_contains_template(value: object) -> bool | None: ... -def config_contains_macro_call(value: object) -> bool | None: ... def expand_config_templates( value: object, sources: tuple[dict[str, object], object, dict[str, str | None]], @@ -193,6 +191,7 @@ class NativeModelConfigBuilder: sources: tuple[dict[str, object], object], run: tuple[str | None, str], target_namespace: tuple[str | None, str | None] | None, + python: tuple[tuple[int, int], str], ) -> None: ... def path_default(self, model_path: str) -> str | NativeConfigError | None: ... def build( @@ -206,7 +205,6 @@ class NativeModelConfigBuilder: list[tuple[str, str]], ] | NativeConfigError - | None ): ... class NativeModelValidator: @@ -215,13 +213,18 @@ class NativeModelValidator: names: tuple[set[str], set[str], set[str], set[str], set[str]], custom_materializations: set[str], microbatch_concurrency: bool, + python: tuple[int, int], + unicode_version: str, ) -> None: ... def validate( self, values: dict[str, object], model: tuple[str, str, str], - facts: tuple[tuple[object, ...], list[str] | None, bool, bool], - ) -> bool | NativeConfigError: ... + facts: tuple[list[tuple[str, str, bool]], list[str] | None, bool, bool], + ) -> int | NativeConfigError | None: ... + def references( + self, model: tuple[str, str], references: list[tuple[str, str, bool]] + ) -> int | NativeConfigError | None: ... class SqlReferenceScanner: def __init__(self, syntax: dict[str, object]) -> None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 07fa71fd0d..95282af174 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -15,20 +15,16 @@ from sqlbuild.compiler.compile._helpers.attachment.model_config import ( _model_sql_validation_gate, _resolve_model_schema, - _validate_model_header_tags, build_layered_model_values, build_model_header_schema_entry, build_native_model_config, - contains_config_macro_calls, - contains_config_templates, - find_matching_path_default, find_schema_model_match, native_model_config_session, native_path_default, strip_model_header_metadata_from_config, validate_declared_schema_models_are_attached, validate_model_config, - validate_no_macros_in_config_value, + validate_model_references, ) from sqlbuild.compiler.compile._helpers.attachment.references import ( build_known_function_names, @@ -36,12 +32,7 @@ build_known_seed_names, build_known_source_names, build_known_table_function_names, - validate_model_references, -) -from sqlbuild.compiler.compile._helpers.config.namespace_validation import ( - validate_preserved_logical_namespace, ) -from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( rejected_sql_analysis_opt_out, ) @@ -59,11 +50,7 @@ from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile._helpers.render.arguments import render_parameterized_sql from sqlbuild.compiler.compile._helpers.render.context_templates import ( - apply_environment_database_schema_overrides, build_model_context_values, - resolve_chained_model_context_templates, - resolve_early_model_templates, - resolve_target_context_templates, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, @@ -92,8 +79,6 @@ ) from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, - MODEL_FULL_REFRESH_CONFIG_KEY, - MODEL_SCHEMA_CONFIG_KEY, ) from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroArgumentError from sqlbuild.compiler.compile.models import ( @@ -113,8 +98,6 @@ MacroContext, MacroExpansionResult, MappedOffset, - ModelConfigBuildRequest, - ModelConfigScanCache, ModelHeaderColumnCache, ModelInputBuildContext, ModelResourceNames, @@ -157,7 +140,6 @@ scan_native_declaration_references, ) from sqlbuild.compiler.model_loop.types import NativeDeclarationScan -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, @@ -178,7 +160,6 @@ from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, - MaterializationDefaultsConfig, ProjectConfig, SchemaColumn, SchemaModelEntry, @@ -284,7 +265,6 @@ class _ModelInputLoop: sql_hook_definitions: dict[str, DiscoveredSqlHookFile] legacy_schema_files: tuple[DiscoveredSchemaFile, ...] model_header_column_cache: ModelHeaderColumnCache - config_scan_cache: ModelConfigScanCache reusable_config_cache: _ReusableModelConfigCache declaration_cache: _VisibleModelDeclarationCache native_header_metadata: dict[Path, NativeHeaderMetadata] @@ -493,7 +473,6 @@ def _build_model_inputs( custom_materialization_names: frozenset[str] = frozenset( mf.name for mf in discovered_inputs.materialization_files ) - native_model_config: bool = native_stage_enabled(NativeStage.MODEL_CONFIG) names: ModelResourceNames = ModelResourceNames( models=known_model_names, seeds=known_seed_names, @@ -519,20 +498,16 @@ def _build_model_inputs( names=names, settings=effective_settings, external_sql_reference_resolver=external_sql_reference_resolver, - native_config=( - native_model_config_session( - inputs=NativeModelConfigInputs( - project_config=discovered_inputs.project_config, - target_config=context.target_config, - effective_vars=effective_vars, - effective_target_name=context.effective_target_name, - run_id=context.run_id, - microbatch_concurrency=effective_settings.microbatch_concurrency, - ), - names=names, - ) - if native_model_config - else None + native_config=native_model_config_session( + inputs=NativeModelConfigInputs( + project_config=discovered_inputs.project_config, + target_config=context.target_config, + effective_vars=effective_vars, + effective_target_name=context.effective_target_name, + run_id=context.run_id, + microbatch_concurrency=effective_settings.microbatch_concurrency, + ), + names=names, ), ), ) @@ -567,16 +542,13 @@ def _build_model_inputs( sql_hook_definitions=sql_hook_definitions, legacy_schema_files=legacy_schema_files, model_header_column_cache=ModelHeaderColumnCache(), - config_scan_cache=ModelConfigScanCache(native=native_model_config), reusable_config_cache=_ReusableModelConfigCache( defaults=discovered_inputs.project_config.defaults, path_defaults=discovered_inputs.project_config.path_defaults, target_config=context.target_config, ), declaration_cache=_VisibleModelDeclarationCache.build(context), - native_header_metadata=( - parse_native_header_metadata(model_files=render_files) if native_model_config else {} - ), + native_header_metadata=parse_native_header_metadata(model_files=render_files), native_declaration_references=dict( zip( (model_file.file_path for model_file in prepared_files), @@ -619,11 +591,9 @@ def _build_model_input( sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = loop.sql_hook_definitions legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = loop.legacy_schema_files model_header_column_cache: ModelHeaderColumnCache = loop.model_header_column_cache - config_scan_cache: ModelConfigScanCache = loop.config_scan_cache reusable_config_cache: _ReusableModelConfigCache = loop.reusable_config_cache declaration_cache: _VisibleModelDeclarationCache = loop.declaration_cache effective_vars: dict[str, object] = context.effective_vars - target_config: TargetConfig | None = context.target_config effective_target_name: str | None = context.effective_target_name run_id: str = context.run_id macro_context: MacroContext = context.macro_context @@ -634,49 +604,20 @@ def _build_model_input( declarations: _VisibleModelDeclarations = declaration_cache.for_model( model_file=model_file, consumer=model_identity ) - native_config: NativeModelConfigSession | None = validation_context.validators.native_config - matched_path_default: str | None = ( - find_matching_path_default( - model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, - ) - if native_config is None - else native_path_default(session=native_config, model_file=model_file) + native_config: NativeModelConfigSession = validation_context.validators.native_config + matched_path_default: str | None = native_path_default( + session=native_config, model_file=model_file ) effective_config: CompileModelConfig | None = reusable_config_cache.get( matched_path_default=matched_path_default, model_header_values=model_file.header_values, ) - if effective_config is None and native_config is not None: + if effective_config is None: effective_config = build_native_model_config( session=native_config, model_file=model_file, matched_path_default=matched_path_default, ) - if effective_config is not None: - reusable_config_cache.remember( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - config=effective_config, - ) - if effective_config is None: - effective_config = build_model_config( - request=ModelConfigBuildRequest( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - effective_vars=effective_vars, - target_config=target_config, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - materialization_defaults=( - discovered_inputs.project_config.materialization_defaults - ), - scan_cache=config_scan_cache, - ) - ) reusable_config_cache.remember( matched_path_default=matched_path_default, model_header_values=model_file.header_values, @@ -949,14 +890,9 @@ def _hook_expanded_config( validation_context: _ModelValidationContext, ) -> CompileModelConfig: validate_model_references( - references=hook_expansion.references, + context=validation_context.validators, model_file=model_file, - known_model_names=validation_context.known_model_names, - known_seed_names=validation_context.known_seed_names, - known_source_names=validation_context.known_source_names, - known_function_names=validation_context.known_function_names, - known_table_function_names=validation_context.known_table_function_names, - external_sql_reference_resolver=validation_context.external_sql_reference_resolver, + references=hook_expansion.references, ) return CompileModelConfig( values=hook_expansion.values, @@ -1312,123 +1248,6 @@ def build_effective_vars( return expand_effective_vars(values) -def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfig: - """Build the pre-semantic effective model config layers.""" - - defaults: DefaultsConfig = request.defaults - path_defaults: dict[str, dict[str, object]] = request.path_defaults - matched_path_default: str | None = request.matched_path_default - model_header_values: dict[str, object] = request.model_header_values - effective_vars: dict[str, object] = request.effective_vars - target_config: TargetConfig | None = request.target_config - model_name: str = request.model_name - effective_target_name: str | None = request.effective_target_name - run_id: str = request.run_id - materialization_defaults: MaterializationDefaultsConfig | None = ( - request.materialization_defaults - ) - scan_cache: ModelConfigScanCache | None = request.scan_cache - - _validate_model_header_tags(model_header_values=model_header_values, model_name=model_name) - layered_values: dict[str, object] = build_layered_model_values( - defaults=defaults, - path_defaults=path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_header_values, - ) - validate_model_hook_config(values=layered_values, model_name=model_name) - raw_hook_values: dict[str, object] = { - hook_key: layered_values[hook_key] - for hook_key in _MODEL_HOOK_KEYS - if hook_key in layered_values - } - for hook_key in raw_hook_values: - del layered_values[hook_key] - has_authored_templates: bool = contains_config_templates( - values=layered_values, scan_cache=scan_cache - ) - if has_authored_templates: - early_resolved_values: dict[str, object] = resolve_early_model_templates( - values=layered_values, - effective_vars=effective_vars, - effective_target_name=effective_target_name, - run_id=run_id, - ) - model_resolved_values: dict[str, object] = resolve_chained_model_context_templates( - values=early_resolved_values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - ) - else: - model_resolved_values = layered_values - raw_logical_schema: object | None = model_resolved_values.get(MODEL_SCHEMA_CONFIG_KEY) - raw_logical_database: object | None = model_resolved_values.get("database") - logical_schema: str | None = raw_logical_schema if isinstance(raw_logical_schema, str) else None - layer_schema_is_configured: bool = MODEL_SCHEMA_CONFIG_KEY in model_header_values or ( - matched_path_default is not None - and MODEL_SCHEMA_CONFIG_KEY in path_defaults[matched_path_default] - ) - logical_database: str | None = ( - raw_logical_database if isinstance(raw_logical_database, str) else None - ) - validate_preserved_logical_namespace( - resource_label=f"Model '{model_name}'", - logical_database=logical_database, - logical_schema=logical_schema, - target_config=target_config, - ) - model_resolved_values = apply_environment_database_schema_overrides( - values=model_resolved_values, - effective_vars=effective_vars, - target_config=target_config, - model_context_values=build_model_context_values( - values=model_resolved_values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=False, - ), - ) - target_resolved_values: dict[str, object] = ( - resolve_target_context_templates( - values=model_resolved_values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - ) - if has_authored_templates - or contains_template_data(model_resolved_values.get("database")) - or contains_template_data(model_resolved_values.get("schema")) - else model_resolved_values - ) - target_resolved_values.update(raw_hook_values) - target_resolved_values, retention, table_type = resolve_storage_policies( - resolved_values=target_resolved_values, - model_header_values=model_header_values, - materialization_defaults=materialization_defaults, - target_config=target_config, - model_name=model_name, - ) - if ( - MODEL_FULL_REFRESH_CONFIG_KEY not in model_header_values - and target_resolved_values.get("materialized") != MaterializationType.INCREMENTAL - ): - target_resolved_values.pop(MODEL_FULL_REFRESH_CONFIG_KEY, None) - if contains_config_macro_calls(values=target_resolved_values, scan_cache=scan_cache): - validate_model_config_has_no_macros(values=target_resolved_values) - return CompileModelConfig( - values=target_resolved_values, - model_header_keys=tuple(sorted(model_header_values)), - matched_path_default=matched_path_default, - logical_schema=logical_schema, - layer_schema=logical_schema if layer_schema_is_configured else None, - logical_database=logical_database, - time_travel_retention=retention, - table_type=table_type, - ) - - def expand_model_hook_macros_result( *, values: dict[str, object], @@ -1476,26 +1295,6 @@ def expand_model_hook_macros_result( ) -def validate_model_hook_config(*, values: dict[str, object], model_name: str) -> None: - hook_key: str - for hook_key in sorted(_MODEL_HOOK_KEYS): - if hook_key not in values: - continue - raw_value: object = values[hook_key] - if not isinstance(raw_value, list | tuple): - raise CompileInputError( - f"model '{model_name}' {hook_key} must be a list of typed hook entries" - ) - hook_entry: object - for hook_entry in raw_value: - if isinstance(hook_entry, SqlHookEntry | NamedSqlHookEntry | PythonHookEntry): - continue - raise CompileInputError( - f"model '{model_name}' {hook_key} entries must use typed inline_sql(...), " - "sql(...), or python(...) hook syntax" - ) - - def validate_python_hook_config( *, values: dict[str, object], @@ -1871,12 +1670,6 @@ def _index_sql_hook_definitions( return definitions -def validate_model_config_has_no_macros(*, values: dict[str, object]) -> None: - """Reject macro calls in declarative model config while allowing hook SQL strings.""" - - validate_no_macros_in_config_value(value=values, path=()) - - def _authored_model_location( *, model_file: DiscoveredSqlModelFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 3e4d49941d..0998ecaf7a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -2,6 +2,8 @@ from __future__ import annotations +import sys +import unicodedata from dataclasses import replace from pathlib import Path from typing import cast @@ -9,7 +11,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.authored_values.main._optional_named_string import optional_named_string -from sqlbuild.compiler.compile._helpers.attachment.references import validate_model_references +from sqlbuild.compiler.compile._helpers.attachment.references import validate_external_reference from sqlbuild.compiler.compile._helpers.audit_factories.core import ( merge_validated_model_audits, parse_model_header_audit_factories, @@ -19,25 +21,14 @@ ) from sqlbuild.compiler.compile._helpers.config.model_validation import ( validate_column_migration_config, - validate_contract_config, - validate_custom_materialization_config, - validate_incremental_config, - validate_microbatch_project_capability, - validate_model_migration_config, - validate_non_incremental_config, - validate_placeholder_config, - validate_snapshot_config, - validate_storage_policies, ) from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type from sqlbuild.compiler.compile._helpers.render.templating import ( - contains_template_data, record_template_reads, ) from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.constants import ( - MACRO_CALL_PATTERN, MODEL_AUDIT_OVERRIDE_KEYS, MODEL_HEADER_METADATA_KEYS, ) @@ -46,8 +37,7 @@ CachedModelHeaderColumns, CompileModelConfig, CompileModelInput, - IdentityPresenceCache, - ModelConfigScanCache, + CompileSqlReference, ModelHeaderColumnCache, ModelResourceNames, ModelValidationRequest, @@ -68,16 +58,11 @@ from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.main.report_native_fallback import report_native_fallback from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage -from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( - native_config_contains_macro_call, -) -from sqlbuild.compiler.model_config.main._native_config_contains_template import ( - native_config_contains_template, -) from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.compiler.path_defaults.main._select import select_path_default from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, @@ -100,6 +85,7 @@ MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT, ) +_PYTHON_VERSION: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) _MATERIALIZED_CONFIG_KEY: str = "materialized" type _BuiltNamespace = tuple[str | None, bool, str | None] type _StorageOverrides = tuple[tuple[int | None, bool] | None, str | None] @@ -114,107 +100,6 @@ _MODEL_SCHEMA_KEY: str = "model_schema" -def _contains_template_data_cached(*, value: object, cache: IdentityPresenceCache | None) -> bool: - if cache is None or not isinstance(value, dict | list | tuple): - return contains_template_data(value) - cached: bool | None = cache.get(value) - if cached is not None: - return cached - if isinstance(value, dict): - result: bool = any( - _contains_template_data_cached(value=item, cache=cache) for item in value.values() - ) - else: - result = any(_contains_template_data_cached(value=item, cache=cache) for item in value) - cache.put(value=value, result=result) - return result - - -def contains_config_templates( - *, values: dict[str, object], scan_cache: ModelConfigScanCache | None -) -> bool: - """Return whether layered model config holds a `${...}` template, scanning natively if set.""" - - if scan_cache is not None and scan_cache.native: - native: bool | None = native_config_contains_template(values) - if native is not None: - return native - report_native_fallback(site=NativeFallbackSite.CONFIG_PRESENCE_SCAN, kind="templates") - return _contains_template_data_cached( - value=values, cache=scan_cache.template_presence if scan_cache is not None else None - ) - - -def contains_config_macro_calls( - *, values: dict[str, object], scan_cache: ModelConfigScanCache | None -) -> bool: - """Return whether model config outside hooks holds a macro call, scanning natively if set.""" - - if scan_cache is not None and scan_cache.native: - native: bool | None = native_config_contains_macro_call( - [value for key, value in values.items() if key not in _MODEL_HOOK_KEYS] - ) - if native is not None: - return native - report_native_fallback(site=NativeFallbackSite.CONFIG_PRESENCE_SCAN, kind="macros") - return _contains_model_config_macro_cached( - values=values, cache=scan_cache.macro_presence if scan_cache is not None else None - ) - - -def _contains_model_config_macro_cached( - *, values: dict[str, object], cache: IdentityPresenceCache | None -) -> bool: - return any( - _contains_macro_data_cached(value=value, cache=cache) - for key, value in values.items() - if key not in _MODEL_HOOK_KEYS - ) - - -def _contains_macro_data_cached(*, value: object, cache: IdentityPresenceCache | None) -> bool: - if isinstance(value, str): - return MACRO_CALL_PATTERN.search(value) is not None - if not isinstance(value, dict | list | tuple): - return False - if cache is not None: - cached: bool | None = cache.get(value) - if cached is not None: - return cached - if isinstance(value, dict): - result: bool = any( - _contains_macro_data_cached(value=item, cache=cache) for item in value.values() - ) - else: - result = any(_contains_macro_data_cached(value=item, cache=cache) for item in value) - if cache is not None: - cache.put(value=value, result=result) - return result - - -def validate_no_macros_in_config_value(*, value: object, path: tuple[str, ...]) -> None: - """Recursively reject macro calls outside hook fields.""" - - if path and path[0] in _MODEL_HOOK_KEYS: - return - if isinstance(value, str): - if MACRO_CALL_PATTERN.search(value) is not None: - field_path: str = ".".join(path) if path else "" - raise CompileInputError(f"model config field '{field_path}' does not allow macros") - return - if isinstance(value, dict): - key: object - item_value: object - for key, item_value in value.items(): - if isinstance(key, str): - validate_no_macros_in_config_value(value=item_value, path=(*path, key)) - return - if isinstance(value, list | tuple): - item: object - for item in value: - validate_no_macros_in_config_value(value=item, path=path) - - def build_layered_model_values( *, defaults: DefaultsConfig, @@ -313,24 +198,6 @@ def _as_string_list(value: object) -> list[str]: return [] -def _validate_model_header_tags( - *, - model_header_values: dict[str, object], - model_name: str, -) -> None: - """Validate that tags in a MODEL header is a list of strings.""" - - raw_tags: object | None = model_header_values.get("tags") - if raw_tags is None: - return - if not isinstance(raw_tags, list): - raise CompileInputError(f"model '{model_name}' tags must be a list") - item: object - for item in raw_tags: - if not isinstance(item, str): - raise CompileInputError(f"model '{model_name}' tags entries must be strings") - - def build_model_header_schema_entry( *, model_name: str, @@ -885,11 +752,14 @@ def native_model_config_session( (inputs.effective_vars, UnicodeEnvironment()), (inputs.effective_target_name, inputs.run_id), None if target is None else (target.database, target.schema), + (_PYTHON_VERSION, unicodedata.unidata_version), ), validator=_native.NativeModelValidator( (names.models, names.seeds, names.sources, names.functions, names.table_functions), set(names.custom_materializations), inputs.microbatch_concurrency, + _PYTHON_VERSION, + unicodedata.unidata_version, ), inherited_storage={ materialized: ( @@ -931,15 +801,12 @@ def build_native_model_config( session: NativeModelConfigSession, model_file: DiscoveredSqlModelFile, matched_path_default: str | None, -) -> CompileModelConfig | None: - """Build a model's effective config natively, raising Python's error; None defers to Python.""" +) -> CompileModelConfig: + """Build a model's effective config natively, raising its first config error.""" - built: _BuiltConfig | _native.NativeConfigError | None = session.builder.build( + built: _BuiltConfig | _native.NativeConfigError = session.builder.build( model_file.header_values, matched_path_default, model_file.file_path.stem ) - if built is None: - report_native_fallback(site=NativeFallbackSite.CONFIG_BUILD) - return None report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="config_builds") if isinstance(built, _native.NativeConfigError): raise native_config_error(error=built, bridge_independent=True) @@ -982,9 +849,9 @@ def build_native_model_config( def native_model_validation( - *, session: NativeModelConfigSession, request: ModelValidationRequest -) -> bool | _native.NativeConfigError: - """Return True when every model validator accepts, the first error, or False to run Python.""" + *, session: NativeModelConfigSession, request: ModelValidationRequest, rejected: list[bool] +) -> int | _native.NativeConfigError | None: + """Validate natively: None, the first error, or the index of a rejected dbt reference.""" config: CompileModelConfig = request.config return session.validator.validate( @@ -995,7 +862,10 @@ def native_model_validation( str(request.model_file.relative_path), ), ( - request.references, + [ + (str(reference.ref_kind), reference.ref_name, reference_rejected) + for reference, reference_rejected in zip(request.references, rejected, strict=True) + ], ( None if request.declared_columns is None @@ -1018,63 +888,75 @@ def native_validation_error( def validate_model_config( *, context: ModelValidatorContext, request: ModelValidationRequest ) -> None: - """Validate one model natively, raising the first validator error; deferrals run Python.""" + """Validate one model natively, raising the first validator error.""" - if context.native_config is not None: - outcome: bool | _native.NativeConfigError = native_model_validation( - session=context.native_config, request=request + external_errors: list[Exception | None] = [ + _external_reference_error( + context=context, model_file=request.model_file, reference=reference ) - if isinstance(outcome, _native.NativeConfigError): - raise native_validation_error(error=outcome, values=request.config.values) - if outcome: - report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="validations") - return - report_native_fallback(site=NativeFallbackSite.CONFIG_VALIDATORS) - run_python_model_validators(context=context, request=request) + for reference in request.references + ] + outcome: int | _native.NativeConfigError | None = native_model_validation( + session=context.native_config, + request=request, + rejected=[error is not None for error in external_errors], + ) + report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="validations") + _raise_validation_outcome( + outcome=outcome, external_errors=external_errors, values=request.config.values + ) -def run_python_model_validators( - *, context: ModelValidatorContext, request: ModelValidationRequest +def validate_model_references( + *, + context: ModelValidatorContext, + model_file: DiscoveredSqlModelFile, + references: tuple[CompileSqlReference, ...], ) -> None: - """Run every Python model validator in order, raising the first error.""" - - names: ModelResourceNames = context.names - model_name: str = request.model_file.file_path.stem - validate_model_references( - references=request.references, - model_file=request.model_file, - known_model_names=names.models, - known_seed_names=names.seeds, - known_source_names=names.sources, - known_function_names=names.functions, - known_table_function_names=names.table_functions, - external_sql_reference_resolver=context.external_sql_reference_resolver, - ) - validate_incremental_config( - config=request.config, - model_name=model_name, - ref_count=len(request.references), - known_input_names=frozenset(reference.ref_name for reference in request.references), - declared_columns=request.declared_columns, - ) - validate_microbatch_project_capability( - config=request.config, settings=context.settings, model_name=model_name - ) - validate_contract_config(config=request.config, model_name=model_name) - validate_non_incremental_config(config=request.config, model_name=model_name) - validate_snapshot_config( - config=request.config, model_name=model_name, declared_columns=request.declared_columns - ) - validate_custom_materialization_config( - config=request.config, - model_name=model_name, - custom_materialization_names=names.custom_materializations, - ) - validate_storage_policies(config=request.config, model_name=model_name) - validate_model_migration_config(config=request.config, model_name=model_name) - validate_placeholder_config( - config=request.config, - model_name=model_name, - query_sql=request.query_sql, - custom_materialization_names=names.custom_materializations, + """Check references alone, as hook references are, raising the first problem.""" + + external_errors: list[Exception | None] = [ + _external_reference_error(context=context, model_file=model_file, reference=reference) + for reference in references + ] + outcome: int | _native.NativeConfigError | None = context.native_config.validator.references( + (model_file.file_path.stem, str(model_file.relative_path)), + [ + (str(reference.ref_kind), reference.ref_name, error is not None) + for reference, error in zip(references, external_errors, strict=True) + ], ) + _raise_validation_outcome(outcome=outcome, external_errors=external_errors, values={}) + + +def _raise_validation_outcome( + *, + outcome: int | _native.NativeConfigError | None, + external_errors: list[Exception | None], + values: dict[str, object], +) -> None: + if isinstance(outcome, _native.NativeConfigError): + raise native_validation_error(error=outcome, values=values) + if outcome is not None: + raise cast(Exception, external_errors[outcome]) + + +def _external_reference_error( + *, + context: ModelValidatorContext, + model_file: DiscoveredSqlModelFile, + reference: CompileSqlReference, +) -> Exception | None: + """The error the dbt manifest gives a `__dbt_ref`, or None when it accepts the reference.""" + + if reference.ref_kind != SqlReferenceKind.DBT_REF: + return None + try: + validate_external_reference( + reference=reference, + model_file=model_file, + external_sql_reference_resolver=context.external_sql_reference_resolver, + ) + except CompileInputError as error: + return error + return None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py index 8196bbcaf5..e527371794 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py @@ -18,114 +18,30 @@ from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -def validate_model_references( +def validate_external_reference( *, - references: tuple[CompileSqlReference, ...], + reference: CompileSqlReference, model_file: DiscoveredSqlModelFile, - known_model_names: set[str], - known_seed_names: set[str], - known_source_names: set[str], - known_function_names: set[str], - known_table_function_names: set[str], external_sql_reference_resolver: ExternalSqlReferenceResolver | None, ) -> None: - """Validate extracted model refs against discovered project inputs.""" + """Validate one `__dbt_ref` against the dbt manifest, which the external resolver owns.""" - reference: CompileSqlReference - for reference in references: - if ( - reference.ref_kind == SqlReferenceKind.REF - and reference.ref_name not in known_model_names - ): - if reference.ref_name in known_seed_names: - raise CompileInputError( - f"Model file {model_file.relative_path} references seed '{reference.ref_name}' " - f"with {SqlReferenceKind.REF.placeholder_call('...')}. Use " - f"{SqlReferenceKind.SEED.example_call(reference.ref_name, quote='"')} for seed " - f"references; {SqlReferenceKind.REF.function_name} only resolves models." - ) - raise CompileInputError( - f"Model file {model_file.relative_path} references unknown model " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SEED - and reference.ref_name not in known_seed_names - ): - if reference.ref_name in known_model_names: - raise CompileInputError( - f"Model file {model_file.relative_path} references model " - f"'{reference.ref_name}' " - f"with {SqlReferenceKind.SEED.placeholder_call('...')}. Use " - f"{SqlReferenceKind.REF.example_call(reference.ref_name, quote='"')} for model " - "references." - ) - raise CompileInputError( - f"Model file {model_file.relative_path} references unknown seed " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SOURCE - and reference.ref_name not in known_source_names - ): - raise CompileInputError( - f"Model file {model_file.relative_path} references unknown source " - f"'{reference.ref_name}'" - ) - if reference.ref_kind == SqlReferenceKind.DBT_REF: - if external_sql_reference_resolver is None: - raise CompileInputError( - f"Model file {model_file.relative_path} uses " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} " - "but no dbt manifest was found", - code="C214", - help=( - "Run dbt compile or configure dbt target_path so SQLBuild can read " - "manifest.json." - ), - ) - external_sql_reference_resolver.validate_reference( - ref_kind=reference.ref_kind, - ref_name=reference.ref_name, - ref_package=reference.ref_package, - owner_relative_sql_path=model_file.relative_path, - ) - if ( - reference.ref_kind == SqlReferenceKind.UDF - and reference.ref_name not in known_function_names - ): - raise CompileInputError( - f"Model file {model_file.relative_path} references unknown SQL function " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.UDF - and reference.ref_name in known_table_function_names - ): - raise CompileInputError( - f"Model file {model_file.relative_path} references table function " - f"'{reference.ref_name}' with {SqlReferenceKind.UDF.placeholder_call()}; " - f"use {SqlReferenceKind.TABLE_FUNCTION.placeholder_call()} in SQL contexts " - "that support table-valued functions" - ) - if ( - reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - and reference.ref_name not in known_function_names - ): - raise CompileInputError( - f"Model file {model_file.relative_path} references unknown table function " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - and reference.ref_name not in known_table_function_names - ): - raise CompileInputError( - f"Model file {model_file.relative_path} references scalar function " - f"'{reference.ref_name}' with " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()}; use " - f"{SqlReferenceKind.UDF.placeholder_call()} for scalar UDFs" - ) + if external_sql_reference_resolver is None: + raise CompileInputError( + f"Model file {model_file.relative_path} uses " + f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} " + "but no dbt manifest was found", + code="C214", + help=( + "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json." + ), + ) + external_sql_reference_resolver.validate_reference( + ref_kind=reference.ref_kind, + ref_name=reference.ref_name, + ref_package=reference.ref_package, + owner_relative_sql_path=model_file.relative_path, + ) def validate_table_function_call_arities( diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index 90b33177da..d922fda9dd 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -2,129 +2,19 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import UTC, date, datetime -from decimal import Decimal, InvalidOperation -from typing import cast - -from sqlbuild.compiler.authored_values.main._change_policy_header_help import ( - change_policy_header_help, -) -from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem from sqlbuild.compiler.compile.constants import ( - MICROBATCH_LIMIT_ACTION_KEY, - MICROBATCH_LIMIT_MAX_BATCHES_KEY, - MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY, - SQL_WILDCARD_TOKEN, - WATERMARK_CURSOR_INPUT_BLOCK_KEYS, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig -from sqlbuild.compiler.discovery.main.microbatch_guidance import microbatch_guidance -from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_CONFIG_KEY from sqlbuild.compiler.planner.types import ( - ContractPolicy, - CursorGrain, - CursorInputRole, - CursorType, - CursorWatermarkMode, - HistoricalInput, - IncrementalMode, - IncrementalStrategy, - InitialValidFrom, MaterializationType, - MicrobatchStrategy, - SnapshotFullRefreshPolicy, - SnapshotSchemaChangePolicy, - SnapshotStrategy, -) -from sqlbuild.cursor_algebra.constants import GRAIN_BATCH_SIZE -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help -from sqlbuild.spec.contracts.constants import ( - CURSOR_POLICY_DISABLED, - EFFECTIVE_BATCH_SIZE_TOKEN, - ZERO_DAY_CURSOR_DURATION, ) from sqlbuild.spec.contracts.main.get_config_str import get_config_str from sqlbuild.spec.contracts.models import ( - ResolvedTimeTravelRetention, SchemaColumn, - SettingsConfig, ) -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction _QUALIFIED_NAME_SEPARATOR: str = "." -_VALID_STRATEGIES: frozenset[str] = frozenset(s.value for s in IncrementalStrategy) -_VALID_CURSOR_TYPES: frozenset[str] = frozenset(ct.value for ct in CursorType) -_VALID_CURSOR_GRAINS: frozenset[str] = frozenset(cg.value for cg in CursorGrain) -_VALID_INCREMENTAL_MODES: frozenset[str] = frozenset(m.value for m in IncrementalMode) -_VALID_MICROBATCH_STRATEGIES: frozenset[str] = frozenset(m.value for m in MicrobatchStrategy) -_VALID_WATERMARK_MODES: frozenset[str] = frozenset(m.value for m in CursorWatermarkMode) -_VALID_CURSOR_INPUT_ROLES: frozenset[str] = frozenset(role.value for role in CursorInputRole) -_VALID_CONTRACT_POLICIES: frozenset[str] = frozenset(p.value for p in ContractPolicy) -_VALID_SNAPSHOT_STRATEGIES: frozenset[str] = frozenset(s.value for s in SnapshotStrategy) -_VALID_HISTORICAL_INPUTS: frozenset[str] = frozenset(h.value for h in HistoricalInput) -_VALID_INITIAL_VALID_FROM: frozenset[str] = frozenset(v.value for v in InitialValidFrom) -_VALID_SNAPSHOT_FULL_REFRESH_POLICIES: frozenset[str] = frozenset( - p.value for p in SnapshotFullRefreshPolicy -) -_VALID_SNAPSHOT_SCHEMA_CHANGE_POLICIES: frozenset[str] = frozenset( - p.value for p in SnapshotSchemaChangePolicy -) -_BUILTIN_MATERIALIZATION_TYPES: frozenset[str] = frozenset( - ( - MaterializationType.VIEW, - MaterializationType.TABLE, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, - ) -) -_INCREMENTAL_ONLY_KEYS: tuple[str, ...] = ( - "on_schema_change", - "replay_on_change", - "append_cursor_inclusive", - "merge_exclude_columns", - "full_refresh", -) -_SNAPSHOT_DISALLOWED_KEYS: tuple[str, ...] = ( - "incremental_strategy", - "incremental_mode", - "append_cursor_inclusive", - "batch_size", - "cursor", - "cursor_type", - "cursor_grain", - "cursor_inputs", - "cursor_filter_inputs", - "cursor_watermark_inputs", - "cursor_end", - "microbatch_strategy", - "cursor_watermark_mode", - "max_microbatches", - "microbatch_limit", - "lookback", -) -_CUSTOM_MATERIALIZATION_DISALLOWED_KEYS: tuple[str, ...] = ( - "on_schema_change", - "incremental_strategy", - "incremental_mode", - "append_cursor_inclusive", - "batch_size", - "cursor", - "cursor_type", - "cursor_grain", - "cursor_inputs", - "cursor_filter_inputs", - "cursor_watermark_inputs", - "cursor_end", - "microbatch_strategy", - "cursor_watermark_mode", - "max_microbatches", - "microbatch_limit", - "lookback", -) _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} @@ -139,909 +29,6 @@ ) -@dataclass(frozen=True) -class _IncrementalConfigValues: - strategy: str | None - cursor: str | None - cursor_type: str | None - cursor_start: object | None - cursor_end: object | None - cursor_start_max_ahead: object | None - cursor_start_max_action: object | None - cursor_future_max_distance: object | None - cursor_future_action: object | None - append_cursor_inclusive: object | None - unique_key: object | None - lookback: str | None - incremental_mode: str | None - batch_size: object | None - batch_concurrency: object | None - unaccounted_partition_policy: object | None - cursor_inputs: object | None - cursor_filter_inputs: object | None - cursor_watermark_inputs: object | None - microbatch_strategy: str | None - cursor_watermark_mode: str | None - max_microbatches: object | None - microbatch_limit: object | None - cursor_grain: str | None - merge_exclude_columns: object | None - full_refresh: object | None - - -@dataclass(frozen=True) -class _CursorInputValidation: - values: _IncrementalConfigValues - ref_count: int - - -def validate_incremental_config( - *, - config: CompileModelConfig, - model_name: str, - ref_count: int, - known_input_names: frozenset[str], - declared_columns: tuple[SchemaColumn, ...] | None = None, -) -> None: - """Validate incremental model config rules after layering.""" - - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized in {MaterializationType.TABLE, MaterializationType.VIEW}: - cursor_field: str - for cursor_field in ( - "cursor_inputs", - "cursor_filter_inputs", - "cursor_watermark_inputs", - ): - if config.values.get(cursor_field) is not None: - raise CompileInputError( - f"model '{model_name}': {cursor_field} requires cursor-based incremental " - "materialization" - ) - if materialized != MaterializationType.INCREMENTAL: - return - - values: _IncrementalConfigValues = _resolve_incremental_config_values(config=config) - _validate_incremental_core(config=config, model_name=model_name, values=values) - _validate_incremental_batching( - model_name=model_name, - ref_count=ref_count, - known_input_names=known_input_names, - values=values, - ) - _validate_incremental_write_strategy(model_name=model_name, values=values) - _validate_incremental_contract( - config=config, - model_name=model_name, - declared_columns=declared_columns, - values=values, - ) - if values.lookback is not None and values.cursor is None: - raise CompileInputError( - f"model '{model_name}': lookback is only valid with cursor-based incremental" - ) - - -def _validate_incremental_core( - *, config: CompileModelConfig, model_name: str, values: _IncrementalConfigValues -) -> None: - key: str - for key in ("on_schema_change", "replay_on_change"): - value: object | None = config.values.get(key) - problem: str | None = None if value is None else change_policy_problem(key=key, value=value) - if problem is not None: - raise CompileInputError( - f"model '{model_name}': {problem}", - help=change_policy_header_help(key=key), - ) - if values.strategy is None: - raise CompileInputError( - f"model '{model_name}': incremental materialization requires incremental_strategy" - ) - if values.strategy not in _VALID_STRATEGIES: - raise CompileInputError( - f"model '{model_name}': unknown incremental_strategy '{values.strategy}'; " - f"valid values: {', '.join(sorted(_VALID_STRATEGIES))}" - ) - _validate_incremental_cursor_rules( - model_name=model_name, - cursor=values.cursor, - cursor_type=values.cursor_type, - cursor_grain=values.cursor_grain, - cursor_start=values.cursor_start, - cursor_end=values.cursor_end, - append_cursor_inclusive=values.append_cursor_inclusive, - strategy=values.strategy, - ) - _validate_cursor_safety_overrides( - model_name=model_name, - cursor=values.cursor, - cursor_start_max_ahead=values.cursor_start_max_ahead, - cursor_start_max_action=values.cursor_start_max_action, - cursor_future_max_distance=values.cursor_future_max_distance, - cursor_future_action=values.cursor_future_action, - ) - - -def _validate_incremental_batching( - *, - model_name: str, - ref_count: int, - known_input_names: frozenset[str], - values: _IncrementalConfigValues, -) -> None: - if ( - values.incremental_mode is not None - and values.incremental_mode not in _VALID_INCREMENTAL_MODES - ): - raise CompileInputError( - f"model '{model_name}': unknown incremental_mode '{values.incremental_mode}'; " - f"valid values: {', '.join(sorted(_VALID_INCREMENTAL_MODES))}" - ) - _validate_microbatch_batch_size( - model_name=model_name, - batch_size=values.batch_size, - incremental_mode=values.incremental_mode, - cursor=values.cursor, - cursor_type=values.cursor_type, - cursor_grain=values.cursor_grain, - ) - _validate_microbatch_state_config( - model_name=model_name, - strategy=values.strategy, - incremental_mode=values.incremental_mode, - batch_concurrency=values.batch_concurrency, - unaccounted_partition_policy=values.unaccounted_partition_policy, - ) - _validate_cursor_input_config( - model_name=model_name, - known_input_names=known_input_names, - inputs=_CursorInputValidation(values=values, ref_count=ref_count), - ) - resolved_limit, resolved_limit_action = _validate_model_microbatch_limit( - model_name=model_name, - max_microbatches=values.max_microbatches, - microbatch_limit=values.microbatch_limit, - microbatch_strategy=values.microbatch_strategy, - ) - if resolved_limit is not None: - _validate_static_watermark_limit( - model_name=model_name, - max_microbatches=resolved_limit, - lookback=values.lookback, - batch_size=values.batch_size, - incremental_strategy=values.strategy, - action=resolved_limit_action, - cursor_grain=values.cursor_grain, - ) - - -def _validate_incremental_write_strategy( - *, model_name: str, values: _IncrementalConfigValues -) -> None: - has_unique_key: bool = values.unique_key not in (None, (), []) - if ( - values.strategy == IncrementalStrategy.DELETE_INSERT - and values.cursor is None - and not has_unique_key - ): - raise CompileInputError( - f"model '{model_name}': delete_insert without cursor requires unique_key" - ) - if values.strategy == IncrementalStrategy.MERGE and not has_unique_key: - raise CompileInputError(f"model '{model_name}': merge strategy requires unique_key") - if values.merge_exclude_columns is not None: - excluded_columns: tuple[str, ...] = _validated_string_sequence( - value=values.merge_exclude_columns, - config_key="merge_exclude_columns", - model_name=model_name, - ) - if values.strategy != IncrementalStrategy.MERGE: - raise CompileInputError( - f"model '{model_name}': merge_exclude_columns requires incremental_strategy=merge" - ) - if len({column.lower() for column in excluded_columns}) != len(excluded_columns): - raise CompileInputError( - f"model '{model_name}': merge_exclude_columns contains duplicate columns" - ) - unique_key_columns: frozenset[str] = frozenset( - column.lower() for column in _string_sequence(values.unique_key) - ) - overlap: tuple[str, ...] = tuple( - column for column in excluded_columns if column.lower() in unique_key_columns - ) - if overlap: - raise CompileInputError( - f"model '{model_name}': merge_exclude_columns cannot include unique_key " - f"column(s): {', '.join(overlap)}" - ) - if values.full_refresh is not None and not isinstance(values.full_refresh, bool): - raise CompileInputError(f"model '{model_name}': full_refresh must be a boolean") - - -def _validate_incremental_contract( - *, - config: CompileModelConfig, - model_name: str, - declared_columns: tuple[SchemaColumn, ...] | None, - values: _IncrementalConfigValues, -) -> None: - declared_column_names: frozenset[str] | None = _contract_declared_column_names( - config=config, - declared_columns=declared_columns, - ) - if declared_column_names is not None: - if values.cursor is not None: - _validate_declared_config_column( - column_name=values.cursor, - config_key="cursor", - declared_column_names=declared_column_names, - model_name=model_name, - ) - _validate_declared_config_columns( - column_names=_string_sequence(values.unique_key), - config_key="unique_key", - declared_column_names=declared_column_names, - model_name=model_name, - ) - _validate_declared_config_columns( - column_names=_string_sequence(values.merge_exclude_columns), - config_key="merge_exclude_columns", - declared_column_names=declared_column_names, - model_name=model_name, - ) - - -def _resolve_incremental_config_values(*, config: CompileModelConfig) -> _IncrementalConfigValues: - return _IncrementalConfigValues( - strategy=get_config_str(values=config.values, key="incremental_strategy"), - cursor=get_config_str(values=config.values, key="cursor"), - cursor_type=get_config_str(values=config.values, key="cursor_type"), - cursor_start=config.values.get("cursor_start"), - cursor_end=config.values.get("cursor_end"), - cursor_start_max_ahead=config.values.get("cursor_start_max_ahead"), - cursor_start_max_action=config.values.get("cursor_start_max_action"), - cursor_future_max_distance=config.values.get("cursor_future_max_distance"), - cursor_future_action=config.values.get("cursor_future_action"), - append_cursor_inclusive=config.values.get("append_cursor_inclusive"), - unique_key=config.values.get("unique_key"), - lookback=get_config_str(values=config.values, key="lookback"), - incremental_mode=get_config_str(values=config.values, key="incremental_mode"), - batch_size=get_config_str(values=config.values, key="batch_size"), - batch_concurrency=config.values.get("batch_concurrency"), - unaccounted_partition_policy=config.values.get("unaccounted_partition_policy"), - cursor_inputs=config.values.get("cursor_inputs"), - cursor_filter_inputs=config.values.get("cursor_filter_inputs"), - cursor_watermark_inputs=config.values.get("cursor_watermark_inputs"), - microbatch_strategy=get_config_str(values=config.values, key="microbatch_strategy"), - cursor_watermark_mode=get_config_str(values=config.values, key="cursor_watermark_mode"), - max_microbatches=config.values.get("max_microbatches"), - microbatch_limit=config.values.get("microbatch_limit"), - cursor_grain=get_config_str(values=config.values, key="cursor_grain"), - merge_exclude_columns=config.values.get("merge_exclude_columns"), - full_refresh=config.values.get("full_refresh"), - ) - - -def _validate_microbatch_batch_size( - *, - model_name: str, - batch_size: object | None, - incremental_mode: str | None, - cursor: str | None, - cursor_type: str | None, - cursor_grain: str | None, -) -> None: - if batch_size is not None and incremental_mode != IncrementalMode.MICROBATCH: - raise CompileInputError( - f"model '{model_name}': batch_size is only valid with incremental_mode=microbatch" - ) - if batch_size == EFFECTIVE_BATCH_SIZE_TOKEN and ( - cursor is None or cursor_type != CursorType.TIMESTAMP or cursor_grain is None - ): - raise CompileInputError( - f"model '{model_name}': batch_size=effective requires a timestamp cursor " - "with cursor_grain" - ) - - -def _validate_cursor_input_config( - *, - model_name: str, - known_input_names: frozenset[str], - inputs: _CursorInputValidation, -) -> None: - """Validate the strategy-specific canonical cursor input contract.""" - - for removed_field, value in ( - ("cursor_filter_inputs", inputs.values.cursor_filter_inputs), - ("cursor_watermark_inputs", inputs.values.cursor_watermark_inputs), - ): - if value is not None: - raise CompileInputError( - f"model '{model_name}': {removed_field} has been removed; declare " - "microbatch_strategy and use strategy-specific cursor_inputs" - ) - if ( - inputs.values.microbatch_strategy is not None - and inputs.values.incremental_mode != IncrementalMode.MICROBATCH - ): - raise CompileInputError( - f"model '{model_name}': microbatch_strategy is only valid with " - "incremental_mode=microbatch" - ) - if ( - inputs.values.incremental_mode == IncrementalMode.MICROBATCH - and inputs.values.microbatch_strategy is None - ): - raise CompileInputError( - f"model '{model_name}': incremental_mode=microbatch requires explicit " - "microbatch_strategy (rolling_window or watermark)" - ) - if ( - inputs.values.microbatch_strategy is not None - and inputs.values.microbatch_strategy not in _VALID_MICROBATCH_STRATEGIES - ): - raise CompileInputError( - f"model '{model_name}': unknown microbatch_strategy " - f"'{inputs.values.microbatch_strategy}'; " - f"valid values: {', '.join(sorted(_VALID_MICROBATCH_STRATEGIES))}" - ) - if ( - inputs.values.microbatch_strategy == MicrobatchStrategy.ROLLING_WINDOW - and inputs.values.cursor_type != CursorType.TIMESTAMP - ): - raise CompileInputError( - f"model '{model_name}': rolling_window requires cursor_type=timestamp" - ) - if inputs.values.cursor_inputs is not None and inputs.values.cursor is None: - raise CompileInputError(f"model '{model_name}': cursor_inputs requires cursor") - if ( - inputs.values.microbatch_strategy == MicrobatchStrategy.WATERMARK - and inputs.values.cursor_inputs is None - ) or ( - inputs.values.microbatch_strategy == MicrobatchStrategy.ROLLING_WINDOW - and inputs.ref_count > 0 - and inputs.values.cursor_inputs is None - ): - raise CompileInputError(f"model '{model_name}': microbatch strategy requires cursor_inputs") - if inputs.values.cursor_inputs is not None: - if inputs.values.microbatch_strategy == MicrobatchStrategy.WATERMARK: - _validate_watermark_cursor_inputs( - model_name=model_name, - value=inputs.values.cursor_inputs, - known_input_names=known_input_names, - ) - else: - _validate_cursor_input_map( - model_name=model_name, - config_field="cursor_inputs", - value=inputs.values.cursor_inputs, - ) - _validate_known_cursor_inputs( - model_name=model_name, - value=inputs.values.cursor_inputs, - known_input_names=known_input_names, - ) - if inputs.values.microbatch_strategy == MicrobatchStrategy.WATERMARK: - if inputs.values.cursor_watermark_mode not in _VALID_WATERMARK_MODES: - raise CompileInputError( - f"model '{model_name}': watermark strategy requires " - "cursor_watermark_mode all or any" - ) - elif inputs.values.cursor_watermark_mode is not None: - raise CompileInputError( - f"model '{model_name}': cursor_watermark_mode is only valid with " - "microbatch_strategy=watermark" - ) - if inputs.values.max_microbatches is not None: - if ( - isinstance(inputs.values.max_microbatches, bool) - or not isinstance(inputs.values.max_microbatches, int) - or inputs.values.max_microbatches < 1 - ): - raise CompileInputError( - f"model '{model_name}': max_microbatches must be a positive integer" - ) - if inputs.values.microbatch_strategy != MicrobatchStrategy.WATERMARK: - raise CompileInputError( - f"model '{model_name}': max_microbatches is only valid with " - "microbatch_strategy=watermark" - ) - if ( - inputs.values.cursor is not None - and inputs.ref_count > 1 - and inputs.values.cursor_inputs is None - ): - raise CompileInputError( - f"model '{model_name}': models with cursor and multiple inputs require explicit " - "cursor_inputs" - ) - - -def _validate_model_microbatch_limit( - *, - model_name: str, - max_microbatches: object | None, - microbatch_limit: object | None, - microbatch_strategy: str | None, -) -> tuple[int | None, MicrobatchLimitAction | None]: - if max_microbatches is not None and microbatch_limit is not None: - raise CompileInputError( - f"model '{model_name}': use either max_microbatches or microbatch_limit, not both" - ) - if microbatch_limit is None: - return ( - ( - max_microbatches - if isinstance(max_microbatches, int) and not isinstance(max_microbatches, bool) - else None - ), - None, - ) - if not isinstance(microbatch_limit, dict) or set(microbatch_limit) != { - MICROBATCH_LIMIT_MAX_BATCHES_KEY, - MICROBATCH_LIMIT_ACTION_KEY, - }: - raise CompileInputError( - f"model '{model_name}': microbatch_limit requires exactly max_batches and action" - ) - limit_mapping: dict[str, object] = cast(dict[str, object], microbatch_limit) - max_batches: object | None = limit_mapping.get(MICROBATCH_LIMIT_MAX_BATCHES_KEY) - action: object | None = limit_mapping.get(MICROBATCH_LIMIT_ACTION_KEY) - if isinstance(max_batches, bool) or not isinstance(max_batches, int) or max_batches < 1: - raise CompileInputError( - f"model '{model_name}': microbatch_limit max_batches must be a positive integer" - ) - valid_actions: frozenset[str] = frozenset(item.value for item in MicrobatchLimitAction) - if not isinstance(action, str) or action not in valid_actions: - raise CompileInputError( - f"model '{model_name}': microbatch_limit action must be one of: " - + ", ".join(sorted(valid_actions)) - ) - if microbatch_strategy != MicrobatchStrategy.WATERMARK: - raise CompileInputError( - f"model '{model_name}': microbatch_limit is only valid with " - "microbatch_strategy=watermark" - ) - return max_batches, MicrobatchLimitAction(action) - - -def _validate_known_cursor_inputs( - *, model_name: str, value: object, known_input_names: frozenset[str] -) -> None: - if not isinstance(value, dict): - return - for input_name in value: - if str(input_name) not in known_input_names: - expected_names: str = ", ".join(sorted(known_input_names)) - raise CompileInputError( - f"model '{model_name}': cursor_inputs references unknown input " - f"'{input_name}'; expected one of: {expected_names}" - ) - - -def _validate_watermark_cursor_inputs( - *, model_name: str, value: object, known_input_names: frozenset[str] -) -> None: - if not isinstance(value, dict) or not value: - raise CompileInputError( - f"model '{model_name}': watermark cursor_inputs must be a non-empty relation map" - ) - has_watermark: bool = False - for relation, block in value.items(): - if not isinstance(relation, str) or not relation.strip() or not isinstance(block, dict): - raise CompileInputError( - f"model '{model_name}': watermark cursor_inputs relation '{relation}' must use " - "(column ..., roles [...])" - ) - if set(block) != WATERMARK_CURSOR_INPUT_BLOCK_KEYS: - raise CompileInputError( - f"model '{model_name}': cursor_inputs relation '{relation}' requires exactly " - "column and roles" - ) - typed_block: dict[object, object] = cast(dict[object, object], block) - column: object | None = typed_block.get("column") - roles: object | None = typed_block.get("roles") - if not isinstance(column, str) or not column.strip(): - raise CompileInputError( - f"model '{model_name}': cursor_inputs column for relation '{relation}' must be " - "a non-empty string" - ) - if ( - not isinstance(roles, list) - or not roles - or any(role not in _VALID_CURSOR_INPUT_ROLES for role in roles) - ): - raise CompileInputError( - f"model '{model_name}': cursor_inputs roles for relation '{relation}' must be a " - "non-empty list containing only filter and/or watermark" - ) - if len(set(roles)) != len(roles): - raise CompileInputError( - f"model '{model_name}': cursor_inputs relation '{relation}' contains " - "duplicate roles" - ) - has_watermark = has_watermark or CursorInputRole.WATERMARK in roles - if CursorInputRole.FILTER in roles and relation not in known_input_names: - expected_names: str = ", ".join(sorted(known_input_names)) - raise CompileInputError( - f"model '{model_name}': filter role references input '{relation}' that is not " - f"directly filterable; expected one of: {expected_names}" - ) - if not has_watermark: - raise CompileInputError( - f"model '{model_name}': watermark strategy requires at least one watermark role" - ) - - -def _validate_static_watermark_limit( - *, - model_name: str, - max_microbatches: int, - lookback: str | None, - batch_size: object | None, - incremental_strategy: str | None, - action: MicrobatchLimitAction | None, - cursor_grain: str | None, -) -> None: - effective_batch_size: object = batch_size - if batch_size == EFFECTIVE_BATCH_SIZE_TOKEN and cursor_grain is not None: - effective_batch_size = GRAIN_BATCH_SIZE.get(CursorGrain(cursor_grain), batch_size) - effective_lookback: str | None = lookback - if ( - effective_lookback is None - and action == MicrobatchLimitAction.CAP_FROM_START - and incremental_strategy in {IncrementalStrategy.DELETE_INSERT, IncrementalStrategy.MERGE} - and isinstance(effective_batch_size, str) - ): - effective_lookback = effective_batch_size - if effective_lookback is None or not isinstance(effective_batch_size, str): - return - lookback_duration: Duration | None = Duration.parse(effective_lookback) - batch_duration: Duration | None = Duration.parse(effective_batch_size) - if lookback_duration is None or batch_duration is None: - return - if not lookback_duration.has_calendar_component and not batch_duration.has_calendar_component: - if batch_duration.fixed_seconds == 0: - return - lookback_batches: int = ( - lookback_duration.fixed_seconds + batch_duration.fixed_seconds - 1 - ) // batch_duration.fixed_seconds - elif lookback_duration.fixed_seconds == 0 and batch_duration.fixed_seconds == 0: - if batch_duration.total_months == 0: - return - lookback_batches = ( - lookback_duration.total_months + batch_duration.total_months - 1 - ) // batch_duration.total_months - else: - if action == MicrobatchLimitAction.CAP_FROM_START: - raise CompileInputError( - f"model '{model_name}': cap_from_start cannot prove forward progress when " - "lookback and batch_size mix calendar and fixed duration components; use " - "compatible fixed or calendar durations" - ) - return - required: int = ( - lookback_batches + 1 + (1 if action == MicrobatchLimitAction.CAP_FROM_START else 0) - ) - if max_microbatches < required: - raise CompileInputError( - f"model '{model_name}': max_microbatches {max_microbatches} is below the " - f"ordinary lookback requirement of {required} batches" - ) - - -def _validate_cursor_input_map(*, model_name: str, config_field: str, value: object) -> None: - """Validate one cursor input field as a relation-to-column map.""" - - if not isinstance(value, dict): - raise CompileInputError(f"model '{model_name}': {config_field} must be a relation map") - if not value: - raise CompileInputError(f"model '{model_name}': {config_field} must not be empty") - relation_name: object - cursor_column: object - for relation_name, cursor_column in value.items(): - if not isinstance(relation_name, str) or not relation_name.strip(): - raise CompileInputError( - f"model '{model_name}': {config_field} relation names must be non-empty strings" - ) - if not isinstance(cursor_column, str) or not cursor_column.strip(): - raise CompileInputError( - f"model '{model_name}': {config_field} column for relation " - f"'{relation_name}' must be a non-empty string" - ) - - -def _validate_microbatch_state_config( - *, - model_name: str, - strategy: str | None, - incremental_mode: str | None, - batch_concurrency: object | None, - unaccounted_partition_policy: object | None, -) -> None: - if batch_concurrency is not None: - if isinstance(batch_concurrency, bool) or not isinstance(batch_concurrency, int): - raise CompileInputError( - f"model '{model_name}': batch_concurrency must be a positive integer" - ) - if batch_concurrency <= 0: - raise CompileInputError( - f"model '{model_name}': batch_concurrency must be a positive integer" - ) - if batch_concurrency > 1 and incremental_mode != IncrementalMode.MICROBATCH: - raise CompileInputError( - f"model '{model_name}': batch_concurrency > 1 requires incremental_mode=microbatch" - ) - if batch_concurrency > 1 and strategy != IncrementalStrategy.DELETE_INSERT: - raise CompileInputError( - f"model '{model_name}': batch_concurrency > 1 requires " - "incremental_strategy=delete_insert" - ) - if unaccounted_partition_policy is None: - return - valid_policies: frozenset[str] = frozenset({"synthesize", "recover_empty", "recover_all"}) - if ( - not isinstance(unaccounted_partition_policy, str) - or unaccounted_partition_policy not in valid_policies - ): - raise CompileInputError( - f"model '{model_name}': unaccounted_partition_policy must be one of: " - + ", ".join(sorted(valid_policies)) - ) - if incremental_mode != IncrementalMode.MICROBATCH: - raise CompileInputError( - f"model '{model_name}': unaccounted_partition_policy requires " - "incremental_mode=microbatch" - ) - - -def validate_microbatch_project_capability( - *, config: CompileModelConfig, settings: SettingsConfig, model_name: str -) -> None: - """Require the project capability for a model concurrency ceiling above one.""" - - batch_concurrency: object | None = config.values.get("batch_concurrency") - if ( - isinstance(batch_concurrency, int) - and not isinstance(batch_concurrency, bool) - and batch_concurrency > 1 - and not settings.microbatch_concurrency - ): - note, help_text = microbatch_guidance() - raise CompileInputError( - f"model '{model_name}': batch_concurrency > 1 requires concurrent microbatches; {note}", - help=help_text, - ) - - -def _validate_incremental_cursor_rules( - *, - model_name: str, - cursor: str | None, - cursor_type: str | None, - cursor_grain: str | None, - cursor_start: object | None, - cursor_end: object | None, - append_cursor_inclusive: object | None, - strategy: str | None, -) -> None: - if cursor is not None and cursor_type is None: - raise CompileInputError( - f"model '{model_name}': cursor requires cursor_type " - f"(valid values: {', '.join(sorted(_VALID_CURSOR_TYPES))})" - ) - if cursor_type is not None and cursor_type not in _VALID_CURSOR_TYPES: - raise CompileInputError( - f"model '{model_name}': unknown cursor_type '{cursor_type}'; " - f"valid values: {', '.join(sorted(_VALID_CURSOR_TYPES))}" - ) - if cursor_grain is not None and cursor_type != CursorType.TIMESTAMP: - raise CompileInputError( - f"model '{model_name}': cursor_grain is only valid with cursor_type=timestamp" - ) - if cursor_grain is not None and cursor_grain not in _VALID_CURSOR_GRAINS: - raise CompileInputError( - f"model '{model_name}': unknown cursor_grain '{cursor_grain}'; " - f"valid values: {', '.join(sorted(_VALID_CURSOR_GRAINS))}" - ) - if cursor is not None and cursor_type == CursorType.TIMESTAMP and cursor_grain is None: - raise CompileInputError( - f"model '{model_name}': cursor_type=timestamp requires cursor_grain " - f"(valid values: {', '.join(sorted(_VALID_CURSOR_GRAINS))})" - ) - if cursor_start is not None and cursor is None: - raise CompileInputError(f"model '{model_name}': cursor_start requires cursor") - if cursor_start is not None and cursor_type is None: - raise CompileInputError(f"model '{model_name}': cursor_start requires cursor_type") - if cursor_start is not None and cursor_type == CursorType.TIMESTAMP: - _validate_timestamp_cursor_start(cursor_start=cursor_start, model_name=model_name) - if cursor_start is not None and cursor_type == CursorType.INTEGER: - _validate_integer_cursor_start(cursor_start=cursor_start, model_name=model_name) - if cursor_end is not None and cursor is None: - raise CompileInputError(f"model '{model_name}': cursor_end requires cursor") - if cursor_end is not None and cursor_type == CursorType.TIMESTAMP: - _validate_timestamp_cursor_start(cursor_start=cursor_end, model_name=model_name) - if cursor_end is not None and cursor_type == CursorType.INTEGER: - _validate_integer_cursor_start(cursor_start=cursor_end, model_name=model_name) - if ( - cursor_start is not None - and cursor_end is not None - and _cursor_contract_key( - value=cursor_start, cursor_type=cursor_type, key="cursor_start", model_name=model_name - ) - >= _cursor_contract_key( - value=cursor_end, cursor_type=cursor_type, key="cursor_end", model_name=model_name - ) - ): - raise CompileInputError( - f"model '{model_name}': cursor_start must be before exclusive cursor_end" - ) - if append_cursor_inclusive is not None and not isinstance(append_cursor_inclusive, bool): - raise CompileInputError(f"model '{model_name}': append_cursor_inclusive must be a boolean") - if append_cursor_inclusive is not None and strategy != IncrementalStrategy.APPEND: - raise CompileInputError( - f"model '{model_name}': append_cursor_inclusive is only valid with append strategy" - ) - if append_cursor_inclusive is not None and cursor is None: - raise CompileInputError(f"model '{model_name}': append_cursor_inclusive requires cursor") - - -def _cursor_contract_key( - *, value: object, cursor_type: str | None, key: str, model_name: str -) -> Decimal: - if cursor_type == CursorType.INTEGER: - return Decimal(str(value)) - parsed: datetime = datetime.fromisoformat(str(value)) - if parsed.tzinfo is None: - return Decimal(str(parsed.replace(tzinfo=UTC).timestamp())) - try: - utc_parsed: datetime = parsed.astimezone(UTC) - except OverflowError: - raise CompileInputError( - f"model '{model_name}': {key} value '{value}' falls outside years 1-9999 once " - "converted to UTC", - help=model_header_help( - purpose=f"keep {key} within years 1-9999 in UTC", - entry=f"{key} '{parsed.replace(tzinfo=UTC).isoformat()}'", - ), - ) from None - return Decimal(str(utc_parsed.timestamp())) - - -def _validate_cursor_safety_overrides( - *, - model_name: str, - cursor: str | None, - cursor_start_max_ahead: object | None, - cursor_start_max_action: object | None, - cursor_future_max_distance: object | None, - cursor_future_action: object | None, -) -> None: - values: tuple[object | None, ...] = ( - cursor_start_max_ahead, - cursor_start_max_action, - cursor_future_max_distance, - cursor_future_action, - ) - if cursor is None and any(value is not None for value in values): - raise CompileInputError(f"model '{model_name}': cursor safety overrides require cursor") - if cursor_start_max_ahead is not None and ( - not isinstance(cursor_start_max_ahead, str) - or ( - cursor_start_max_ahead not in {CURSOR_POLICY_DISABLED, ZERO_DAY_CURSOR_DURATION} - and Duration.parse(cursor_start_max_ahead) is None - ) - ): - raise CompileInputError( - f"model '{model_name}': cursor_start_max_ahead must be a duration or 'disabled'" - ) - if cursor_future_max_distance is not None and ( - not isinstance(cursor_future_max_distance, str) - or ( - cursor_future_max_distance not in {CURSOR_POLICY_DISABLED, ZERO_DAY_CURSOR_DURATION} - and Duration.parse(cursor_future_max_distance) is None - ) - ): - raise CompileInputError( - f"model '{model_name}': cursor_future_max_distance must be a duration or 'disabled'" - ) - for key, value in ( - ("cursor_start_max_action", cursor_start_max_action), - ("cursor_future_action", cursor_future_action), - ): - if value is not None and value not in set(FutureCursorAction): - raise CompileInputError(f"model '{model_name}': {key} must be one of: cap, error") - - -def validate_contract_config( - *, - config: CompileModelConfig, - model_name: str, -) -> None: - """Validate model contract config values after layering.""" - - raw_contract: object | None = config.values.get("contract") - if raw_contract is None: - return - if not isinstance(raw_contract, str): - raise CompileInputError(f"model '{model_name}': contract must be a string") - contract: str = raw_contract - if contract not in _VALID_CONTRACT_POLICIES: - raise CompileInputError( - f"model '{model_name}': unknown contract '{contract}'; valid values: " - f"{', '.join(sorted(_VALID_CONTRACT_POLICIES))}" - ) - - -def validate_non_incremental_config( - *, - config: CompileModelConfig, - model_name: str, -) -> None: - """Reject incremental-only config keys on non-incremental models.""" - - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized == MaterializationType.INCREMENTAL: - return - - key: str - for key in _INCREMENTAL_ONLY_KEYS: - if config.values.get(key) is not None: - raise CompileInputError( - f"model '{model_name}': {key} is only valid for incremental models" - ) - - -def validate_model_migration_config(*, config: CompileModelConfig, model_name: str) -> None: - """Validate migrate_from, migrate_force, and old_name_view on model headers.""" - - _validate_old_name_view_config(config=config, model_name=model_name) - migrate_from: object | None = config.values.get(MIGRATE_FROM_CONFIG_KEY) - migrate_force: object | None = config.values.get(MIGRATE_FORCE_CONFIG_KEY) - if migrate_from is None and migrate_force is None: - return - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized not in _MIGRATABLE_MATERIALIZATIONS: - raise CompileInputError( - f"model '{model_name}': migrate_from is only valid for table, view, incremental " - f"and snapshot models" - ) - if migrate_force is not None and materialized not in _HISTORY_MATERIALIZATIONS: - raise CompileInputError( - f"model '{model_name}': migrate_force is only valid for incremental and snapshot " - f"models; nothing is replaced when a '{materialized}' model migrates, because " - "tables and views are rebuilt under their new name", - help="remove migrate_force from the model header", - ) - if migrate_from is None: - raise CompileInputError(f"model '{model_name}': migrate_force requires migrate_from") - if not isinstance(migrate_from, str) or not migrate_from.strip(): - raise CompileInputError( - f"model '{model_name}': migrate_from must be a model name or a qualified relation" - ) - if migrate_from.strip().lower() == model_name.lower(): - raise CompileInputError(f"model '{model_name}': migrate_from cannot name the model itself") - if migrate_force is not None and not isinstance(migrate_force, bool): - raise CompileInputError(f"model '{model_name}': migrate_force must be true or false") - - -def _validate_old_name_view_config(*, config: CompileModelConfig, model_name: str) -> None: - if OLD_NAME_VIEW_CONFIG_KEY not in config.values: - return - value: object = config.values[OLD_NAME_VIEW_CONFIG_KEY] - if value is False: - return - if isinstance(value, str) and Duration.parse(value.strip()) is not None: - return - raise CompileInputError( - f"model '{model_name}': old_name_view must be a positive duration such as 7d, or " - f"false; got {value!r}", - help="write for example old_name_view 7d, or old_name_view false", - ) - - def validate_column_migration_config( *, config_values: dict[str, object], model_name: str, columns: tuple[SchemaColumn, ...] ) -> None: @@ -1101,407 +88,3 @@ def validate_column_migration_config( "name no other renamed column uses, or rebuild the model with --full-refresh" ), ) - - -def validate_snapshot_config( - *, - config: CompileModelConfig, - model_name: str, - declared_columns: tuple[SchemaColumn, ...] | None = None, -) -> None: - """Validate snapshot model config combinations after layering.""" - - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized != MaterializationType.SNAPSHOT: - return - - strategy: str | None = get_config_str(values=config.values, key="snapshot_strategy") - updated_at: str | None = get_config_str(values=config.values, key="updated_at") - observed_at: str | None = get_config_str(values=config.values, key="observed_at") - historical_input: str | None = get_config_str(values=config.values, key="historical_input") - initial_valid_from: str | None = get_config_str(values=config.values, key="initial_valid_from") - snapshot_full_refresh: str | None = get_config_str( - values=config.values, key="snapshot_full_refresh" - ) - snapshot_schema_change: str | None = get_config_str( - values=config.values, key="snapshot_schema_change" - ) - unique_key: object | None = config.values.get("unique_key") - check_columns: object | None = config.values.get("check_columns") - invalidate_hard_deletes: object | None = config.values.get("invalidate_hard_deletes") - valid_from_column: str | None = get_config_str(values=config.values, key="valid_from_column") - valid_to_column: str | None = get_config_str(values=config.values, key="valid_to_column") - - key: str - for key in _SNAPSHOT_DISALLOWED_KEYS: - if config.values.get(key) is not None: - raise CompileInputError( - f"model '{model_name}': {key} is not allowed on snapshot models" - ) - - if not _has_config_value(unique_key): - raise CompileInputError( - f"model '{model_name}': snapshot materialization requires unique_key" - ) - if strategy is None: - raise CompileInputError( - f"model '{model_name}': snapshot materialization requires snapshot_strategy" - ) - if strategy not in _VALID_SNAPSHOT_STRATEGIES: - raise CompileInputError( - f"model '{model_name}': unknown snapshot_strategy '{strategy}'; " - f"valid values: {', '.join(sorted(_VALID_SNAPSHOT_STRATEGIES))}" - ) - - if strategy == SnapshotStrategy.TIMESTAMP and updated_at is None: - raise CompileInputError( - f"model '{model_name}': snapshot_strategy=timestamp requires updated_at" - ) - if strategy == SnapshotStrategy.CHECK and not _has_config_value(check_columns): - raise CompileInputError( - f"model '{model_name}': snapshot_strategy=check requires check_columns" - ) - if ( - strategy == SnapshotStrategy.CHECK - and isinstance(check_columns, list) - and SQL_WILDCARD_TOKEN in check_columns - and len(check_columns) != 1 - ): - raise CompileInputError( - f"model '{model_name}': check_columns [*] cannot be combined with explicit columns" - ) - - if historical_input is not None and observed_at is None: - raise CompileInputError(f"model '{model_name}': historical_input requires observed_at") - if historical_input is not None and historical_input not in _VALID_HISTORICAL_INPUTS: - raise CompileInputError( - f"model '{model_name}': unknown historical_input '{historical_input}'; " - f"valid values: {', '.join(sorted(_VALID_HISTORICAL_INPUTS))}" - ) - if ( - observed_at is not None - and strategy == SnapshotStrategy.TIMESTAMP - and historical_input is None - ): - raise CompileInputError( - f"model '{model_name}': timestamp snapshots with observed_at require " - "historical_input snapshot or changes" - ) - if strategy == SnapshotStrategy.CHECK and historical_input == HistoricalInput.CHANGES: - raise CompileInputError( - f"model '{model_name}': historical_input=changes is not valid with " - "snapshot_strategy=check" - ) - if invalidate_hard_deletes is not None and not isinstance(invalidate_hard_deletes, bool): - raise CompileInputError(f"model '{model_name}': invalidate_hard_deletes must be a boolean") - if invalidate_hard_deletes is True and historical_input == HistoricalInput.CHANGES: - raise CompileInputError( - f"model '{model_name}': invalidate_hard_deletes is not valid with " - "historical_input=changes" - ) - - if initial_valid_from is not None and initial_valid_from not in _VALID_INITIAL_VALID_FROM: - raise CompileInputError( - f"model '{model_name}': unknown initial_valid_from '{initial_valid_from}'; " - f"valid values: {', '.join(sorted(_VALID_INITIAL_VALID_FROM))}" - ) - if initial_valid_from == InitialValidFrom.UPDATED_AT and updated_at is None: - raise CompileInputError( - f"model '{model_name}': initial_valid_from=updated_at requires updated_at" - ) - if initial_valid_from == InitialValidFrom.OBSERVED_AT and observed_at is None: - raise CompileInputError( - f"model '{model_name}': initial_valid_from=observed_at requires observed_at" - ) - - if snapshot_full_refresh is not None: - if snapshot_full_refresh not in _VALID_SNAPSHOT_FULL_REFRESH_POLICIES: - raise CompileInputError( - f"model '{model_name}': unknown snapshot_full_refresh " - f"'{snapshot_full_refresh}'; valid values: " - f"{', '.join(sorted(_VALID_SNAPSHOT_FULL_REFRESH_POLICIES))}" - ) - - if snapshot_schema_change is not None: - if snapshot_schema_change not in _VALID_SNAPSHOT_SCHEMA_CHANGE_POLICIES: - raise CompileInputError( - f"model '{model_name}': unknown snapshot_schema_change " - f"'{snapshot_schema_change}'; valid values: " - f"{', '.join(sorted(_VALID_SNAPSHOT_SCHEMA_CHANGE_POLICIES))}" - ) - if ( - config.values.get("contract") == ContractPolicy.ENFORCED.value - and snapshot_schema_change == SnapshotSchemaChangePolicy.APPEND_NEW_COLUMNS - ): - raise CompileInputError( - f"model '{model_name}': snapshot_schema_change=append_new_columns " - "is not valid with contract enforced; add new columns to the contract " - "or set contract none", - code="K012", - ) - - if valid_from_column is not None and valid_to_column is not None: - if valid_from_column.lower() == valid_to_column.lower(): - raise CompileInputError( - f"model '{model_name}': valid_from_column and valid_to_column must differ" - ) - - declared_column_names: frozenset[str] | None = _contract_declared_column_names( - config=config, - declared_columns=declared_columns, - ) - if declared_column_names is not None: - _validate_declared_config_columns( - column_names=_string_sequence(unique_key), - config_key="unique_key", - declared_column_names=declared_column_names, - model_name=model_name, - ) - if updated_at is not None: - _validate_declared_config_column( - column_name=updated_at, - config_key="updated_at", - declared_column_names=declared_column_names, - model_name=model_name, - ) - if observed_at is not None: - _validate_declared_config_column( - column_name=observed_at, - config_key="observed_at", - declared_column_names=declared_column_names, - model_name=model_name, - ) - if check_columns != [SQL_WILDCARD_TOKEN]: - _validate_declared_config_columns( - column_names=_string_sequence(check_columns), - config_key="check_columns", - declared_column_names=declared_column_names, - model_name=model_name, - ) - - -def validate_custom_materialization_config( - *, - config: CompileModelConfig, - model_name: str, - custom_materialization_names: frozenset[str], -) -> None: - """Validate config for custom materialization models.""" - - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized is None or materialized in _BUILTIN_MATERIALIZATION_TYPES: - return - - if materialized not in custom_materialization_names: - raise CompileInputError( - f"model '{model_name}': unknown materialization '{materialized}'; " - f"not a built-in type and no custom materialization with that name was discovered" - ) - - key: str - for key in _CUSTOM_MATERIALIZATION_DISALLOWED_KEYS: - if config.values.get(key) is not None: - raise CompileInputError( - f"model '{model_name}': {key} is not allowed on custom materializations" - ) - - -def validate_time_travel_retention(*, config: CompileModelConfig, model_name: str) -> None: - """Reject managed retention on materializations without physical managed tables.""" - - retention: ResolvedTimeTravelRetention = config.time_travel_retention - if retention.unmanaged: - return - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized == MaterializationType.VIEW: - raise CompileInputError( - f"model '{model_name}': managed time_travel_retention is not valid for views; " - "set time_travel_retention disabled" - ) - if not MaterializationType.is_table_backed(materialized=materialized): - raise CompileInputError( - f"model '{model_name}': managed time_travel_retention is not supported for " - f"materialization '{materialized}'" - ) - - -def validate_table_type(*, config: CompileModelConfig, model_name: str) -> None: - """Reject declared table type on materializations without managed tables.""" - - if not config.table_type.declared: - return - materialized: str | None = get_config_str(values=config.values, key="materialized") - if materialized == MaterializationType.VIEW: - raise CompileInputError(f"model '{model_name}': table_type is not valid for views") - if not MaterializationType.is_table_backed(materialized=materialized): - raise CompileInputError( - f"model '{model_name}': table_type is not supported for materialization " - f"'{materialized}'" - ) - - -def validate_storage_policies(*, config: CompileModelConfig, model_name: str) -> None: - """Validate retention and table-type materialization applicability.""" - - validate_time_travel_retention(config=config, model_name=model_name) - validate_table_type(config=config, model_name=model_name) - - -def validate_placeholder_config( - *, - config: CompileModelConfig, - model_name: str, - query_sql: str, - custom_materialization_names: frozenset[str], -) -> None: - """Validate @@@placeholder usage and placeholders config consistency.""" - - import re - - materialized: str | None = get_config_str(values=config.values, key="materialized") - is_custom: bool = materialized is not None and materialized in custom_materialization_names - placeholders_config: object | None = config.values.get("placeholders") - sql_placeholders: frozenset[str] = frozenset(re.findall(r"@@@(\w+)", query_sql)) - - if not is_custom and sql_placeholders: - raise CompileInputError( - f"model '{model_name}': @@@placeholders are only allowed on custom materializations" - ) - if not is_custom and placeholders_config is not None: - raise CompileInputError( - f"model '{model_name}': placeholders config is only allowed on custom materializations" - ) - - if is_custom and not sql_placeholders and placeholders_config is None: - return - - declared_names: frozenset[str] = frozenset() - if isinstance(placeholders_config, dict): - declared_names = frozenset(str(k) for k in placeholders_config) - - missing_defaults: frozenset[str] = sql_placeholders - declared_names - if missing_defaults: - sorted_missing: str = ", ".join(sorted(missing_defaults)) - raise CompileInputError( - f"model '{model_name}': @@@placeholders without default values " - f"in placeholders config: {sorted_missing}" - ) - - unused_defaults: frozenset[str] = declared_names - sql_placeholders - if unused_defaults: - sorted_unused: str = ", ".join(sorted(unused_defaults)) - raise CompileInputError( - f"model '{model_name}': placeholders config entries not used in SQL: {sorted_unused}" - ) - - -def _has_config_value(value: object | None) -> bool: - return value is not None and value != () and value != [] - - -def _contract_declared_column_names( - *, - config: CompileModelConfig, - declared_columns: tuple[SchemaColumn, ...] | None = None, -) -> frozenset[str] | None: - if config.values.get("contract") != ContractPolicy.ENFORCED: - return None - resolved_names: frozenset[str] = frozenset(column.name for column in declared_columns or ()) - raw_columns: object | None = config.values.get("columns") - if not isinstance(raw_columns, dict): - return resolved_names - return resolved_names | frozenset(name for name in raw_columns if isinstance(name, str)) - - -def _string_sequence(value: object | None) -> tuple[str, ...]: - if isinstance(value, str): - return (value,) - if isinstance(value, list | tuple): - return tuple(item for item in value if isinstance(item, str)) - return () - - -def _validated_string_sequence( - *, value: object, config_key: str, model_name: str -) -> tuple[str, ...]: - if not isinstance(value, list | tuple) or not all( - isinstance(item, str) and item for item in value - ): - raise CompileInputError( - f"model '{model_name}': {config_key} must be a list of non-empty strings" - ) - return tuple(item for item in value if isinstance(item, str)) - - -def _validate_declared_config_columns( - *, - column_names: tuple[str, ...], - config_key: str, - declared_column_names: frozenset[str], - model_name: str, -) -> None: - column_name: str - for column_name in column_names: - _validate_declared_config_column( - column_name=column_name, - config_key=config_key, - declared_column_names=declared_column_names, - model_name=model_name, - ) - - -def _validate_declared_config_column( - *, - column_name: str, - config_key: str, - declared_column_names: frozenset[str], - model_name: str, -) -> None: - if column_name in declared_column_names: - return - raise CompileInputError( - f"model '{model_name}': {config_key} references column '{column_name}' " - "not declared in enforced contract" - ) - - -def _validate_timestamp_cursor_start(*, cursor_start: object, model_name: str) -> None: - if isinstance(cursor_start, datetime | date): - return - if isinstance(cursor_start, str): - try: - datetime.fromisoformat(cursor_start) - except ValueError as error: - raise CompileInputError( - f"model '{model_name}': cursor_start value '{cursor_start}' " - f"is not a valid ISO timestamp: {error}" - ) from None - return - raise CompileInputError( - f"model '{model_name}': cursor_start for cursor_type=timestamp must be " - "a string or date-like value" - ) - - -def _validate_integer_cursor_start(*, cursor_start: object, model_name: str) -> None: - if isinstance(cursor_start, bool): - raise CompileInputError( - f"model '{model_name}': cursor_start for cursor_type=integer must be an integer" - ) - if isinstance(cursor_start, int): - return - if isinstance(cursor_start, str): - try: - decimal_value: Decimal = Decimal(cursor_start) - except InvalidOperation: - raise CompileInputError( - f"model '{model_name}': cursor_start value '{cursor_start}' is not a valid integer" - ) from None - if decimal_value != int(decimal_value): - raise CompileInputError( - f"model '{model_name}': cursor_start value '{cursor_start}' is not a whole number" - ) - return - raise CompileInputError( - f"model '{model_name}': cursor_start for cursor_type=integer must be a string or integer" - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/table_type.py b/src/sqlbuild/compiler/compile/_helpers/config/table_type.py index 523f26732e..846de7f3c8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/table_type.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/table_type.py @@ -2,52 +2,15 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ResolvedTableType from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.spec.contracts.models import ( MaterializationDefaultsConfig, - ResolvedTimeTravelRetention, TargetConfig, ) from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TableTypeValue -_STORAGE_POLICY_KEYS: frozenset[str] = frozenset({"time_travel_retention", "table_type"}) - - -def resolve_storage_policies( - *, - resolved_values: dict[str, object], - model_header_values: dict[str, object], - materialization_defaults: MaterializationDefaultsConfig | None, - target_config: TargetConfig | None, - model_name: str, -) -> tuple[dict[str, object], ResolvedTimeTravelRetention, ResolvedTableType]: - """Resolve storage policies and remove authored values from ordinary model config.""" - - defaults: MaterializationDefaultsConfig = ( - materialization_defaults or MaterializationDefaultsConfig() - ) - retention: ResolvedTimeTravelRetention = resolve_time_travel_retention( - materialized=resolved_values.get("materialized"), - model_value=model_header_values.get("time_travel_retention"), - materialization_defaults=defaults, - target_config=target_config, - model_name=model_name, - ) - table_type: ResolvedTableType = resolve_table_type( - materialized=resolved_values.get("materialized"), - model_value=model_header_values.get("table_type"), - materialization_defaults=defaults, - target_config=target_config, - model_name=model_name, - ) - cleaned_values: dict[str, object] = { - key: value for key, value in resolved_values.items() if key not in _STORAGE_POLICY_KEYS - } - return cleaned_values, retention, table_type - def resolve_table_type( *, diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 3780d44ad3..1314964fb8 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -60,15 +60,11 @@ MODEL_DIRECTORY_NAME: str = "models" NOT_NULL_AUDIT_NAME: str = "not_null" MODEL_AUDIT_OVERRIDE_KEYS: frozenset[str] = frozenset({"by_type", "by_column"}) -MODEL_FULL_REFRESH_CONFIG_KEY: str = "full_refresh" CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" COLUMN_MIGRATE_FROM_KEY: str = "migrate_from" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" -MICROBATCH_LIMIT_MAX_BATCHES_KEY: str = "max_batches" -MICROBATCH_LIMIT_ACTION_KEY: str = "action" -WATERMARK_CURSOR_INPUT_BLOCK_KEYS: frozenset[str] = frozenset({"column", "roles"}) MODEL_HEADER_METADATA_KEYS: frozenset[str] = frozenset( { "description", @@ -84,13 +80,11 @@ SQL_WILDCARD_TOKEN: str = "*" SQL_OPEN_PAREN_TOKEN: str = "(" -SQL_CLOSE_PAREN_TOKEN: str = ")" SQL_ARGUMENT_SEPARATOR_TOKEN: str = "," SQL_STATEMENT_TERMINATOR_TOKEN: str = ";" SQL_SINGLE_QUOTE_TOKEN: str = "'" SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." -SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) SQL_WITH_KEYWORD: str = "WITH" SQL_CEREMONIAL_SELECT_VALUE: str = "1" OMITTED_CEREMONIAL_SELECT_SQL: str = f"\nSELECT {SQL_CEREMONIAL_SELECT_VALUE}" @@ -165,7 +159,6 @@ COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH: int = 3 COMPACT_ANALYSIS_FACT_LENGTH: int = 6 COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 -MODEL_SCHEMA_CONFIG_KEY: str = "schema" SQL_TEST_SCAN_STORE_FILE_NAME: str = "sql-test-scans.bin" RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" SQL_TEST_SCAN_STORE_VERSION: str = "sql-test-scan-store-v1" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index f4ccdefbec..aeeea875b4 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -69,9 +69,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import ( - DefaultsConfig, LocalConfig, - MaterializationDefaultsConfig, ProjectConfig, ResolvedTableType, ResolvedTimeTravelRetention, @@ -1606,31 +1604,6 @@ class NativeCompactAnalysis: resource_names: dict[int, str] = field(default_factory=dict) -@dataclass(frozen=True) -class IdentityPresenceCache: - """Identity-safe memoization for recursive authored configuration scans.""" - - _values: dict[int, tuple[object, bool]] = field(default_factory=dict) - - def get(self, value: object) -> bool | None: - cached: tuple[object, bool] | None = self._values.get(id(value)) - if cached is None or cached[0] is not value: - return None - return cached[1] - - def put(self, *, value: object, result: bool) -> None: - self._values[id(value)] = (value, result) - - -@dataclass(frozen=True) -class ModelConfigScanCache: - """Memoized recursive scans reused while attaching model configuration.""" - - template_presence: IdentityPresenceCache = field(default_factory=IdentityPresenceCache) - macro_presence: IdentityPresenceCache = field(default_factory=IdentityPresenceCache) - native: bool = False - - @dataclass(frozen=True) class NativeModelConfigSession: """The native config builder, validator, and inherited storage policies for one compile.""" @@ -1678,29 +1651,12 @@ class ModelValidationRequest: @dataclass(frozen=True) class ModelValidatorContext: - """The project facts the model validators read, and the native session in preview.""" + """The project facts the model validators read, and the native config session.""" names: ModelResourceNames settings: SettingsConfig external_sql_reference_resolver: ExternalSqlReferenceResolver | None - native_config: NativeModelConfigSession | None - - -@dataclass(frozen=True) -class ModelConfigBuildRequest: - """Cohesive inputs for one effective model-configuration build.""" - - defaults: DefaultsConfig - path_defaults: dict[str, dict[str, object]] - matched_path_default: str | None - model_header_values: dict[str, object] - effective_vars: dict[str, object] - target_config: TargetConfig | None - model_name: str - effective_target_name: str | None - run_id: str - materialization_defaults: MaterializationDefaultsConfig | None = None - scan_cache: ModelConfigScanCache | None = None + native_config: NativeModelConfigSession @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 4b4a9be988..caa9ffa10c 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -60,10 +60,7 @@ class NativeFallbackSite(StrEnum): MACRO_UNBRIDGED_EXPANSION = "macro_calls.unbridged_expansion" MACRO_BRIDGE_UNAVAILABLE = "macro_calls.bridge_unavailable" MACRO_CALL_MOCKED = "macro_calls.mocked_evaluation" - CONFIG_BUILD = "model_config.build" CONFIG_HEADER_METADATA = "model_config.header_metadata" - CONFIG_PRESENCE_SCAN = "model_config.presence_scan" - CONFIG_VALIDATORS = "model_config.validators" AUDIT_RENDERING = "attachments.audit_rendering" FUNCTION_HEADER = "attachments.function_header" CURSOR_INTRINSIC_REJECTION = "attachments.cursor_intrinsic_rejection" diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py b/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py deleted file mode 100644 index 0a4fc7eed4..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_config_contains_macro_call.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Macro call presence over authored model config, answered natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def native_config_contains_macro_call(value: object) -> bool | None: - """Return whether any nested string holds a macro call, or None where Python must scan.""" - - return _native.config_contains_macro_call(value) diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py b/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py deleted file mode 100644 index 897e41ee03..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_config_contains_template.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Template presence over authored model config, answered natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def native_config_contains_template(value: object) -> bool | None: - """Return whether any nested string holds `${`, or None where Python must scan.""" - - return _native.config_contains_template(value) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py index e3b74d071c..14e29fc0a1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_config_errors.py @@ -7,7 +7,6 @@ import pytest -import sqlbuild.compiler.compile._helpers.attachment.core as attachment_core import sqlbuild.compiler.compile._helpers.attachment.model_config as model_config import sqlbuild.compiler.compile._helpers.render.context_templates as context_templates from scripts.compiler_differential.constants import FAILURE_BASE_MART @@ -50,10 +49,8 @@ " cursor_type integer,\n" ) _FALLBACKS: tuple[tuple[ModuleType, str], ...] = ( - (model_config, "run_python_model_validators"), (model_config, "_parse_model_header_columns"), (model_config, "parse_audit_instances"), - (attachment_core, "build_model_config"), (context_templates, "expand_template_data"), ) _PROJECT_FILES: dict[str, str] = { diff --git a/tests/goldens/compiler/failure/engine-error-non-ascii-replay-duration.json b/tests/goldens/compiler/failure/engine-error-non-ascii-replay-duration.json new file mode 100644 index 0000000000..a15a3e46f2 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-non-ascii-replay-duration.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "model 'customer_totals': replay_on_change '١d' uses digits outside ASCII", + "help": "write replay_on_change with ASCII digits 0-9, add this to the MODEL header:\n MODEL (\n replay_on_change 'bounded-14d',\n ...\n );" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/engine-error-oversized-retention.json b/tests/goldens/compiler/failure/engine-error-oversized-retention.json new file mode 100644 index 0000000000..0993480614 --- /dev/null +++ b/tests/goldens/compiler/failure/engine-error-oversized-retention.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "model 'customer_totals': time_travel_retention '99999999999999999999d' has a number larger than a 64-bit integer", + "help": "use a time_travel_retention whose days fit in 64 bits, add this to the MODEL header:\n MODEL (\n time_travel_retention '7d',\n ...\n );" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py index 4c46aa9acf..5b8023c515 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py @@ -1,4 +1,4 @@ -"""Test case types for the native model config build and validator parity tests.""" +"""Test case types for the native model validator error tests.""" from __future__ import annotations @@ -6,45 +6,9 @@ @dataclass(frozen=True) -class ModelValidationParityTestCase: - """Seeded effective configs validated natively and by the Python validators.""" - - description: str - seed: int - count: int - expected_minimum_native_accepted: int - expected_minimum_python_rejected: int - expected_minimum_acceptance_percent: int - expected_minimum_error_percent: int - - -@dataclass(frozen=True) -class ModelConfigBuildParityTestCase: - """Seeded defaults, path defaults, headers and targets built natively and in Python.""" - - description: str - seed: int - count: int - expected_minimum_built: int - expected_minimum_python_raised: int - expected_minimum_build_percent: int - expected_minimum_error_percent: int - - -@dataclass(frozen=True) -class ModelConfigProjectTestCase: - """A whole project compiled and built on DuckDB under one compiler engine.""" - - description: str - engine: str - expected_models: tuple[str, ...] - - -@dataclass(frozen=True) -class NativeErrorParityTestCase: - """An invalid config whose native outcome must be Python's exact error or a deferral.""" +class NativeErrorTestCase: + """An invalid config and the exact error the native validators raise for it.""" description: str values: dict[str, object] - expected_native_outcome: object - expected_python_outcome: object + expected_outcome: object diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index d7a7d0106d..a6efbdcc43 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -1,63 +1,29 @@ -"""Seeded model configs, layers and targets compared between the native and Python paths.""" +"""Validate one model's effective config with a native model config session.""" from __future__ import annotations -import random -from dataclasses import dataclass, replace -from datetime import date -from itertools import compress -from operator import itemgetter from pathlib import Path from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment.core import build_model_config from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - build_native_model_config, - find_matching_path_default, native_model_config_session, native_model_validation, - native_path_default, native_validation_error, - run_python_model_validators, ) -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileModelConfig, CompileSqlReference, - ModelConfigBuildRequest, ModelResourceNames, ModelValidationRequest, - ModelValidatorContext, NativeModelConfigInputs, NativeModelConfigSession, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlModelFile, - NamedSqlHookEntry, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.spec.contracts.models import ( - AuthoredTimeTravelRetention, - DefaultsConfig, - MaterializationDefaultsConfig, - MaterializationRetentionDefaults, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - SchemaColumn, - SettingsConfig, - TargetConfig, -) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.spec.contracts.models import ProjectConfig -_ABSENT: object = object() _ACCEPTED: str = "accepted" -_DEFERRED: str = "native defers" -_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_RUN_ID: str = "20261008T000000Z_orders" -BUILD_ENVIRONMENT: dict[str, str] = {"SQB_BUILD_DB": "ops", "SQB_BUILD_SCHEMA": "nightly"} _NAMES: ModelResourceNames = ModelResourceNames( models={"orders", "customers"}, seeds={"regions"}, @@ -66,317 +32,48 @@ table_functions={"order_lines"}, custom_materializations=frozenset({"ledger"}), ) -_VALIDATOR_CONTEXT: ModelValidatorContext = ModelValidatorContext( - names=_NAMES, - settings=SettingsConfig(), - external_sql_reference_resolver=None, - native_config=None, -) - - -def _reference(kind: str, name: str) -> CompileSqlReference: - return CompileSqlReference(ref_kind=kind, ref_name=name) - - -_ORDERS: CompileSqlReference = _reference("ref", "orders") -_REFERENCES: tuple[CompileSqlReference, ...] = ( - _ORDERS, - _reference("ref", "customers"), - _reference("ref", "invoices"), - _reference("seed", "regions"), - _reference("seed", "orders"), - _reference("source", "raw.orders"), - _reference("source", "raw.refunds"), - _reference("udf", "cents"), - _reference("udf", "order_lines"), - _reference("table_fn", "order_lines"), - _reference("table_fn", "cents"), - _reference("dbt_ref", "orders"), -) -_WATERMARK_INPUTS: dict[str, object] = { - "orders": {"column": "updated_at", "roles": ["filter", "watermark"]} -} -_INCREMENTAL: dict[str, object] = { - "materialized": "incremental", - "incremental_strategy": "append", - "cursor": "updated_at", - "cursor_type": "timestamp", - "cursor_grain": "day", -} -_PROFILES: tuple[tuple[dict[str, object], tuple[CompileSqlReference, ...]], ...] = ( - ({"materialized": "table", "tags": ["core"]}, (_ORDERS,)), - ({"materialized": "view", "contract": "none"}, ()), - ({**_INCREMENTAL, "cursor_start": "2024-01-01", "cursor_end": "2024-03-01"}, (_ORDERS,)), - ( - { - **_INCREMENTAL, - "incremental_strategy": "delete_insert", - "incremental_mode": "microbatch", - "microbatch_strategy": "watermark", - "cursor_watermark_mode": "all", - "batch_size": "1d", - "lookback": "2d", - "cursor_inputs": _WATERMARK_INPUTS, - "microbatch_limit": {"max_batches": 6, "action": "cap_from_start"}, - }, - (_ORDERS,), - ), - ( - { - **_INCREMENTAL, - "incremental_mode": "microbatch", - "microbatch_strategy": "rolling_window", - "cursor_inputs": {"orders": "updated_at"}, - }, - (_ORDERS,), - ), - ( - { - "materialized": "incremental", - "incremental_strategy": "merge", - "unique_key": ["order_id"], - "merge_exclude_columns": ["loaded_at"], - "on_schema_change": "append_new_columns", - "replay_on_change": "bounded-14d", - }, - (_ORDERS,), - ), - ( - { - "materialized": "incremental", - "incremental_strategy": "delete_insert", - "cursor": "order_id", - "cursor_type": "integer", - "cursor_start": "10", - "cursor_end": 20, - "unique_key": "order_id", - }, - (_ORDERS,), - ), - ( - { - "materialized": "snapshot", - "unique_key": "order_id", - "snapshot_strategy": "timestamp", - "updated_at": "updated_at", - "observed_at": "loaded_at", - "historical_input": "snapshot", - }, - (_ORDERS,), - ), - ( - { - "materialized": "snapshot", - "unique_key": ["order_id"], - "snapshot_strategy": "check", - "check_columns": ["*"], - "contract": "enforced", - "columns": {"order_id": None, "status": None}, - }, - (_ORDERS,), - ), - ({"materialized": "ledger", "placeholders": {"region": "emea"}}, ()), - ( - {"materialized": "table", "migrate_from": "legacy_orders", "old_name_view": "7d"}, - (_ORDERS,), - ), -) -_VALIDATION_POOLS: dict[str, tuple[object, ...]] = { - "materialized": ("table", "view", "incremental", "snapshot", "ledger", "archive", 5), - "incremental_strategy": ("append", "delete_insert", "merge", "upsert", 3, _ABSENT), - "cursor": ("updated_at", "order_id", 7, _ABSENT), - "cursor_type": ("timestamp", "integer", "date", _ABSENT), - "cursor_grain": ("day", "hour", "month", "week", _ABSENT), - "cursor_start": ( - "2024-01-01", - "2024-02-01T00:00:00", - "2024-01-01 10:00:00+02:00", - "2024-01-01T00:00:00.123Z", - "2024-13-01", - "20240101", - "2024-W01-1", - "0001-01-01T00:00:00+01:00", - 10, - "15", - "1e3", - "abc", - date(2024, 1, 1), - True, - _ABSENT, - ), - "cursor_end": ( - "2024-01-01", - "2024-06-01 00:00", - 5, - 40, - "30", - "2025-01-01T00:00:00-05:00", - "9999-12-31T23:59:00-01:00", - ), - "unique_key": ("order_id", ["order_id", "customer_id"], ["\udcff"], [], (), 5, _ABSENT), - "merge_exclude_columns": (["amount"], ["ORDER_ID"], ["a", "A"], [""], "amount", _ABSENT), - "incremental_mode": ("full", "microbatch", "batch", _ABSENT), - "microbatch_strategy": ("rolling_window", "watermark", "hourly", _ABSENT), - "cursor_watermark_mode": ("all", "any", "none", _ABSENT), - "batch_size": ("1d", "effective", "2h", "1mo", 5, _ABSENT), - "batch_concurrency": (1, 2, 0, -1, True, "2", 2**70, _ABSENT), - "unaccounted_partition_policy": ("synthesize", "recover_all", "skip", _ABSENT), - "cursor_inputs": ( - {"orders": "updated_at"}, - _WATERMARK_INPUTS, - {"customers": {"column": "c", "roles": ["watermark"]}}, - {"orders": {"column": "c", "roles": ["filter", "filter"]}}, - {}, - {"orders": ""}, - {"invoices": "updated_at"}, - ["orders"], - _ABSENT, - ), - "max_microbatches": (1, 3, 10, 0, True, "3", _ABSENT), - "microbatch_limit": ( - {"max_batches": 5, "action": "cap_from_start"}, - {"max_batches": 2, "action": "warn"}, - {"max_batches": 0, "action": "error"}, - {"max_batches": 5}, - _ABSENT, - ), - "lookback": ("3d", "1mo", "2h", "1mo2d", "soon", _ABSENT), - "on_schema_change": ("fail", "ignore", "drop", 5, _ABSENT), - "replay_on_change": ("forward", "full", "bounded-14d", "bounded- 1mo ", "bounded-x", _ABSENT), - "append_cursor_inclusive": (True, False, "yes", _ABSENT), - "full_refresh": (True, False, "no", _ABSENT), - "cursor_start_max_ahead": ("disabled", "0d", "7d", "soon", _ABSENT), - "cursor_start_max_action": ("cap", "error", "warn", _ABSENT), - "cursor_future_max_distance": ("1d", "disabled", "later", _ABSENT), - "cursor_future_action": ("cap", "error", ["cap"], _ABSENT), - "contract": ("enforced", "none", "strict", 1, _ABSENT), - "columns": ({"order_id": None, "updated_at": None}, _ABSENT), - "snapshot_strategy": ("timestamp", "check", "hybrid", _ABSENT), - "updated_at": ("updated_at", "status", 5, _ABSENT), - "observed_at": ("loaded_at", _ABSENT), - "historical_input": ("snapshot", "changes", "events", _ABSENT), - "initial_valid_from": ("updated_at", "observed_at", "execution_time", "now", _ABSENT), - "snapshot_full_refresh": ("deny", "allow", "always", _ABSENT), - "snapshot_schema_change": ("deny", "append_new_columns", "drop", _ABSENT), - "check_columns": (["*"], ["*", "status"], ["status"], ["\udcff"], [], ("*",), _ABSENT), - "invalidate_hard_deletes": (True, False, "yes", _ABSENT), - "valid_from_column": ("valid_from", "VALID_TO", _ABSENT), - "valid_to_column": ("valid_to", _ABSENT), - "migrate_from": ("legacy_orders", " ", "orders_daily", " ORDERS_DAILY ", 5, _ABSENT), - "migrate_force": (True, False, "yes", _ABSENT), - "old_name_view": ("7d", " 2d ", False, True, "soon", None, _ABSENT), - "placeholders": ({"region": "emea"}, {"other": "x"}, ["region"], _ABSENT), -} -_RETENTIONS: tuple[ResolvedTimeTravelRetention, ...] = ( - ResolvedTimeTravelRetention(), - ResolvedTimeTravelRetention(), - ResolvedTimeTravelRetention(desired_days=7, unmanaged=False), -) -_TABLE_TYPES: tuple[ResolvedTableType, ...] = ( - ResolvedTableType(), - ResolvedTableType(), - ResolvedTableType(value=TableType.PERMANENT, source=TableTypeSource.MODEL, declared=True), -) -_DECLARED_COLUMNS: tuple[tuple[SchemaColumn, ...] | None, ...] = ( - None, - None, - (SchemaColumn(name="order_id"), SchemaColumn(name="updated_at")), - (SchemaColumn(name="customer_id"),), -) -_QUERIES: tuple[str, ...] = ( - "SELECT 1", - "SELECT 1", - "SELECT '@@@region' AS marker, @@@region AS region", - "SELECT @@@other", - "SELECT '@@@' AS marker", -) - - -@dataclass(frozen=True) -class ModelValidationParity: - """How native validation compared with the Python validators over one corpus.""" - - mismatches: list[tuple[object, object, object]] - native_accepted: int - native_errors: int - python_accepted: int - python_rejected: int - - -@dataclass(frozen=True) -class ConfigBuildCase: - """One project's layers and target and one model file to build config for.""" - - project_config: ProjectConfig - target_config: TargetConfig | None - effective_target_name: str | None - model_file: DiscoveredSqlModelFile - - -@dataclass(frozen=True) -class ModelConfigBuildParity: - """How native config builds compared with Python's over one corpus.""" - - mismatches: list[tuple[object, object, object]] - built: int - native_errors: int - deferred: int - python_raised: int -def generated_validation_requests( - *, rng: random.Random, count: int -) -> list[ModelValidationRequest]: - """Return seeded effective configs mixing valid profiles with invalid and unusual values.""" +def validation_outcome(*, values: dict[str, object]) -> object: + """Return `accepted`, or the raised error's shape, for model `orders_daily` reading orders.""" - return [_validation_request(rng=rng) for _ in range(count)] - - -def model_validation_parity(*, requests: list[ModelValidationRequest]) -> ModelValidationParity: - """Validate natively and in Python; native may defer but never differ from Python.""" - - session: NativeModelConfigSession = _session( - project_config=ProjectConfig(name="orders", adapter="duckdb"), - target_config=None, - effective_target_name=None, - ) - outcomes: list[tuple[object, object, object]] = [ - ( - request.config.values, - _python_validation(request), - _native_validation(session=session, request=request), - ) - for request in requests - ] - return ModelValidationParity( - mismatches=list( - compress( - outcomes, [native not in (_DEFERRED, python) for _, python, native in outcomes] - ) + session: NativeModelConfigSession = native_model_config_session( + inputs=NativeModelConfigInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + target_config=None, + effective_vars={}, + effective_target_name=None, + run_id="20261008T000000Z_orders", + microbatch_concurrency=False, ), - native_accepted=sum(native == _ACCEPTED for _, _, native in outcomes), - native_errors=sum(native not in (_ACCEPTED, _DEFERRED) for _, _, native in outcomes), - python_accepted=sum(python == _ACCEPTED for _, python, _ in outcomes), - python_rejected=sum(python != _ACCEPTED for _, python, _ in outcomes), + names=_NAMES, ) - - -def validation_outcomes(*, values: dict[str, object]) -> tuple[object, object]: - """Return native validation's outcome for `values` and the Python validators' outcome.""" - request: ModelValidationRequest = ModelValidationRequest( - model_file=_model_file(relative_path="marts/orders_daily.sql", header_values={}), + model_file=DiscoveredSqlModelFile( + file_path=Path("/project/models/marts/orders_daily.sql"), + relative_path=Path("models/marts/orders_daily.sql"), + contents="", + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="SELECT 1", + ), config=CompileModelConfig(values=values), - references=(_ORDERS,), + references=(CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name="orders"),), declared_columns=None, query_sql="SELECT 1", ) - session: NativeModelConfigSession = _session( - project_config=ProjectConfig(name="orders", adapter="duckdb"), - target_config=None, - effective_target_name=None, + outcome: _native.NativeConfigError | None = cast( + _native.NativeConfigError | None, + native_model_validation(session=session, request=request, rejected=[False]), ) - return (_native_validation(session=session, request=request), _python_validation(request)) + return [ + _ACCEPTED, + *( + raised_error_shape(native_validation_error(error=error, values=values)) + for error in filter(None, (outcome,)) + ), + ][-1] def header_help(*, purpose: str, entry: str) -> str: @@ -392,283 +89,10 @@ def header_help(*, purpose: str, entry: str) -> str: def raised_error_shape(error: Exception) -> tuple[object, ...]: """Return the type, message, code and help of an error, and whether it skips the bridge.""" - bridge_independent: object = getattr(error, "bridge_independent", None) return ( type(error).__name__, str(error), getattr(error, "code", None), getattr(error, "help", None), - bridge_independent, - ) - - -def _python_error_shape(error: Exception) -> tuple[object, ...]: - shape: tuple[object, ...] = raised_error_shape(error) - return (*shape[:4], shape[4] is not None or None) - - -def _python_validation(request: ModelValidationRequest) -> object: - try: - run_python_model_validators(context=_VALIDATOR_CONTEXT, request=request) - except Exception as error: # noqa: BLE001 - the exact Python outcome, whatever it is - return raised_error_shape(error) - return _ACCEPTED - - -def _native_validation( - *, session: NativeModelConfigSession, request: ModelValidationRequest -) -> object: - outcome: bool | _native.NativeConfigError = native_model_validation( - session=session, request=request - ) - errors: list[object] = [ - raised_error_shape( - native_validation_error( - error=cast(_native.NativeConfigError, error), values=request.config.values - ) - ) - for error in compress([outcome], [isinstance(outcome, _native.NativeConfigError)]) - ] - return (*errors, {True: _ACCEPTED, False: _DEFERRED}[outcome is True])[0] - - -def _validation_request(*, rng: random.Random) -> ModelValidationRequest: - profile, references = rng.choice(_PROFILES) - values: dict[str, object] = dict(profile) - mutated: list[str] = rng.sample( - sorted(_VALIDATION_POOLS), k=rng.choice((0, 0, 1, 1, 2, 3, 5, 8)) - ) - values.update((key, rng.choice(_VALIDATION_POOLS[key])) for key in mutated) - return ModelValidationRequest( - model_file=_model_file(relative_path="marts/orders_daily.sql", header_values={}), - config=CompileModelConfig( - values=dict( - compress(values.items(), [value is not _ABSENT for value in values.values()]) - ), - time_travel_retention=rng.choice(_RETENTIONS), - table_type=rng.choice(_TABLE_TYPES), - ), - references=rng.choice( - (references, references, references, tuple(rng.sample(_REFERENCES, k=2))) - ), - declared_columns=rng.choice(_DECLARED_COLUMNS), - query_sql=rng.choice(_QUERIES), - ) - - -_DEFAULTS: tuple[DefaultsConfig, ...] = ( - DefaultsConfig(), - DefaultsConfig(materialized="table", schema="analytics", tags=("core",)), - DefaultsConfig( - materialized="view", - database="warehouse", - schema="${region}_core", - full_refresh=True, - row_diff_exclude_columns=("loaded_at",), - row_diff_tolerances={"by_type": {"DOUBLE": 0.1}, "sample": 3}, - ), - DefaultsConfig(pre_hooks=[SqlHookEntry(statement="SELECT 1")], tags=("core", "daily")), - DefaultsConfig(incremental_strategy="append", cursor_start="2024-01-01", contract="none"), -) -_PATH_DEFAULTS: tuple[dict[str, dict[str, object]], ...] = ( - {}, - {"staging": {"materialized": "view", "tags": ["staging", "core"]}}, - { - "staging/*": {"schema": "${CTX:model.name}_stg", "row_diff_exclude_columns": ["x"]}, - "marts": { - "materialized": "table", - "row_diff_tolerances": {"by_column": {"amount": 0.5}, "by_type": "loose"}, - "post_hooks": [NamedSqlHookEntry(name="audit_refresh", kwargs={})], - }, - }, - { - "**/finance": {"database": "${ENV:SQB_BUILD_DB}", "tags": "finance"}, - "marts/**": {"tags": [1, 2], "schema": "${CTX:run.target}"}, - }, - {"*/orders.sql": {"materialized": "table"}, "staging/*": {"materialized": "view"}}, -) -_HEADER_POOLS: dict[str, tuple[object, ...]] = { - "materialized": ("table", "incremental", "view", "snapshot", 5), - "schema": ( - "orders", - "${region}_${CTX:model.name}", - "${CTX:run.target}", - "${ENV:SQB_BUILD_SCHEMA}", - "${missing}", - "${CTX:destination.table}", - "${coalesce(ENV:SQB_UNSET_SCHEMA, 'fallback')}", - ), - "database": ("warehouse", "${ENV:SQB_BUILD_DB}", None), - "alias": ("orders_v2", "${CTX:model.schema}_x", "@cents(amount)"), - "tags": (["daily"], ["core", "daily"], "daily", [1]), - "full_refresh": (True, False), - "time_travel_retention": ("7d", "inherit", "disabled", "0d", "6h", 7), - "table_type": ("permanent", "transient", "inherit", "temporary"), - "row_diff_exclude_columns": (["updated_at"], ("x", "y")), - "row_diff_tolerances": ({"by_type": {"INTEGER": 1}}, {"by_column": {"a": 2}, "x": 1}, "loose"), - "description": ("Orders.", "Totals for @cents(1)"), - "unique_key": (["order_id"],), - "pre_hooks": ([SqlHookEntry(statement="SELECT 1")], ["SELECT 1"], None), - "post_hooks": ([PythonHookEntry(name="notify", kwargs={})],), -} -_TARGETS: tuple[TargetConfig | None, ...] = ( - None, - TargetConfig(), - TargetConfig(database="dev_db", schema="${CTX:model.schema}_dev"), - TargetConfig(schema="preserve"), - TargetConfig(database="preserve", schema="analytics"), - TargetConfig( - schema="${ENV:SQB_BUILD_SCHEMA}", - default_table_type=TableType.PERMANENT, - time_travel_retention=AuthoredTimeTravelRetention(desired_days=3), - time_travel_retention_by_materialization={ - "incremental": AuthoredTimeTravelRetention(unmanaged=True) - }, - ), -) -_MATERIALIZATION_DEFAULTS: tuple[MaterializationDefaultsConfig, ...] = ( - MaterializationDefaultsConfig(), - MaterializationDefaultsConfig( - table=MaterializationRetentionDefaults( - time_travel_retention=AuthoredTimeTravelRetention(desired_days=1), - table_type=TableType.PERMANENT, - ) - ), -) -_MODEL_PATHS: tuple[str, ...] = ( - "staging/orders.sql", - "marts/finance/revenue.sql", - "orders.sql", -) - - -def generated_config_builds(*, rng: random.Random, count: int) -> list[ConfigBuildCase]: - """Return seeded project layers, targets and MODEL headers, valid and invalid.""" - - return [_config_build_case(rng=rng) for _ in range(count)] - - -def model_config_build_parity(*, cases: list[ConfigBuildCase]) -> ModelConfigBuildParity: - """Build natively and in Python; native may defer but never differ or hide an error.""" - - outcomes: list[tuple[object, object, object]] = [ - (case.model_file.header_values, _python_build(case), _native_build(case)) for case in cases - ] - return ModelConfigBuildParity( - mismatches=list( - compress( - outcomes, - [native not in (_DEFERRED, python) for _, python, native in outcomes], - ) - ), - built=sum(isinstance(native, list) for _, _, native in outcomes), - native_errors=sum(isinstance(native, tuple) for _, _, native in outcomes), - deferred=sum(native == _DEFERRED for _, _, native in outcomes), - python_raised=sum(isinstance(python, tuple) for _, python, _ in outcomes), - ) - - -def _config_build_case(*, rng: random.Random) -> ConfigBuildCase: - keys: list[str] = rng.sample(sorted(_HEADER_POOLS), k=rng.choice((0, 1, 2, 3, 4))) - return ConfigBuildCase( - project_config=ProjectConfig( - name="orders", - adapter="duckdb", - defaults=rng.choice(_DEFAULTS), - path_defaults=rng.choice(_PATH_DEFAULTS), - materialization_defaults=rng.choice(_MATERIALIZATION_DEFAULTS), - ), - target_config=rng.choice(_TARGETS), - effective_target_name=rng.choice(("dev", None)), - model_file=_model_file( - relative_path=rng.choice(_MODEL_PATHS), - header_values={key: rng.choice(_HEADER_POOLS[key]) for key in keys}, - ), - ) - - -def _python_build(case: ConfigBuildCase) -> object: - with COMPILE_INPUT_READS.recording() as reads: - try: - matched: str | None = find_matching_path_default( - model_file=case.model_file, path_defaults=case.project_config.path_defaults - ) - config: CompileModelConfig = build_model_config( - request=ModelConfigBuildRequest( - defaults=case.project_config.defaults, - path_defaults=case.project_config.path_defaults, - matched_path_default=matched, - model_header_values=case.model_file.header_values, - effective_vars={"region": "emea"}, - target_config=case.target_config, - model_name=case.model_file.file_path.stem, - effective_target_name=case.effective_target_name, - run_id=_RUN_ID, - materialization_defaults=case.project_config.materialization_defaults, - ) - ) - except Exception as error: # noqa: BLE001 - the exact Python outcome, whatever it is - return _python_error_shape(error) - return [_config_shape(config), reads.environment_names, reads.read_run_id] - - -def _native_build(case: ConfigBuildCase) -> object: - session: NativeModelConfigSession = _session( - project_config=case.project_config, - target_config=case.target_config, - effective_target_name=case.effective_target_name, - ) - with COMPILE_INPUT_READS.recording() as reads: - try: - matched: str | None = native_path_default(session=session, model_file=case.model_file) - config: CompileModelConfig | None = build_native_model_config( - session=session, model_file=case.model_file, matched_path_default=matched - ) - except Exception as error: # noqa: BLE001 - the exact native outcome, whatever it is - return raised_error_shape(error) - shapes: list[object] = [ - [_config_shape(built), reads.environment_names, reads.read_run_id] - for built in cast(list[CompileModelConfig], list(compress([config], [config is not None]))) - ] - return (*shapes, _DEFERRED)[0] - - -def _config_shape(config: CompileModelConfig) -> tuple[object, ...]: - items: list[tuple[str, object]] = list(config.values.items()) - hooks: list[bool] = [key in _HOOK_KEYS for key, _ in items] - return ( - list(compress(items, [not hook for hook in hooks])), - sorted(compress(items, hooks), key=itemgetter(0)), - replace(config, values={}), - ) - - -def _session( - *, - project_config: ProjectConfig, - target_config: TargetConfig | None, - effective_target_name: str | None, -) -> NativeModelConfigSession: - return native_model_config_session( - inputs=NativeModelConfigInputs( - project_config=project_config, - target_config=target_config, - effective_vars={"region": "emea"}, - effective_target_name=effective_target_name, - run_id=_RUN_ID, - microbatch_concurrency=False, - ), - names=_NAMES, - ) - - -def _model_file(*, relative_path: str, header_values: dict[str, object]) -> DiscoveredSqlModelFile: - return DiscoveredSqlModelFile( - file_path=Path("/project/models") / relative_path, - relative_path=Path("models") / relative_path, - contents="", - header_values=header_values, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1", + getattr(error, "bridge_independent", None), ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py index a8c19e353f..420d977265 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py @@ -1,26 +1,15 @@ -"""Native model config builds and validator checks match Python exactly or defer to it.""" +"""Native model validators raise the exact error, code and help for invalid configs.""" from __future__ import annotations -import random - import pytest from tests.integration.src.sqlbuild.compiler.compile.native_model_config._test_types import ( - ModelConfigBuildParityTestCase, - ModelValidationParityTestCase, - NativeErrorParityTestCase, + NativeErrorTestCase, ) from tests.integration.src.sqlbuild.compiler.compile.native_model_config.helpers import ( - BUILD_ENVIRONMENT, - ModelConfigBuildParity, - ModelValidationParity, - generated_config_builds, - generated_validation_requests, header_help, - model_config_build_parity, - model_validation_parity, - validation_outcomes, + validation_outcome, ) _INCREMENTAL: dict[str, object] = { @@ -35,95 +24,14 @@ @pytest.mark.parametrize( "test_case", [ - ModelValidationParityTestCase( - description="seeded profiles with invalid and unusual values", - seed=20261008, - count=6000, - expected_minimum_native_accepted=1000, - expected_minimum_python_rejected=1500, - expected_minimum_acceptance_percent=90, - expected_minimum_error_percent=85, - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_configs_when_validating_then_native_matches_python_or_defers( - test_case: ModelValidationParityTestCase, -) -> None: - parity: ModelValidationParity = model_validation_parity( - requests=generated_validation_requests( - rng=random.Random(test_case.seed), count=test_case.count - ) - ) - - assert ( - parity.mismatches[:3], - parity.native_accepted >= test_case.expected_minimum_native_accepted, - parity.python_rejected >= test_case.expected_minimum_python_rejected, - parity.native_accepted * 100 - >= parity.python_accepted * test_case.expected_minimum_acceptance_percent, - parity.native_errors * 100 - >= parity.python_rejected * test_case.expected_minimum_error_percent, - ) == ([], True, True, True, True), ( - parity.native_accepted, - parity.native_errors, - parity.python_accepted, - parity.python_rejected, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ModelConfigBuildParityTestCase( - description="seeded defaults, path defaults, headers and targets", - seed=20261009, - count=6000, - expected_minimum_built=2000, - expected_minimum_python_raised=500, - expected_minimum_build_percent=90, - expected_minimum_error_percent=80, - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_layers_when_building_config_then_native_matches_python_or_defers( - test_case: ModelConfigBuildParityTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - for name, value in BUILD_ENVIRONMENT.items(): - monkeypatch.setenv(name, value) - - parity: ModelConfigBuildParity = model_config_build_parity( - cases=generated_config_builds(rng=random.Random(test_case.seed), count=test_case.count) - ) - - assert ( - parity.mismatches[:3], - parity.built >= test_case.expected_minimum_built, - parity.python_raised >= test_case.expected_minimum_python_raised, - parity.built * 100 - >= (test_case.count - parity.python_raised) * test_case.expected_minimum_build_percent, - parity.native_errors * 100 - >= parity.python_raised * test_case.expected_minimum_error_percent, - ) == ([], True, True, True, True), ( - parity.built, - parity.native_errors, - parity.deferred, - parity.python_raised, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - NativeErrorParityTestCase( + NativeErrorTestCase( description="a start bound shifted before year 1 in UTC", values={ **_INCREMENTAL, "cursor_start": "0001-01-01T00:00:00+01:00", "cursor_end": "2024-01-01", }, - expected_native_outcome=( + expected_outcome=( "CompileInputError", "model 'orders_daily': cursor_start value '0001-01-01T00:00:00+01:00' falls " "outside years 1-9999 once converted to UTC", @@ -134,17 +42,15 @@ def test_given_generated_layers_when_building_config_then_native_matches_python_ ), False, ), - expected_python_outcome="native outcome", ), - NativeErrorParityTestCase( + NativeErrorTestCase( description="an end bound shifted past year 9999 in UTC", values={ **_INCREMENTAL, "cursor_start": "2024-01-01", "cursor_end": "9999-12-31T23:59:30.5-01:00", }, - expected_native_outcome="native defers", - expected_python_outcome=( + expected_outcome=( "CompileInputError", "model 'orders_daily': cursor_end value '9999-12-31T23:59:30.5-01:00' falls " "outside years 1-9999 once converted to UTC", @@ -156,10 +62,10 @@ def test_given_generated_layers_when_building_config_then_native_matches_python_ False, ), ), - NativeErrorParityTestCase( + NativeErrorTestCase( description="an invalid replay policy with its header help", values={**_INCREMENTAL, "replay_on_change": "bounded-soon"}, - expected_native_outcome=( + expected_outcome=( "CompileInputError", "model 'orders_daily': replay_on_change 'bounded-soon' has an invalid duration " "'soon'; use a positive duration such as 14d, 12h or 1mo", @@ -169,24 +75,22 @@ def test_given_generated_layers_when_building_config_then_native_matches_python_ ), False, ), - expected_python_outcome="native outcome", ), - NativeErrorParityTestCase( + NativeErrorTestCase( description="a non-string materialization", values={"materialized": 5}, - expected_native_outcome=( + expected_outcome=( "ConfigValueTypeError", "config key 'materialized' expected a string, got int", None, None, None, ), - expected_python_outcome="native outcome", ), - NativeErrorParityTestCase( + NativeErrorTestCase( description="an old-name view that is not a duration", values={"materialized": "table", "old_name_view": "soon"}, - expected_native_outcome=( + expected_outcome=( "CompileInputError", "model 'orders_daily': old_name_view must be a positive duration such as 7d, or " "false; got 'soon'", @@ -194,41 +98,14 @@ def test_given_generated_layers_when_building_config_then_native_matches_python_ "write for example old_name_view 7d, or old_name_view false", False, ), - expected_python_outcome="native outcome", - ), - NativeErrorParityTestCase( - description="an unreadable unique key under an enforced contract", - values={ - "materialized": "incremental", - "incremental_strategy": "merge", - "unique_key": ["\udcff"], - "contract": "enforced", - "columns": {"order_id": None}, - }, - expected_native_outcome="native defers", - expected_python_outcome=( - "CompileInputError", - "model 'orders_daily': unique_key references column '\udcff' not declared in " - "enforced contract", - "P001", - None, - False, - ), ), ], ids=lambda case: case.description, ) -def test_given_invalid_configs_when_validating_then_native_raises_python_error_or_defers( - test_case: NativeErrorParityTestCase, +def test_given_invalid_configs_when_validating_then_native_raises_the_exact_error( + test_case: NativeErrorTestCase, ) -> None: - native, python = validation_outcomes(values=test_case.values) - - assert (native, python) == ( - test_case.expected_native_outcome, - {"native outcome": test_case.expected_native_outcome}.get( - str(test_case.expected_python_outcome), test_case.expected_python_outcome - ), - ) + assert validation_outcome(values=test_case.values) == test_case.expected_outcome if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index d9f326ba3d..27ffb0e466 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -17,17 +17,6 @@ class HeaderMetadataParityTestCase: expected_minimum_unsupported: int -@dataclass(frozen=True) -class ConfigPresenceParityTestCase: - """Seeded config values scanned for templates and macro calls natively and in Python.""" - - description: str - seed: int - count: int - expected_minimum_present: int - expected_minimum_deferred: int - - @dataclass(frozen=True) class ModelConfigTierTestCase: """One compiler engine and how often its model loop calls each native model config entry.""" diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 48806a06c3..25af086600 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -14,10 +14,6 @@ import sqlbuild._native as _native from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances -from sqlbuild.compiler.compile._helpers.render.templating import ( - contains_template_data, -) -from sqlbuild.compiler.compile.constants import MACRO_CALL_PATTERN from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs @@ -25,12 +21,6 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( - native_config_contains_macro_call, -) -from sqlbuild.compiler.model_config.main._native_config_contains_template import ( - native_config_contains_template, -) from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.model_config.main._parse_native_header_metadata import ( parse_native_header_metadata, @@ -99,9 +89,10 @@ NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ( "parse_model_header_metadata", "expand_config_templates", - "config_contains_template", - "config_contains_macro_call", ) +_NATIVE_ENTRIES: dict[str, Callable[..., object]] = { + name: getattr(_native, name) for name in NATIVE_MODEL_CONFIG_ENTRIES +} MODEL_CONFIG_PROJECT: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' @@ -135,27 +126,12 @@ class HeaderMetadataParity: unsupported: int -@dataclass(frozen=True) -class ConfigPresenceParity: - """How the native presence scans compared with Python's over one corpus.""" - - mismatches: list[tuple[object, object, object]] - present: int - deferred: int - - def generated_header_metadata(*, rng: random.Random, count: int) -> list[tuple[object, object]]: """Return seeded `(columns, audits)` header values mixing valid and invalid shapes.""" return [(_columns(rng=rng), _audit_list(rng=rng)) for _ in range(count)] -def generated_config_values(*, rng: random.Random, count: int) -> list[object]: - """Return seeded nested config values holding templates, macro calls and plain text.""" - - return [_config_value(rng=rng, depth=0) for _ in range(count)] - - def model_config_engine_outcome( *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch ) -> tuple[dict[str, int], str]: @@ -207,7 +183,7 @@ def error_shape(error: Exception) -> tuple[object, ...]: def _counted(*, calls: dict[str, int], name: str) -> Callable[..., object]: - entry: Callable[..., object] = getattr(_native, name) + entry: Callable[..., object] = _NATIVE_ENTRIES[name] def counted(*args: object) -> object: calls[name] += 1 @@ -246,62 +222,6 @@ def header_metadata_parity(*, headers: list[tuple[object, object]]) -> HeaderMet ) -def config_presence_parity(*, values: list[object]) -> ConfigPresenceParity: - """Compare the native template and macro scans with Python's wherever they answer.""" - - native: list[tuple[bool | None, bool | None]] = [ - (native_config_contains_template(value), native_config_contains_macro_call(value)) - for value in values - ] - python: list[tuple[bool, bool]] = [ - (contains_template_data(value), _python_contains_macro(value)) for value in values - ] - answered: list[tuple[object, tuple[bool, bool], tuple[bool | None, bool | None]]] = list( - compress( - zip(values, python, native, strict=True), - [None not in answer for answer in native], - ) - ) - return ConfigPresenceParity( - mismatches=mismatches( - inputs=[value for value, _, _ in answered], - expected=[expected for _, expected, _ in answered], - actual=[actual for _, _, actual in answered], - ), - present=sum(any(expected) for _, expected, _ in answered), - deferred=len(values) - len(answered), - ) - - -def _python_contains_macro(value: object) -> bool: - return _MACRO_SCANS.get(type(value), _absent)(value) - - -def _absent(value: object) -> bool: - del value - return False - - -def _string_contains_macro(value: object) -> bool: - return MACRO_CALL_PATTERN.search(str(value)) is not None - - -def _mapping_contains_macro(value: object) -> bool: - return any(_python_contains_macro(item) for item in cast(dict[object, object], value).values()) - - -def _sequence_contains_macro(value: object) -> bool: - return any(_python_contains_macro(item) for item in cast(tuple[object, ...], value)) - - -_MACRO_SCANS: dict[type, Callable[[object], bool]] = { - str: _string_contains_macro, - dict: _mapping_contains_macro, - list: _sequence_contains_macro, - tuple: _sequence_contains_macro, -} - - def _python_metadata(*, model_file: DiscoveredSqlModelFile) -> object: try: return ( @@ -481,16 +401,3 @@ def _audit(*, rng: random.Random) -> object: ), weights=(35, 10, 5, 50), )[0] - - -def _config_value(*, rng: random.Random, depth: int) -> object: - factories: tuple[Callable[[], object], ...] = ( - lambda: rng.choice(_TEXTS), - lambda: [_config_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))], - lambda: tuple(_config_value(rng=rng, depth=depth + 1) for _ in range(rng.randint(0, 3))), - lambda: { - rng.choice(("schema", "${key}", "tags", 1)): _config_value(rng=rng, depth=depth + 1) - for _ in range(rng.randint(0, 3)) - }, - ) - return rng.choices(factories, weights=(50 + 1000 * (depth > 2), 20, 10, 20))[0]() diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 823e323941..14e682d19e 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -8,15 +8,11 @@ import pytest from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( - ConfigPresenceParityTestCase, HeaderMetadataParityTestCase, ModelConfigTierTestCase, ) from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( - ConfigPresenceParity, HeaderMetadataParity, - config_presence_parity, - generated_config_values, generated_header_metadata, header_metadata_parity, model_config_engine_outcome, @@ -52,33 +48,6 @@ def test_given_generated_header_metadata_when_parsing_then_native_matches_python ) == ([], True, True, True), (parity.parsed, parity.rejected, parity.unsupported) -@pytest.mark.parametrize( - "test_case", - [ - ConfigPresenceParityTestCase( - description="seeded nested config with templates and macro calls", - seed=20261008, - count=4000, - expected_minimum_present=400, - expected_minimum_deferred=20, - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_config_when_scanning_presence_then_native_matches_python_or_defers( - test_case: ConfigPresenceParityTestCase, -) -> None: - parity: ConfigPresenceParity = config_presence_parity( - values=generated_config_values(rng=random.Random(test_case.seed), count=test_case.count) - ) - - assert ( - parity.mismatches, - parity.present >= test_case.expected_minimum_present, - parity.deferred >= test_case.expected_minimum_deferred, - ) == ([], True, True), (parity.present, parity.deferred) - - @pytest.mark.parametrize( "test_case", [ @@ -86,10 +55,8 @@ def test_given_generated_config_when_scanning_presence_then_native_matches_pytho description="python oracle", engine="python", expected_native_calls={ - "parse_model_header_metadata": 0, - "expand_config_templates": 14, - "config_contains_template": 0, - "config_contains_macro_call": 0, + "parse_model_header_metadata": 1, + "expand_config_templates": 1, }, ), ModelConfigTierTestCase( @@ -97,9 +64,7 @@ def test_given_generated_config_when_scanning_presence_then_native_matches_pytho engine="native", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 8, - "config_contains_template": 0, - "config_contains_macro_call": 0, + "expand_config_templates": 1, }, ), ModelConfigTierTestCase( @@ -107,15 +72,13 @@ def test_given_generated_config_when_scanning_presence_then_native_matches_pytho engine="native-preview", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 8, - "config_contains_template": 0, - "config_contains_macro_call": 0, + "expand_config_templates": 1, }, ), ], ids=lambda case: case.description, ) -def test_given_engine_tier_when_building_model_inputs_then_native_config_runs_only_in_native_engines( +def test_given_engine_tier_when_building_model_inputs_then_every_engine_runs_native_config( test_case: ModelConfigTierTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: calls, config = model_config_engine_outcome( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 73fedc0eee..fa56cb533f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -1,42 +1,64 @@ -"""Helpers for model configuration reuse tests.""" +"""Helpers for model configuration reuse and native validation tests.""" +import sys +import unicodedata from collections.abc import Callable from pathlib import Path from typing import Any import pytest +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core +from sqlbuild.compiler.compile._helpers.attachment.model_config import ( + build_native_model_config, + native_model_config_session, + native_validation_error, +) from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileModelConfig, CompileProjectInputs +from sqlbuild.compiler.compile.models import ( + CompileModelConfig, + CompileProjectInputs, + ModelResourceNames, + NativeModelConfigInputs, + NativeModelConfigSession, +) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine -from sqlbuild.spec.contracts.models import SchemaColumn +from sqlbuild.spec.contracts.models import ( + DefaultsConfig, + MaterializationDefaultsConfig, + ProjectConfig, + SchemaColumn, + TargetConfig, +) from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) +_NO_MATERIALIZATION_DEFAULTS: MaterializationDefaultsConfig = MaterializationDefaultsConfig() + def compile_with_config_build_count( *, project_dir: Path, monkeypatch: pytest.MonkeyPatch, ) -> tuple[CompileProjectInputs, int]: - """Compile a project on the Python engine while counting its model configuration builds.""" + """Compile a project on the Python engine while counting its native model config builds.""" monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.PYTHON.value) build_counts: list[int] = [0] original_build_model_config: Callable[..., CompileModelConfig] = ( - attachment_core.build_model_config + attachment_core.build_native_model_config ) def counting_build_model_config(**kwargs: Any) -> CompileModelConfig: build_counts[0] += 1 return original_build_model_config(**kwargs) - monkeypatch.setattr(attachment_core, "build_model_config", counting_build_model_config) + monkeypatch.setattr(attachment_core, "build_native_model_config", counting_build_model_config) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) return ( build_compile_inputs( @@ -54,3 +76,94 @@ def compile_input_schema_columns(inputs: CompileProjectInputs) -> tuple[SchemaCo for model_input in inputs.model_inputs: columns.extend(getattr(model_input.schema_entry, "columns", ())) return tuple(columns) + + +def validate_natively( # noqa: PLR0913 + *, + config: CompileModelConfig, + model_name: str = "test_model", + input_names: frozenset[str] = frozenset(), + ref_count: int = 0, + declared_columns: tuple[str, ...] | None = None, + custom_materialization_names: frozenset[str] = frozenset(), + query_sql: str = "SELECT 1", + microbatch_concurrency: bool = False, +) -> None: + """Run the native model validators, raising their first error. + + The model reads `ref_count` models, the first named from `input_names`; every one exists. + """ + + names: list[str] = sorted(input_names)[:ref_count] + names.extend(f"upstream_{index}" for index in range(len(names), ref_count)) + validator: _native.NativeModelValidator = _native.NativeModelValidator( + (set(names), set(), set(), set(), set()), + set(custom_materialization_names), + microbatch_concurrency, + (sys.version_info[0], sys.version_info[1]), + unicodedata.unidata_version, + ) + outcome: int | _native.NativeConfigError | None = validator.validate( + config.values, + (model_name, query_sql, f"models/{model_name}.sql"), + ( + [("ref", name, False) for name in names], + None if declared_columns is None else list(declared_columns), + config.time_travel_retention.unmanaged, + config.table_type.declared, + ), + ) + assert not isinstance(outcome, int) + for error in filter(None, (outcome,)): + raise native_validation_error(error=error, values=config.values) + + +def build_config_natively( # noqa: PLR0913 + *, + defaults: DefaultsConfig, + path_defaults: dict[str, dict[str, object]], + matched_path_default: str | None, + model_header_values: dict[str, object], + target_config: TargetConfig | None, + materialization_defaults: MaterializationDefaultsConfig = _NO_MATERIALIZATION_DEFAULTS, + effective_target_name: str | None = None, +) -> CompileModelConfig: + """Build the effective config of model `orders` with the native config builder.""" + + session: NativeModelConfigSession = native_model_config_session( + inputs=NativeModelConfigInputs( + project_config=ProjectConfig( + name="shop", + adapter="duckdb", + defaults=defaults, + path_defaults=path_defaults, + materialization_defaults=materialization_defaults, + ), + target_config=target_config, + effective_vars={}, + effective_target_name=effective_target_name, + run_id="run_123", + microbatch_concurrency=False, + ), + names=ModelResourceNames( + models={"orders"}, + seeds=set(), + sources=set(), + functions=set(), + table_functions=set(), + custom_materializations=frozenset(), + ), + ) + return build_native_model_config( + session=session, + model_file=DiscoveredSqlModelFile( + file_path=Path("models/marts/orders.sql"), + relative_path=Path("models/marts/orders.sql"), + contents="SELECT 1", + header_values=model_header_values, + header_column_locations={}, + output_column_locations={}, + query_sql="SELECT 1", + ), + matched_path_default=matched_path_default, + ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index 390b45004e..ac7589ee26 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -2,7 +2,6 @@ import pytest -from sqlbuild.compiler.compile._helpers.config.model_validation import validate_table_type from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileModelConfig @@ -18,6 +17,7 @@ TableTypeResolutionTestCase, TableTypeValidationErrorTestCase, ) +from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import validate_natively @pytest.mark.parametrize( @@ -176,7 +176,11 @@ def test_given_non_table_materialization_when_validating_table_type_then_rejects ) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_table_type(config=config, model_name="orders") + validate_natively( + config=config, + model_name="orders", + custom_materialization_names=frozenset({test_case.materialized}), + ) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 26eb29822b..bf6eac5e51 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -4,9 +4,6 @@ import pytest -from sqlbuild.compiler.compile._helpers.config.model_validation import ( - validate_incremental_config, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.planner._helpers.resolve.cursor import ( @@ -25,6 +22,7 @@ MicrobatchRedesignBehaviorTestCase, WatermarkLimitValidationTestCase, ) +from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import validate_natively @pytest.mark.parametrize( @@ -85,7 +83,7 @@ def test_given_consumer_and_producer_grains_when_resolving_then_strategy_owns_co def test_given_watermark_strategy_when_inputs_have_explicit_roles_then_config_is_valid( test_case: MicrobatchRedesignBehaviorTestCase, ) -> None: - validate_incremental_config( + validate_natively( config=CompileModelConfig( values={ "materialized": "incremental", @@ -109,7 +107,7 @@ def test_given_watermark_strategy_when_inputs_have_explicit_roles_then_config_is ), model_name="events", ref_count=2, - known_input_names=frozenset({"archive", "live"}), + input_names=frozenset({"archive", "live"}), ) assert test_case.expected_outcome is None @@ -126,7 +124,7 @@ def test_given_watermark_strategy_when_inputs_have_explicit_roles_then_config_is def test_given_watermark_strategy_when_nested_limit_caps_from_end_then_config_is_valid( test_case: MicrobatchRedesignBehaviorTestCase, ) -> None: - validate_incremental_config( + validate_natively( config=CompileModelConfig( values={ "materialized": "incremental", @@ -146,7 +144,7 @@ def test_given_watermark_strategy_when_nested_limit_caps_from_end_then_config_is ), model_name="events", ref_count=1, - known_input_names=frozenset({"events"}), + input_names=frozenset({"events"}), ) assert test_case.expected_outcome is None @@ -164,7 +162,7 @@ def test_given_watermark_strategy_when_cursor_inputs_use_shorthand_then_compilat test_case: MicrobatchRedesignBehaviorTestCase, ) -> None: with pytest.raises(CompileInputError, match=str(test_case.expected_outcome)): - validate_incremental_config( + validate_natively( config=CompileModelConfig( values={ "materialized": "incremental", @@ -181,7 +179,7 @@ def test_given_watermark_strategy_when_cursor_inputs_use_shorthand_then_compilat ), model_name="events", ref_count=1, - known_input_names=frozenset({"events"}), + input_names=frozenset({"events"}), ) @@ -199,7 +197,7 @@ def test_given_model_limit_below_lookback_when_validating_then_compilation_fails test_case: MicrobatchRedesignBehaviorTestCase, ) -> None: with pytest.raises(CompileInputError, match=str(test_case.expected_outcome)): - validate_incremental_config( + validate_natively( config=CompileModelConfig( values={ "materialized": "incremental", @@ -223,7 +221,7 @@ def test_given_model_limit_below_lookback_when_validating_then_compilation_fails ), model_name="events", ref_count=1, - known_input_names=frozenset({"events"}), + input_names=frozenset({"events"}), ) @@ -308,11 +306,11 @@ def test_given_insufficient_capped_watermark_limit_when_validating_then_error_ex assert test_case.expected_error_fragment is not None with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_incremental_config( + validate_natively( config=config, model_name="events", ref_count=1, - known_input_names=frozenset({"events"}), + input_names=frozenset({"events"}), ) @@ -335,7 +333,7 @@ def test_given_sufficient_cap_from_end_limit_when_validating_then_config_is_acce test_case: WatermarkLimitValidationTestCase, ) -> None: assert test_case.expected_error_fragment is None - validate_incremental_config( + validate_natively( config=CompileModelConfig( values={ "materialized": "incremental", @@ -362,7 +360,7 @@ def test_given_sufficient_cap_from_end_limit_when_validating_then_config_is_acce ), model_name="events", ref_count=1, - known_input_names=frozenset({"events"}), + input_names=frozenset({"events"}), ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index ef3f2a1c28..a2dedcf3c7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -6,15 +6,6 @@ import pytest -from sqlbuild.compiler.compile._helpers.config.model_validation import ( - validate_contract_config, - validate_custom_materialization_config, - validate_incremental_config, - validate_non_incremental_config, - validate_placeholder_config, - validate_snapshot_config, - validate_time_travel_retention, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError @@ -37,8 +28,10 @@ SnapshotConfigErrorTestCase, SnapshotConfigValidTestCase, ) +from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import validate_natively _SNIPPET_INDENT: str = " " * 12 +_CUSTOM: frozenset[str] = frozenset({"partition_tracked"}) @pytest.mark.parametrize( @@ -64,7 +57,7 @@ def test_given_valid_contract_config_when_validating_then_passes( ) -> None: config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) - validate_contract_config(config=config, model_name="test_model") + validate_natively(config=config, model_name="test_model") assert test_case.expected_valid is True @@ -91,7 +84,7 @@ def test_given_invalid_contract_config_when_validating_then_raises( config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError) as exc_info: - validate_contract_config(config=config, model_name="test_model") + validate_natively(config=config, model_name="test_model") assert test_case.expected_error_fragment in str(exc_info.value) @@ -321,11 +314,12 @@ def test_given_valid_config_when_validating_then_passes( ) known_input_names: frozenset[str] = frozenset(str(name) for name in cursor_input_dict) - validate_incremental_config( + validate_natively( config=config, model_name="test_model", ref_count=test_case.ref_count, - known_input_names=known_input_names, + input_names=known_input_names, + microbatch_concurrency=True, ) assert test_case.expected_valid @@ -811,11 +805,11 @@ def test_given_invalid_config_when_validating_then_raises( config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_incremental_config( + validate_natively( config=config, model_name="test_model", ref_count=test_case.ref_count, - known_input_names=frozenset({"orders", "shipments"}), + input_names=frozenset({"orders", "shipments"}), ) @@ -861,11 +855,11 @@ def test_given_unknown_change_policy_when_validating_then_help_shows_valid_heade ) with pytest.raises(CompileInputError) as error_info: - validate_incremental_config( + validate_natively( config=config, model_name="test_model", ref_count=1, - known_input_names=frozenset({"orders"}), + input_names=frozenset({"orders"}), ) assert f"unknown {test_case.key} '{test_case.value}'" in str(error_info.value) @@ -898,11 +892,11 @@ def test_given_integer_batch_size_when_validating_then_raises_typed_config_error config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(ConfigValueTypeError) as error_info: - validate_incremental_config( + validate_natively( config=config, model_name="test_model", ref_count=test_case.ref_count, - known_input_names=frozenset({"orders"}), + input_names=frozenset({"orders"}), ) assert error_info.value.key == "batch_size" @@ -923,12 +917,17 @@ def test_given_integer_batch_size_when_validating_then_raises_typed_config_error ), NonIncrementalConfigValidTestCase( description="incremental model with on_schema_change passes", - config_values={"materialized": "incremental", "on_schema_change": "append_new_columns"}, + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "on_schema_change": "append_new_columns", + }, ), NonIncrementalConfigValidTestCase( description="incremental model with replay_on_change passes", config_values={ "materialized": "incremental", + "incremental_strategy": "append", "replay_on_change": "bounded-30d", }, ), @@ -951,7 +950,7 @@ def test_given_valid_non_incremental_config_when_validating_then_passes( ) -> None: config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) - validate_non_incremental_config( + validate_natively( config=config, model_name="test_model", ) @@ -1012,7 +1011,7 @@ def test_given_non_incremental_config_with_incremental_keys_when_validating_then config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_non_incremental_config( + validate_natively( config=config, model_name="test_model", ) @@ -1146,7 +1145,7 @@ def test_given_valid_snapshot_config_when_validating_then_passes( ) -> None: config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) - validate_snapshot_config(config=config, model_name="test_model") + validate_natively(config=config, model_name="test_model") assert test_case.expected_valid is True @@ -1364,7 +1363,7 @@ def test_given_invalid_snapshot_config_when_validating_then_raises( config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment) as exc_info: - validate_snapshot_config(config=config, model_name="test_model") + validate_natively(config=config, model_name="test_model") assert exc_info.value.code == test_case.expected_error_code @@ -1413,7 +1412,7 @@ def test_given_valid_custom_materialization_config_when_validating_then_passes( ) -> None: config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) - validate_custom_materialization_config( + validate_natively( config=config, model_name="test_model", custom_materialization_names=test_case.custom_materialization_names, @@ -1453,7 +1452,7 @@ def test_given_valid_custom_materialization_config_when_validating_then_passes( description="on_schema_change disallowed on custom materialization", config_values={"materialized": "partition_tracked", "on_schema_change": "fail"}, custom_materialization_names=frozenset({"partition_tracked"}), - expected_error_fragment="on_schema_change is not allowed on custom materializations", + expected_error_fragment="on_schema_change is only valid for incremental models", ), CustomMaterializationConfigErrorTestCase( description="batch_size disallowed on custom materialization", @@ -1476,7 +1475,7 @@ def test_given_invalid_custom_materialization_config_when_validating_then_raises config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_custom_materialization_config( + validate_natively( config=config, model_name="test_model", custom_materialization_names=test_case.custom_materialization_names, @@ -1518,7 +1517,7 @@ def test_given_unsupported_managed_retention_when_validating_then_raises( ) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_time_travel_retention(config=config, model_name="test_model") + validate_natively(config=config, custom_materialization_names=_CUSTOM) @pytest.mark.parametrize( @@ -1544,7 +1543,7 @@ def test_given_disabled_retention_when_validating_then_passes( time_travel_retention=test_case.retention, ) - validate_time_travel_retention(config=config, model_name="test_model") + validate_natively(config=config, model_name="test_model") assert config.time_travel_retention.unmanaged is test_case.expected_unmanaged @@ -1590,7 +1589,7 @@ def test_given_valid_placeholder_config_when_validating_then_passes( ) -> None: config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) - validate_placeholder_config( + validate_natively( config=config, model_name="test_model", query_sql=test_case.query_sql, @@ -1653,7 +1652,7 @@ def test_given_invalid_placeholder_config_when_validating_then_raises( config: CompileModelConfig = CompileModelConfig(values=test_case.config_values) with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_placeholder_config( + validate_natively( config=config, model_name="test_model", query_sql=test_case.query_sql, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py index 8e3379e8c7..e8381aa308 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.attachment.core import find_matching_path_default +from sqlbuild.compiler.compile._helpers.attachment.model_config import find_matching_path_default from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 99ebb91d9f..3ed278a55d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -7,13 +7,11 @@ import pytest -from sqlbuild.compiler.compile._helpers.attachment.core import build_model_config from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, CompileProjectInputs, CompileSqlFunctionInput, - ModelConfigBuildRequest, ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( @@ -38,6 +36,7 @@ TargetConfig, ) from sqlbuild.spec.contracts.types import TimeTravelRetentionSource +from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import build_config_natively from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -4615,18 +4614,12 @@ def test_given_typed_hook_defaults_when_building_config_then_layers_values( "models/marts": {"post_hooks": test_case.path_default_hook} } - config: CompileModelConfig = build_model_config( - request=ModelConfigBuildRequest( - defaults=defaults, - path_defaults=path_defaults, - matched_path_default="models/marts", - model_header_values={}, - effective_vars={}, - target_config=None, - model_name="orders", - effective_target_name=None, - run_id="run_123", - ) + config: CompileModelConfig = build_config_natively( + defaults=defaults, + path_defaults=path_defaults, + matched_path_default="models/marts", + model_header_values={}, + target_config=None, ) assert config.values["pre_hooks"] == test_case.expected_pre_hooks @@ -4702,19 +4695,14 @@ def test_given_typed_hook_defaults_when_building_config_then_layers_values( def test_given_retention_layers_when_building_model_config_then_precedence_is_resolved( test_case: BuildModelRetentionConfigTestCase, ) -> None: - config: CompileModelConfig = build_model_config( - request=ModelConfigBuildRequest( - defaults=test_case.defaults, - path_defaults={}, - matched_path_default=None, - model_header_values=test_case.model_header_values, - effective_vars={}, - target_config=test_case.target_config, - model_name="orders", - effective_target_name="prod", - run_id="run_123", - materialization_defaults=test_case.materialization_defaults, - ) + config: CompileModelConfig = build_config_natively( + defaults=test_case.defaults, + path_defaults={}, + matched_path_default=None, + model_header_values=test_case.model_header_values, + target_config=test_case.target_config, + materialization_defaults=test_case.materialization_defaults, + effective_target_name="prod", ) assert config.time_travel_retention.desired_days == test_case.expected_desired_days From 208eedd0c5064b92c4d1743073ed6e652b939614 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:57:52 +0100 Subject: [PATCH 15/40] feat(native): parse MODEL header columns and audits natively for every model; delete the Python header metadata fallback --- .../src/model_files/_helpers/parsing.rs | 15 ++ .../src/header_metadata/_helpers/audits.rs | 22 +-- .../src/header_metadata/_helpers/columns.rs | 48 +++---- .../src/header_metadata/_helpers/identity.rs | 27 ++-- .../src/header_metadata/_helpers/mod.rs | 1 + .../src/header_metadata/_helpers/text.rs | 49 ++----- .../header_metadata/_helpers/thresholds.rs | 136 ++++++++++++++++++ .../src/header_metadata/constants.rs | 5 + .../src/header_metadata/models.rs | 36 +++-- .../src/header_metadata/tests/helpers.rs | 18 +-- .../tests/test_parse_header_metadata.rs | 71 ++++++++- .../src/tests/test_types.rs | 9 ++ crates/sqlbuild-model-config/src/types.rs | 4 + .../_helpers/model_config/authored_nodes.rs | 16 ++- .../_helpers/model_config/header_metadata.rs | 126 +++++++++------- .../_helpers/corpus/failure_cases.py | 16 +++ .../native_fallbacks.toml | 48 +------ src/sqlbuild/_native.pyi | 10 +- .../compile/_helpers/attachment/core.py | 12 -- .../_helpers/attachment/model_config.py | 129 ++--------------- src/sqlbuild/compiler/compile/models.py | 25 ---- src/sqlbuild/compiler/frontier/types.py | 1 - .../compiler/model_config/constants.py | 28 +--- .../main/_parse_native_header_metadata.py | 95 ++++-------- src/sqlbuild/compiler/model_config/models.py | 24 +--- .../main/compile/test_native_config_errors.py | 3 - ...-error-non-ascii-digit-model-constant.json | 19 +++ .../compiler/model_config/_test_types.py | 1 - .../sqlbuild/compiler/model_config/helpers.py | 76 +++++++--- .../model_config/test_native_model_config.py | 14 +- 30 files changed, 556 insertions(+), 528 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs create mode 100644 tests/goldens/compiler/failure/engine-error-non-ascii-digit-model-constant.json diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index abfed5e535..5bbad2b735 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -3,6 +3,8 @@ use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::_helpers::header_syntax::{FailedHeader, header_syntax_failure}; use crate::_helpers::locations::header_column_locations; +use crate::declaration_files::_helpers::checks::python_values::{WordRules, check_bare_numbers}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::model_files::_helpers::output_columns::output_column_locations; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use crate::models::{DiscoveryFailure, FailureKind}; @@ -73,6 +75,19 @@ pub(crate) fn parse_model_file( python: options.python, })); } + for (_, value) in &header_values { + check_bare_numbers( + value, + WordRules { + python: options.python, + file_path, + }, + ) + .map_err(|ParseStop::Failed(failure)| DiscoveryFailure { + kind: FailureKind::ModelSql, + ..failure + })?; + } let query_sql: String = python_strip(&contents[sql_start..]).to_owned(); if query_sql.is_empty() { return Err(failure(format!( diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs index 5e7915de96..10696f36b8 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs @@ -1,18 +1,20 @@ //! Audit lists and instances, as `parse_audit_instances` reads them with `null_as_empty`. +use crate::errors::ConfigError; use crate::header_metadata::_helpers::identity::check_identity; use crate::header_metadata::_helpers::text::{ Site, entry, non_blank_text, optional_bool, optional_count, optional_text, }; +use crate::header_metadata::_helpers::thresholds::thresholds; use crate::header_metadata::constants::{AUDIT_OPTION_KEYS, AUDIT_SEVERITIES}; -use crate::header_metadata::models::{HeaderMetadataStop, ParsedAudit}; +use crate::header_metadata::models::ParsedAudit; use crate::types::{AuthoredNode, NodeKind}; /// Parse an authored audit list; Python's `None` is an empty list. pub(crate) fn audit_list( node: &N, site: Site<'_>, -) -> Result>, HeaderMetadataStop> { +) -> Result>, ConfigError> { match node.kind() { NodeKind::Null => Ok(Vec::new()), NodeKind::List => node @@ -27,10 +29,10 @@ pub(crate) fn audit_list( fn audit_instance( node: &N, site: Site<'_>, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let definition_kind = format!("{} audit definition", site.label); if node.kind() == NodeKind::Str { - let Some(text) = non_blank_text(node)? else { + let Some(text) = non_blank_text(node) else { return Err(site.error("audits must not contain empty names")); }; check_identity(&text, &definition_kind, site.path)?; @@ -43,7 +45,7 @@ fn audit_instance( if node.kind() != NodeKind::Map { return Err(site.error("audits must be strings or single-key mappings")); } - let Some(definition) = non_blank_text(definition_name)? else { + let Some(definition) = non_blank_text(definition_name) else { return Err(site.error("audit names must be non-empty strings")); }; check_identity(&definition, &definition_kind, site.path)?; @@ -59,7 +61,7 @@ fn configured_audit( definition: &str, options: &[(N, N)], site: Site<'_>, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let option_label = format!("{} audit '{definition}'", site.label); let option_site = Site { path: site.path, @@ -67,7 +69,7 @@ fn configured_audit( }; let name = optional_text(entry(options, "name"), option_site, "name")?; if let Some(name) = &name { - let text = name.text().ok_or(HeaderMetadataStop::Unsupported)?; + let text = name.text().unwrap_or_default(); check_identity(&text, &format!("{} audit instance", site.label), site.path)?; } let description = optional_text(entry(options, "description"), option_site, "description")?; @@ -82,9 +84,7 @@ fn configured_audit( } let run_scope = optional_text(entry(options, "run_scope"), option_site, "run_scope")?; let always_run = optional_bool(entry(options, "always_run"), option_site, "always_run")?; - if entry(options, "thresholds").is_some_and(|value| value.kind() != NodeKind::Null) { - return Err(HeaderMetadataStop::Unsupported); - } + let thresholds = thresholds(entry(options, "thresholds"), option_site)?; let minimum_samples = optional_count( entry(options, "minimum_samples"), option_site, @@ -108,6 +108,7 @@ fn configured_audit( severity, run_scope, always_run, + thresholds, minimum_samples, evidence_limit, }) @@ -122,6 +123,7 @@ fn bare_audit(definition_name: N) -> ParsedAudit { severity: None, run_scope: None, always_run: None, + thresholds: None, minimum_samples: None, evidence_limit: None, } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs index df77dcdcb0..f03f31a08f 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs @@ -10,14 +10,14 @@ use crate::header_metadata::_helpers::text::{ use crate::header_metadata::constants::{ MODEL_COLUMN_KEYS, MODEL_LABEL, NOT_NULL_AUDIT_NAME, NULLABLE_NOT_NULL_CODE, }; -use crate::header_metadata::models::{HeaderMetadataStop, ParsedColumn}; +use crate::header_metadata::models::ParsedColumn; use crate::types::{AuthoredNode, NodeKind}; /// Parse an authored columns mapping; Python's `None` declares no columns. pub(crate) fn column_mapping( node: &N, path: &str, -) -> Result>, HeaderMetadataStop> { +) -> Result>, ConfigError> { let site = Site { path, label: MODEL_LABEL, @@ -30,13 +30,10 @@ pub(crate) fn column_mapping( let mut seen: HashSet = HashSet::new(); let mut columns: Vec> = Vec::new(); for (name, metadata) in node.entries() { - let Some(text) = non_blank_text(&name)? else { + let Some(text) = non_blank_text(&name) else { return Err(site.error("column names must be non-empty strings")); }; - if !text.is_ascii() { - return Err(HeaderMetadataStop::Unsupported); - } - if !seen.insert(text.to_ascii_lowercase()) { + if !seen.insert(text.to_lowercase()) { return Err(site.error(&format!( "has duplicate column '{text}' (column names are case-insensitive)" ))); @@ -51,7 +48,7 @@ fn column( text: &str, metadata: &N, path: &str, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let label = format!("{MODEL_LABEL} column '{text}'"); let site = Site { path, @@ -61,7 +58,7 @@ fn column( return Err(site.error("metadata must be a mapping")); } let entries = metadata.entries(); - let unknown = unknown_keys(&entries)?; + let unknown = unknown_keys(&entries); if !unknown.is_empty() { return Err(site.error(&format!( "has unknown metadata keys: {}", @@ -81,13 +78,11 @@ fn column( .iter() .any(|audit| audit.definition_name.is_text(NOT_NULL_AUDIT_NAME)) { - return Err(HeaderMetadataStop::Error( - ConfigError::compile(format!( - "{path} column '{text}' cannot set nullable = true and audit not_null" - )) - .with_code(NULLABLE_NOT_NULL_CODE) - .with_help("remove the not_null audit or set nullable = false"), - )); + return Err(ConfigError::compile(format!( + "{path} column '{text}' cannot set nullable = true and audit not_null" + )) + .with_code(NULLABLE_NOT_NULL_CODE) + .with_help("remove the not_null audit or set nullable = false")); } Ok(ParsedColumn { name: name.clone(), @@ -99,18 +94,13 @@ fn column( }) } -/// Return the sorted metadata keys outside `MODEL_COLUMN_KEYS`, deferring keys Python cannot sort. -fn unknown_keys(entries: &[(N, N)]) -> Result, HeaderMetadataStop> { - let mut unknown: Vec = Vec::new(); - for (key, _) in entries { - if MODEL_COLUMN_KEYS.iter().any(|name| key.is_text(name)) { - continue; - } - if key.kind() != NodeKind::Str { - return Err(HeaderMetadataStop::Unsupported); - } - unknown.push(key.text().ok_or(HeaderMetadataStop::Unsupported)?); - } +/// Return the sorted metadata keys outside `MODEL_COLUMN_KEYS`; header keys are always strings. +fn unknown_keys(entries: &[(N, N)]) -> Vec { + let mut unknown: Vec = entries + .iter() + .filter(|(key, _)| !MODEL_COLUMN_KEYS.iter().any(|name| key.is_text(name))) + .map(|(key, _)| key.python_str()) + .collect(); unknown.sort_unstable(); - Ok(unknown) + unknown } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs index c046a733ab..b32e40efc8 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs @@ -1,26 +1,21 @@ -//! `validate_resource_identity` for ASCII names, with its suggested snake_case spelling. +//! `validate_resource_identity`, with its suggested snake_case spelling. use crate::errors::{ConfigError, ErrorClass}; -use crate::header_metadata::_helpers::text::ascii_strip; +use sqlbuild_core::text::main::python_strip::python_strip; + use crate::header_metadata::constants::{FALLBACK_IDENTITY, IDENTITY_HELP}; -use crate::header_metadata::models::HeaderMetadataStop; -/// Raise `ResourceIdentityError` unless `name` is canonical snake_case; defer non-ASCII names. -pub(crate) fn check_identity(name: &str, kind: &str, path: &str) -> Result<(), HeaderMetadataStop> { +/// Raise `ResourceIdentityError` unless `name` is canonical snake_case. +pub(crate) fn check_identity(name: &str, kind: &str, path: &str) -> Result<(), ConfigError> { if is_snake_case(name) { return Ok(()); } - if !name.is_ascii() { - return Err(HeaderMetadataStop::Unsupported); - } let suggestion = suggested_identity(name); - Err(HeaderMetadataStop::Error( - ConfigError::compile(format!( - "Invalid {kind} identity '{name}' in {path}; use snake_case '{suggestion}'" - )) - .with_class(ErrorClass::ResourceIdentity) - .with_help(IDENTITY_HELP), + Err(ConfigError::compile(format!( + "Invalid {kind} identity '{name}' in {path}; use snake_case '{suggestion}'" )) + .with_class(ErrorClass::ResourceIdentity) + .with_help(IDENTITY_HELP)) } /// Return whether `name` fully matches `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`. @@ -38,9 +33,9 @@ fn is_snake_case(name: &str) -> bool { } } -/// Return `suggest_resource_identity` for an ASCII public identity. +/// Return `suggest_resource_identity` for a public identity; its patterns are ASCII-only. fn suggested_identity(name: &str) -> String { - let characters: Vec = ascii_strip(name).chars().collect(); + let characters: Vec = python_strip(name).chars().collect(); let acronyms = split_where(&characters, |index| { characters[index - 1].is_ascii_uppercase() && characters[index].is_ascii_uppercase() diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs index 3e2c0f3b41..2f9c85d8b4 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod audits; pub(crate) mod columns; pub(crate) mod identity; pub(crate) mod text; +pub(crate) mod thresholds; diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs index 14989b1747..1eb9e58fe0 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs @@ -1,7 +1,8 @@ -//! Text checks that agree with Python's `str` methods or defer, and the errors they raise. +//! Text checks that agree with Python's `str` methods, and the errors they raise. + +use sqlbuild_core::text::main::python_strip::python_strip; use crate::errors::ConfigError; -use crate::header_metadata::models::HeaderMetadataStop; use crate::types::{AuthoredNode, NodeKind}; /// Where one parse reports its errors: the file path and the label the messages name. @@ -13,32 +14,14 @@ pub(crate) struct Site<'a> { impl Site<'_> { /// The `CompileInputError` `