From f6d067d3a7c5006a017de174716f537883288d1f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:26:31 +0100 Subject: [PATCH 1/3] feat(native): discover SQL model files natively behind the compiler engine switch Add the sqlbuild-discovery crate with a parallel directory walk that keeps Python's glob and sort semantics, and native reading and parsing of SQL model files with Python's exact discovery messages, locations and ordering. With SQLBUILD_COMPILER_ENGINE=native the discovery stage takes its model files from the native walk; every other collection still runs in Python. Native discovery skips the Python discovery fact cache. --- Cargo.lock | 11 + Cargo.toml | 3 + crates/README.md | 4 +- .../src/text/_helpers/alnum_ranges.rs | 755 ++++++++++++++++++ crates/sqlbuild-core/src/text/_helpers/mod.rs | 2 + .../src/text/_helpers/sequence_matcher.rs | 171 ++++ .../src/text/main/close_matches.rs | 19 + .../src/text/main/decode_python_text.rs | 3 + .../src/text/main/is_python_alnum.rs | 15 + .../src/text/main/is_python_space.rs | 19 + .../src/text/main/is_python_word.rs | 8 + crates/sqlbuild-core/src/text/main/mod.rs | 6 + .../text/main/python_alnum_unicode_version.rs | 8 + .../src/text/main/python_strip.rs | 8 + crates/sqlbuild-core/src/text/mod.rs | 1 + crates/sqlbuild-core/src/text/tests.rs | 2 + .../src/text/tests/test_python_compat.rs | 161 ++++ .../src/text/tests/test_types.rs | 17 + crates/sqlbuild-discovery/Cargo.toml | 19 + crates/sqlbuild-discovery/src/_helpers/mod.rs | 4 + .../sqlbuild-discovery/src/_helpers/pool.rs | 20 + .../src/_helpers/reading.rs | 11 + .../src/_helpers/scoped_paths.rs | 39 + .../src/_helpers/suggestions.rs | 33 + crates/sqlbuild-discovery/src/constants.rs | 29 + crates/sqlbuild-discovery/src/lib.rs | 9 + .../src/model_files/_helpers/header_keys.rs | 122 +++ .../src/model_files/_helpers/locations.rs | 96 +++ .../src/model_files/_helpers/mod.rs | 4 + .../model_files/_helpers/output_columns.rs | 335 ++++++++ .../src/model_files/_helpers/parsing.rs | 99 +++ .../model_files/main/discover_model_files.rs | 49 ++ .../src/model_files/main/mod.rs | 1 + .../sqlbuild-discovery/src/model_files/mod.rs | 6 + .../src/model_files/models.rs | 24 + .../src/model_files/tests.rs | 6 + .../src/model_files/tests/helpers.rs | 75 ++ .../tests/test_discover_model_files.rs | 119 +++ .../src/model_files/tests/test_types.rs | 15 + crates/sqlbuild-discovery/src/models.rs | 91 +++ .../src/tree/_helpers/listing.rs | 56 ++ .../src/tree/_helpers/mod.rs | 3 + .../src/tree/_helpers/ordering.rs | 7 + .../src/tree/_helpers/walking.rs | 55 ++ .../sqlbuild-discovery/src/tree/main/mod.rs | 1 + .../sqlbuild-discovery/src/tree/main/rglob.rs | 33 + crates/sqlbuild-discovery/src/tree/mod.rs | 6 + crates/sqlbuild-discovery/src/tree/models.rs | 40 + crates/sqlbuild-discovery/src/tree/tests.rs | 6 + .../src/tree/tests/helpers.rs | 38 + .../src/tree/tests/test_project_tree.rs | 51 ++ .../src/tree/tests/test_types.rs | 9 + crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/discovery.rs | 117 +++ .../src/bindings/_helpers/functions.rs | 3 +- .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/oracles.rs | 27 + crates/sqlbuild-python/src/bindings/models.rs | 12 + .../_helpers/model_headers/matching.rs | 38 +- .../_helpers/model_headers/tokenization.rs | 19 +- .../sqlbuild-sqltext/src/compiler/main/mod.rs | 2 + .../main/model_header_byte_matching.rs | 6 + .../main/model_header_single_parsing.rs | 6 + fensu.toml | 1 + scripts/generate_python_char_table.py | 15 + scripts/python_char_table/__init__.py | 0 .../python_char_table/_helpers/__init__.py | 0 scripts/python_char_table/_helpers/table.py | 41 + scripts/python_char_table/constants.py | 14 + scripts/python_char_table/main/__init__.py | 0 scripts/python_char_table/main/generate.py | 17 + src/sqlbuild/_native.pyi | 7 + .../_helpers/filesystem/aggregation.py | 41 +- .../discovery/_helpers/native/model_files.py | 126 +++ .../discovery/_helpers/native/payloads.py | 43 + .../discovery/_helpers/sql/model_files.py | 6 + src/sqlbuild/compiler/discovery/constants.py | 2 + .../compiler/discovery/main/discover.py | 46 +- src/sqlbuild/compiler/discovery/types.py | 1 + .../frontier/_helpers/native_frontier.py | 11 +- .../frontier/main/_compile_frontier.py | 9 +- .../src/sqlbuild/compiler/_test_types.py | 22 + .../compiler/discovery/_test_types.py | 29 + .../sqlbuild/compiler/discovery/helpers.py | 125 +++ .../discovery/test_native_model_discovery.py | 146 ++++ .../src/sqlbuild/compiler/helpers.py | 28 + .../test_native_python_compat_oracle.py | 83 ++ 87 files changed, 3697 insertions(+), 72 deletions(-) create mode 100644 crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs create mode 100644 crates/sqlbuild-core/src/text/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/text/_helpers/sequence_matcher.rs create mode 100644 crates/sqlbuild-core/src/text/main/close_matches.rs create mode 100644 crates/sqlbuild-core/src/text/main/is_python_alnum.rs create mode 100644 crates/sqlbuild-core/src/text/main/is_python_space.rs create mode 100644 crates/sqlbuild-core/src/text/main/is_python_word.rs create mode 100644 crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs create mode 100644 crates/sqlbuild-core/src/text/main/python_strip.rs create mode 100644 crates/sqlbuild-core/src/text/tests/test_python_compat.rs create mode 100644 crates/sqlbuild-discovery/Cargo.toml create mode 100644 crates/sqlbuild-discovery/src/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/_helpers/pool.rs create mode 100644 crates/sqlbuild-discovery/src/_helpers/reading.rs create mode 100644 crates/sqlbuild-discovery/src/_helpers/scoped_paths.rs create mode 100644 crates/sqlbuild-discovery/src/_helpers/suggestions.rs create mode 100644 crates/sqlbuild-discovery/src/constants.rs create mode 100644 crates/sqlbuild-discovery/src/lib.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/mod.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/models.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests/test_discover_model_files.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests/test_types.rs create mode 100644 crates/sqlbuild-discovery/src/models.rs create mode 100644 crates/sqlbuild-discovery/src/tree/_helpers/listing.rs create mode 100644 crates/sqlbuild-discovery/src/tree/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs create mode 100644 crates/sqlbuild-discovery/src/tree/_helpers/walking.rs create mode 100644 crates/sqlbuild-discovery/src/tree/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/tree/main/rglob.rs create mode 100644 crates/sqlbuild-discovery/src/tree/mod.rs create mode 100644 crates/sqlbuild-discovery/src/tree/models.rs create mode 100644 crates/sqlbuild-discovery/src/tree/tests.rs create mode 100644 crates/sqlbuild-discovery/src/tree/tests/helpers.rs create mode 100644 crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs create mode 100644 crates/sqlbuild-discovery/src/tree/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery.rs create mode 100644 crates/sqlbuild-sqltext/src/compiler/main/model_header_byte_matching.rs create mode 100644 crates/sqlbuild-sqltext/src/compiler/main/model_header_single_parsing.rs create mode 100644 scripts/generate_python_char_table.py create mode 100644 scripts/python_char_table/__init__.py create mode 100644 scripts/python_char_table/_helpers/__init__.py create mode 100644 scripts/python_char_table/_helpers/table.py create mode 100644 scripts/python_char_table/constants.py create mode 100644 scripts/python_char_table/main/__init__.py create mode 100644 scripts/python_char_table/main/generate.py create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/model_files.py create mode 100644 src/sqlbuild/compiler/discovery/_helpers/native/payloads.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py create mode 100644 tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py diff --git a/Cargo.lock b/Cargo.lock index 8bbfed4ed9..c87be16302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,6 +660,16 @@ dependencies = [ name = "sqlbuild-core" version = "0.138.0" +[[package]] +name = "sqlbuild-discovery" +version = "0.138.0" +dependencies = [ + "rayon", + "sqlbuild-core", + "sqlbuild-sqltext", + "tempfile", +] + [[package]] name = "sqlbuild-python" version = "0.138.0" @@ -671,6 +681,7 @@ dependencies = [ "sqlbuild-analysis", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-discovery", "sqlbuild-rules", "sqlbuild-sqltext", ] diff --git a/Cargo.toml b/Cargo.toml index 0b0de3dfde..d6915c33b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/sqlbuild-core", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", + "crates/sqlbuild-discovery", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -17,6 +18,7 @@ order = [ "sqlbuild-core", "sqlbuild-sqltext", "sqlbuild-config", + "sqlbuild-discovery", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -64,6 +66,7 @@ sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } +sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } stacker = "0.1" diff --git a/crates/README.md b/crates/README.md index e184439cbd..1775579703 100644 --- a/crates/README.md +++ b/crates/README.md @@ -9,6 +9,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | +| `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -19,7 +20,8 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-discovery -> sqlbuild-config + -> sqlbuild-sqltext -> sqlbuild-core ``` `sqlbuild-config` does not depend on the crates below it today; its place in the order only diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs new file mode 100644 index 0000000000..8c4880334d --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_ranges.rs @@ -0,0 +1,755 @@ +//! Generated by `scripts/generate_python_char_table.py`; do not edit. + +/// The Unicode version of the Python that generated the table. +pub(crate) const PYTHON_ALNUM_UNICODE_VERSION: &str = "15.0.0"; + +/// Inclusive code point ranges where Python's `str.isalnum()` is true, in order. +pub(crate) const PYTHON_ALNUM_RANGES: &[(u32, u32)] = &[ + (0x0030, 0x0039), + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B2, 0x00B3), + (0x00B5, 0x00B5), + (0x00B9, 0x00BA), + (0x00BC, 0x00BE), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x0660, 0x0669), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07C0, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0966, 0x096F), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09E6, 0x09F1), + (0x09F4, 0x09F9), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A66, 0x0A6F), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AE6, 0x0AEF), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B66, 0x0B6F), + (0x0B71, 0x0B77), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0BE6, 0x0BF2), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C66, 0x0C6F), + (0x0C78, 0x0C7E), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CE6, 0x0CEF), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D58, 0x0D61), + (0x0D66, 0x0D78), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0DE6, 0x0DEF), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E50, 0x0E59), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0ED0, 0x0ED9), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F20, 0x0F33), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x1049), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x1090, 0x1099), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1369, 0x137C), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16EE, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1946, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x19D0, 0x19DA), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B50, 0x1B59), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C40, 0x1C49), + (0x1C4D, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2070, 0x2071), + (0x2074, 0x2079), + (0x207F, 0x2089), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2150, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3007), + (0x3021, 0x3029), + (0x3031, 0x3035), + (0x3038, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x3192, 0x3195), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6EF), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA830, 0xA835), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA900, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9D9), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA50, 0xAA59), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xABF0, 0xABF9), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF10, 0xFF19), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x102E1, 0x102FB), + (0x10300, 0x10323), + (0x1032D, 0x1034A), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x103D1, 0x103D5), + (0x10400, 0x1049D), + (0x104A0, 0x104A9), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10858, 0x10876), + (0x10879, 0x1089E), + (0x108A7, 0x108AF), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x108FB, 0x1091B), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BC, 0x109CF), + (0x109D2, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A40, 0x10A48), + (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10AEB, 0x10AEF), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B58, 0x10B72), + (0x10B78, 0x10B91), + (0x10BA9, 0x10BAF), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10CFA, 0x10D23), + (0x10D30, 0x10D39), + (0x10E60, 0x10E7E), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F27), + (0x10F30, 0x10F45), + (0x10F51, 0x10F54), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11052, 0x1106F), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x110F0, 0x110F9), + (0x11103, 0x11126), + (0x11136, 0x1113F), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111D0, 0x111DA), + (0x111DC, 0x111DC), + (0x111E1, 0x111F4), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x112F0, 0x112F9), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x11450, 0x11459), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x114D0, 0x114D9), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11650, 0x11659), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x116C0, 0x116C9), + (0x11700, 0x1171A), + (0x11730, 0x1173B), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118F2), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x11950, 0x11959), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C50, 0x11C6C), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D50, 0x11D59), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11DA0, 0x11DA9), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11F50, 0x11F59), + (0x11FB0, 0x11FB0), + (0x11FC0, 0x11FD4), + (0x12000, 0x12399), + (0x12400, 0x1246E), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A60, 0x16A69), + (0x16A70, 0x16ABE), + (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E140, 0x1E149), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), + (0x1E4D0, 0x1E4EB), + (0x1E4F0, 0x1E4F9), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +]; diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs new file mode 100644 index 0000000000..54370118d3 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod alnum_ranges; +pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/_helpers/sequence_matcher.rs b/crates/sqlbuild-core/src/text/_helpers/sequence_matcher.rs new file mode 100644 index 0000000000..626b4d5f03 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/sequence_matcher.rs @@ -0,0 +1,171 @@ +//! `difflib.SequenceMatcher` ratios over code points, with `autojunk` and no junk function. + +use std::collections::HashMap; + +const AUTOJUNK_MIN_LENGTH: usize = 200; + +/// One candidate and its `SequenceMatcher.ratio()` against the word. +pub(crate) struct ScoredCandidate<'candidate> { + pub(crate) score: f64, + pub(crate) candidate: &'candidate str, +} + +/// `SequenceMatcher.b2j` of the word, without the popular elements `autojunk` removes. +struct TargetIndex { + positions: HashMap>, +} + +/// Half-open ranges of the candidate and the word that one longest-match search covers. +#[derive(Clone, Copy)] +struct SearchWindow { + source: (usize, usize), + target: (usize, usize), +} + +/// The candidates `get_close_matches` keeps, scored, in input order. +pub(crate) fn scored_candidates<'candidate>( + word: &str, + possibilities: &[&'candidate str], + cutoff: f64, +) -> Vec> { + let target: Vec = word.chars().collect(); + let index: TargetIndex = target_index(&target); + let mut scored: Vec> = Vec::new(); + for candidate in possibilities { + let source: Vec = candidate.chars().collect(); + if real_quick_ratio(&source, &target) >= cutoff + && quick_ratio(&source, &target) >= cutoff + && ratio(&source, &target, &index) >= cutoff + { + scored.push(ScoredCandidate { + score: ratio(&source, &target, &index), + candidate, + }); + } + } + scored +} + +fn target_index(target: &[char]) -> TargetIndex { + let mut positions: HashMap> = HashMap::new(); + for (index, character) in target.iter().enumerate() { + positions.entry(*character).or_default().push(index); + } + if target.len() >= AUTOJUNK_MIN_LENGTH { + let threshold: usize = target.len() / 100 + 1; + positions.retain(|_, indexes| indexes.len() <= threshold); + } + TargetIndex { positions } +} + +fn calculate_ratio(matches: usize, length: usize) -> f64 { + if length == 0 { + return 1.0; + } + 2.0 * matches as f64 / length as f64 +} + +fn real_quick_ratio(source: &[char], target: &[char]) -> f64 { + calculate_ratio(source.len().min(target.len()), source.len() + target.len()) +} + +fn quick_ratio(source: &[char], target: &[char]) -> f64 { + let mut available: HashMap = HashMap::new(); + for character in target { + *available.entry(*character).or_default() += 1; + } + let mut matches: usize = 0; + for character in source { + if let Some(count) = available.get_mut(character) { + if *count > 0 { + matches += 1; + } + *count -= 1; + } + } + calculate_ratio(matches, source.len() + target.len()) +} + +fn ratio(source: &[char], target: &[char], index: &TargetIndex) -> f64 { + let matches: usize = matching_block_sizes(source, target, index); + calculate_ratio(matches, source.len() + target.len()) +} + +fn matching_block_sizes(source: &[char], target: &[char], index: &TargetIndex) -> usize { + let mut total: usize = 0; + let mut queue: Vec = vec![SearchWindow { + source: (0, source.len()), + target: (0, target.len()), + }]; + while let Some(window) = queue.pop() { + let (start, target_start, size) = find_longest_match(source, target, index, window); + if size == 0 { + continue; + } + total += size; + if window.source.0 < start && window.target.0 < target_start { + queue.push(SearchWindow { + source: (window.source.0, start), + target: (window.target.0, target_start), + }); + } + if start + size < window.source.1 && target_start + size < window.target.1 { + queue.push(SearchWindow { + source: (start + size, window.source.1), + target: (target_start + size, window.target.1), + }); + } + } + total +} + +fn find_longest_match( + source: &[char], + target: &[char], + index: &TargetIndex, + window: SearchWindow, +) -> (usize, usize, usize) { + let (source_low, source_high) = window.source; + let (target_low, target_high) = window.target; + let (mut best_source, mut best_target, mut best_size) = (source_low, target_low, 0); + let mut lengths: HashMap = HashMap::new(); + for (position, character) in source.iter().enumerate().take(source_high).skip(source_low) { + let mut next_lengths: HashMap = HashMap::new(); + for &target_position in index.positions.get(character).into_iter().flatten() { + if target_position < target_low { + continue; + } + if target_position >= target_high { + break; + } + let length: usize = target_position + .checked_sub(1) + .and_then(|previous| lengths.get(&previous)) + .copied() + .unwrap_or(0) + + 1; + next_lengths.insert(target_position, length); + if length > best_size { + best_source = position + 1 - length; + best_target = target_position + 1 - length; + best_size = length; + } + } + lengths = next_lengths; + } + while best_source > source_low + && best_target > target_low + && source[best_source - 1] == target[best_target - 1] + { + best_source -= 1; + best_target -= 1; + best_size += 1; + } + while best_source + best_size < source_high + && best_target + best_size < target_high + && source[best_source + best_size] == target[best_target + best_size] + { + best_size += 1; + } + (best_source, best_target, best_size) +} diff --git a/crates/sqlbuild-core/src/text/main/close_matches.rs b/crates/sqlbuild-core/src/text/main/close_matches.rs new file mode 100644 index 0000000000..b646663887 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/close_matches.rs @@ -0,0 +1,19 @@ +//! Python's `difflib.get_close_matches` with the default `SequenceMatcher` settings. + +use crate::text::_helpers::sequence_matcher::{ScoredCandidate, scored_candidates}; + +/// The best `count` candidates whose similarity to `word` reaches `cutoff`, best first. +pub fn close_matches(word: &str, possibilities: &[&str], count: usize, cutoff: f64) -> Vec { + let mut scored: Vec> = scored_candidates(word, possibilities, cutoff); + scored.sort_by(|left, right| { + right + .score + .total_cmp(&left.score) + .then_with(|| right.candidate.cmp(left.candidate)) + }); + scored + .into_iter() + .take(count) + .map(|scored| scored.candidate.to_owned()) + .collect() +} diff --git a/crates/sqlbuild-core/src/text/main/decode_python_text.rs b/crates/sqlbuild-core/src/text/main/decode_python_text.rs index 386c039cff..0e32ea3c0f 100644 --- a/crates/sqlbuild-core/src/text/main/decode_python_text.rs +++ b/crates/sqlbuild-core/src/text/main/decode_python_text.rs @@ -7,5 +7,8 @@ pub fn decode_python_text(bytes: &[u8]) -> Result { let text = std::str::from_utf8(bytes).map_err(|error| TextDecodeError { valid_up_to: error.valid_up_to(), })?; + if !bytes.contains(&b'\r') { + return Ok(text.to_owned()); + } Ok(text.replace("\r\n", "\n").replace('\r', "\n")) } diff --git a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs new file mode 100644 index 0000000000..198ac5632c --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs @@ -0,0 +1,15 @@ +//! Python's `str.isalnum()` for one character, from a table generated by Python. + +use crate::text::_helpers::alnum_ranges::PYTHON_ALNUM_RANGES; + +/// Whether Python's `str.isalnum()` is true for one character. +pub fn is_python_alnum(character: char) -> bool { + if character.is_ascii() { + return character.is_ascii_alphanumeric(); + } + let code_point: u32 = u32::from(character); + let index: usize = PYTHON_ALNUM_RANGES.partition_point(|(_, end)| *end < code_point); + PYTHON_ALNUM_RANGES + .get(index) + .is_some_and(|(start, _)| *start <= code_point) +} diff --git a/crates/sqlbuild-core/src/text/main/is_python_space.rs b/crates/sqlbuild-core/src/text/main/is_python_space.rs new file mode 100644 index 0000000000..983168ad90 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_space.rs @@ -0,0 +1,19 @@ +//! Python's `str.isspace()`, which `\s`, `str.strip()` and `str.split()` use. + +/// Whether Python's `str.isspace()` is true for one character. +pub fn is_python_space(character: char) -> bool { + matches!( + character, + '\t'..='\r' + | '\u{1c}'..='\u{20}' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} diff --git a/crates/sqlbuild-core/src/text/main/is_python_word.rs b/crates/sqlbuild-core/src/text/main/is_python_word.rs new file mode 100644 index 0000000000..f63698521c --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_word.rs @@ -0,0 +1,8 @@ +//! A character of Python's `\w` for `str` patterns. + +use crate::text::main::is_python_alnum::is_python_alnum; + +/// Whether one character matches Python's `\w`: `str.isalnum()` or `_`. +pub fn is_python_word(character: char) -> bool { + character == '_' || is_python_alnum(character) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index 57e2d7ac2e..c78964b1c4 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -1 +1,7 @@ +pub mod close_matches; pub mod decode_python_text; +pub mod is_python_alnum; +pub mod is_python_space; +pub mod is_python_word; +pub mod python_alnum_unicode_version; +pub mod python_strip; diff --git a/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs b/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs new file mode 100644 index 0000000000..f6792f65df --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_alnum_unicode_version.rs @@ -0,0 +1,8 @@ +//! The Unicode version of the native `str.isalnum()` table. + +use crate::text::_helpers::alnum_ranges::PYTHON_ALNUM_UNICODE_VERSION; + +/// The Unicode version of the Python whose `str.isalnum()` the native table reproduces. +pub fn python_alnum_unicode_version() -> &'static str { + PYTHON_ALNUM_UNICODE_VERSION +} diff --git a/crates/sqlbuild-core/src/text/main/python_strip.rs b/crates/sqlbuild-core/src/text/main/python_strip.rs new file mode 100644 index 0000000000..4b2c7d9d77 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_strip.rs @@ -0,0 +1,8 @@ +//! Python's `str.strip()` without arguments. + +use crate::text::main::is_python_space::is_python_space; + +/// The text without leading and trailing Python whitespace. +pub fn python_strip(text: &str) -> &str { + text.trim_matches(is_python_space) +} diff --git a/crates/sqlbuild-core/src/text/mod.rs b/crates/sqlbuild-core/src/text/mod.rs index ed469c77cd..004a682592 100644 --- a/crates/sqlbuild-core/src/text/mod.rs +++ b/crates/sqlbuild-core/src/text/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod _helpers; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-core/src/text/tests.rs b/crates/sqlbuild-core/src/text/tests.rs index 26f62e1540..d3b68f30f0 100644 --- a/crates/sqlbuild-core/src/text/tests.rs +++ b/crates/sqlbuild-core/src/text/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_python_compat.rs"] +mod python_compat; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_text.rs"] diff --git a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs new file mode 100644 index 0000000000..02d4907db5 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs @@ -0,0 +1,161 @@ +use crate::text::main::close_matches::close_matches; +use crate::text::main::is_python_alnum::is_python_alnum; +use crate::text::main::is_python_space::is_python_space; +use crate::text::main::is_python_word::is_python_word; +use crate::text::tests::test_types::{CharacterClassTestCase, CloseMatchesTestCase}; + +const KEYS: &[&str] = &[ + "audits", + "columns", + "constants", + "description", + "enums", + "materialized", + "tags", + "unique_key", +]; + +#[test] +fn given_words_when_finding_close_matches_then_results_match_difflib() { + let test_cases = [ + CloseMatchesTestCase { + description: "one near key", + word: "tagz", + possibilities: KEYS, + count: 1, + cutoff: 0.6, + expected_matches: &["tags"], + }, + CloseMatchesTestCase { + description: "ranked with a low cutoff", + word: "materialised", + possibilities: KEYS, + count: 3, + cutoff: 0.3, + expected_matches: &["materialized", "tags", "description"], + }, + CloseMatchesTestCase { + description: "ranked after a deletion", + word: "colums", + possibilities: KEYS, + count: 3, + cutoff: 0.3, + expected_matches: &["columns", "enums", "constants"], + }, + CloseMatchesTestCase { + description: "no candidate reaches the cutoff", + word: "zzz", + possibilities: KEYS, + count: 1, + cutoff: 0.6, + expected_matches: &[], + }, + CloseMatchesTestCase { + description: "equal scores prefer the larger candidate", + word: "ab", + possibilities: &["ba", "ab", "aB"], + count: 3, + cutoff: 0.0, + expected_matches: &["ab", "ba", "aB"], + }, + CloseMatchesTestCase { + description: "rotations score by their longest blocks", + word: "abc", + possibilities: &["xyz", "cab", "bca"], + count: 3, + cutoff: 0.0, + expected_matches: &["cab", "bca", "xyz"], + }, + ]; + for test_case in test_cases { + assert_eq!( + close_matches( + test_case.word, + test_case.possibilities, + test_case.count, + test_case.cutoff + ), + test_case.expected_matches, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_characters_when_classifying_then_python_str_methods_agree() { + let test_cases = [ + CharacterClassTestCase { + description: "ASCII letter", + character: 'a', + expected_alnum: true, + expected_word: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "underscore is a word character only", + character: '_', + expected_alnum: false, + expected_word: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "Arabic-Indic digit", + character: '\u{663}', + expected_alnum: true, + expected_word: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "letter number", + character: '\u{2167}', + expected_alnum: true, + expected_word: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "combining mark that Rust calls alphabetic", + character: '\u{345}', + expected_alnum: false, + expected_word: false, + expected_space: false, + }, + CharacterClassTestCase { + description: "circled letter symbol", + character: '\u{24b6}', + expected_alnum: false, + expected_word: false, + expected_space: false, + }, + CharacterClassTestCase { + description: "information separator is whitespace", + character: '\u{1c}', + expected_alnum: false, + expected_word: false, + expected_space: true, + }, + CharacterClassTestCase { + description: "ideographic space", + character: '\u{3000}', + expected_alnum: false, + expected_word: false, + expected_space: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + is_python_alnum(test_case.character), + is_python_word(test_case.character), + is_python_space(test_case.character) + ), + ( + test_case.expected_alnum, + test_case.expected_word, + test_case.expected_space + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index 850689e5e1..7ed7c2ac12 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -17,3 +17,20 @@ pub(super) struct DecodeTestCase { pub(super) bytes: &'static [u8], pub(super) expected_text: Result<&'static str, usize>, } + +pub(super) struct CloseMatchesTestCase { + pub(super) description: &'static str, + pub(super) word: &'static str, + pub(super) possibilities: &'static [&'static str], + pub(super) count: usize, + pub(super) cutoff: f64, + pub(super) expected_matches: &'static [&'static str], +} + +pub(super) struct CharacterClassTestCase { + pub(super) description: &'static str, + pub(super) character: char, + pub(super) expected_alnum: bool, + pub(super) expected_word: bool, + pub(super) expected_space: bool, +} diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml new file mode 100644 index 0000000000..44344eb15b --- /dev/null +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sqlbuild-discovery" +version.workspace = true +description = "Native project discovery for SQLBuild: the directory walk, file facts and authored file parsing." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +rayon.workspace = true +sqlbuild-core.workspace = true +sqlbuild-sqltext.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-discovery/src/_helpers/mod.rs b/crates/sqlbuild-discovery/src/_helpers/mod.rs new file mode 100644 index 0000000000..d1c99c888e --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod pool; +pub(crate) mod reading; +pub(crate) mod scoped_paths; +pub(crate) mod suggestions; diff --git a/crates/sqlbuild-discovery/src/_helpers/pool.rs b/crates/sqlbuild-discovery/src/_helpers/pool.rs new file mode 100644 index 0000000000..8748adcf96 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/pool.rs @@ -0,0 +1,20 @@ +//! The worker pool native discovery runs on; header parsing recurses, so stacks are large. + +use rayon::{ThreadPool, ThreadPoolBuilder}; +use std::sync::OnceLock; + +const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; + +static POOL: OnceLock> = OnceLock::new(); + +pub(crate) fn discovery_pool() -> Result<&'static ThreadPool, String> { + POOL.get_or_init(|| { + ThreadPoolBuilder::new() + .stack_size(WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-discovery-{index}")) + .build() + .map_err(|error| error.to_string()) + }) + .as_ref() + .map_err(Clone::clone) +} diff --git a/crates/sqlbuild-discovery/src/_helpers/reading.rs b/crates/sqlbuild-discovery/src/_helpers/reading.rs new file mode 100644 index 0000000000..744d673df4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/reading.rs @@ -0,0 +1,11 @@ +//! Read authored files as Python's `Path.read_text(encoding="utf-8")` does. + +use crate::models::ReadFailure; +use sqlbuild_core::text::main::decode_python_text::decode_python_text; +use std::path::Path; + +/// The decoded text, or why Python's read would raise. +pub(crate) fn read_authored_text(path: &Path) -> Result { + let bytes: Vec = std::fs::read(path).map_err(|error| ReadFailure::Io(error.kind()))?; + decode_python_text(&bytes).map_err(ReadFailure::Decode) +} diff --git a/crates/sqlbuild-discovery/src/_helpers/scoped_paths.rs b/crates/sqlbuild-discovery/src/_helpers/scoped_paths.rs new file mode 100644 index 0000000000..e35b4458b5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/scoped_paths.rs @@ -0,0 +1,39 @@ +//! Whether a file lives in a scoped declaration tree rather than a resource tree. + +use crate::constants::{ + CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, MACRO_TESTS_DIRECTORY, + SCOPED_DECLARATION_DIRECTORIES, SQL_FILE_SUFFIX, SQL_TESTS_ROOT, +}; + +/// Python's `is_in_scoped_declaration_tree` for a `/`-separated project-relative file path. +pub(crate) fn is_in_scoped_declaration_tree(relative_path: &str) -> bool { + let parts: Vec<&str> = relative_path.split('/').collect(); + let Some((file_name, directory)) = parts.split_last() else { + return false; + }; + let sql_file = has_sql_suffix(file_name); + for root in CANONICAL_AUTHORED_ROOTS { + if !directory.starts_with(root) { + continue; + } + let scoped = &directory[root.len()..]; + if root == SQL_TESTS_ROOT && sql_file && scoped.first() == Some(&MACRO_TESTS_DIRECTORY) { + return false; + } + return scoped.iter().any(|component| { + *component == DECLARATION_GROUP_DIRECTORY + || SCOPED_DECLARATION_DIRECTORIES.contains(component) + }); + } + false +} + +/// Python's `Path.suffix == ".sql"`: the last dot starts a suffix unless the name starts with it. +fn has_sql_suffix(file_name: &str) -> bool { + match file_name.rfind('.') { + Some(index) if index > 0 && index + 1 < file_name.len() => { + &file_name[index..] == SQL_FILE_SUFFIX + } + _ => false, + } +} diff --git a/crates/sqlbuild-discovery/src/_helpers/suggestions.rs b/crates/sqlbuild-discovery/src/_helpers/suggestions.rs new file mode 100644 index 0000000000..97c02ba398 --- /dev/null +++ b/crates/sqlbuild-discovery/src/_helpers/suggestions.rs @@ -0,0 +1,33 @@ +//! Python's `unsupported_keys_help`: nearest supported key suggestions. + +use sqlbuild_core::text::main::close_matches::close_matches; + +const SUGGESTION_CUTOFF: f64 = 0.6; +const LISTED_SUPPORTED_KEYS_LIMIT: usize = 12; + +pub(crate) fn unsupported_keys_help(keys: &[&str], supported_keys: &[String]) -> String { + let mut candidates: Vec<&str> = supported_keys.iter().map(String::as_str).collect(); + candidates.sort_unstable(); + candidates.dedup(); + let suggestions: Vec = keys + .iter() + .filter_map(|key| suggestion(key, keys.len(), &candidates)) + .collect(); + if !suggestions.is_empty() { + return format!("did you mean {}?", suggestions.join(", ")); + } + if candidates.len() <= LISTED_SUPPORTED_KEYS_LIMIT { + return format!("supported keys: {}", candidates.join(", ")); + } + "remove the key; see the reference for supported keys".to_owned() +} + +fn suggestion(key: &str, key_count: usize, candidates: &[&str]) -> Option { + let matched: String = close_matches(key, candidates, 1, SUGGESTION_CUTOFF) + .into_iter() + .next()?; + if key_count == 1 { + return Some(format!("'{matched}'")); + } + Some(format!("'{matched}' for '{key}'")) +} diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs new file mode 100644 index 0000000000..b4828a179a --- /dev/null +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -0,0 +1,29 @@ +//! Project layout names shared by native discovery, mirroring the Python discovery constants. + +/// Root of SQL model files. +pub const MODELS_DIRECTORY: &str = "models"; +/// Suffix of SQL files. +pub const SQL_FILE_SUFFIX: &str = ".sql"; +/// The folder below a resource tree that holds grouped scoped declarations. +pub const DECLARATION_GROUP_DIRECTORY: &str = "_sqlbuild"; +/// Inherited and local declaration directories whose files are never resources. +pub const SCOPED_DECLARATION_DIRECTORIES: [&str; 6] = [ + "macros", + "enums", + "constants", + "_macros", + "_enums", + "_constants", +]; +/// Resource trees that may hold scoped declarations, in Python's lookup order. +pub const CANONICAL_AUTHORED_ROOTS: [&[&str]; 5] = [ + &["models"], + &["tests", "unit"], + &["tests", "scenarios"], + &["functions", "sql"], + &["sources"], +]; +/// The SQL test root, whose `macros/` SQL files are tests rather than declarations. +pub const SQL_TESTS_ROOT: &[&str] = &["tests", "unit"]; +/// The directory of macro tests below the SQL test root. +pub const MACRO_TESTS_DIRECTORY: &str = "macros"; diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs new file mode 100644 index 0000000000..11e9b34f54 --- /dev/null +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -0,0 +1,9 @@ +//! Native project discovery: the directory walk, file facts and parsing of authored files. + +#![forbid(unsafe_code)] + +pub(crate) mod _helpers; +pub mod constants; +pub mod model_files; +pub mod models; +pub mod tree; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs new file mode 100644 index 0000000000..4623434102 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/header_keys.rs @@ -0,0 +1,122 @@ +//! Python's `reject_unsupported_header_keys`: the message, its line and the key suggestions. + +use crate::_helpers::suggestions::unsupported_keys_help; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; + +/// The facts of one header that declares keys outside its supported set. +pub(crate) struct UnsupportedKeys<'a> { + pub(crate) statement: &'a str, + pub(crate) file_path: &'a str, + pub(crate) header: &'a str, + /// One-based line of the header start. + pub(crate) header_line: usize, + pub(crate) keys: &'a [&'a str], + pub(crate) supported_keys: &'a [String], +} + +pub(crate) fn unsupported_keys_failure(facts: &UnsupportedKeys<'_>) -> DiscoveryFailure { + let UnsupportedKeys { + statement, + file_path, + header, + header_line, + keys, + supported_keys, + } = facts; + let header_chars: Vec = header.chars().collect(); + let key_offset: usize = header_key_offset(&header_chars, keys[0]); + let line: usize = header_line + + header_chars[..key_offset] + .iter() + .filter(|character| **character == '\n') + .count(); + DiscoveryFailure { + kind: FailureKind::ModelSql, + message: format!( + "{statement} in '{file_path}:{line}' has unsupported keys: {}", + keys.join(", ") + ), + help: Some(unsupported_keys_help(keys, supported_keys)), + } +} + +/// `re.search(r"(?:^|[(,])(?:\s|--[^\n]*(?:\n|$)|/\*.*?\*/)*(KEY)\b", header, re.S).start(1)`. +fn header_key_offset(header: &[char], key: &str) -> usize { + let key: Vec = key.chars().collect(); + let length: usize = header.len(); + let mut found: Vec> = vec![None; length + 1]; + for position in (0..=length).rev() { + found[position] = skip_then_key(header, &key, position, &found); + } + for start in 0..=length { + if start == 0 + && let Some(offset) = found[0] + { + return offset; + } + if start < length + && matches!(header[start], '(' | ',') + && let Some(offset) = found[start + 1] + { + return offset; + } + } + 0 +} + +/// Where the key starts when the skip loop is entered at `position`, in backtracking order. +fn skip_then_key( + header: &[char], + key: &[char], + position: usize, + found: &[Option], +) -> Option { + let length = header.len(); + if position < length { + if is_python_space(header[position]) + && let Some(offset) = found[position + 1] + { + return Some(offset); + } + if header[position..].starts_with(&['-', '-']) { + let end = header[position + 2..] + .iter() + .position(|character| *character == '\n') + .map_or(length, |offset| position + 2 + offset + 1); + if let Some(offset) = found[end] { + return Some(offset); + } + } + if header[position..].starts_with(&['/', '*']) { + let mut close = position + 2; + while close + 1 < length { + if header[close] == '*' + && header[close + 1] == '/' + && let Some(offset) = found[close + 2] + { + return Some(offset); + } + close += 1; + } + } + } + let end = position + key.len(); + if header.get(position..end) == Some(key) && word_boundary(header, end) { + return Some(position); + } + None +} + +/// `\b` at `position`: exactly one side is a word character. +fn word_boundary(text: &[char], position: usize) -> bool { + let before = position + .checked_sub(1) + .and_then(|index| text.get(index)) + .is_some_and(|character| is_python_word(*character)); + let after = text + .get(position) + .is_some_and(|character| is_python_word(*character)); + before != after +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs new file mode 100644 index 0000000000..e74876e9b0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/locations.rs @@ -0,0 +1,96 @@ +//! Authored locations as Python computes them: code-point columns over `\n`-separated lines. + +use crate::models::LineColumnSpan; + +/// Python's `header_column_locations` for the native tokenizer's column-key offsets. +pub(crate) fn header_column_locations( + contents: &str, + (header_start, header_end): (usize, usize), + offsets: &[(String, usize, usize)], +) -> Vec<(String, LineColumnSpan)> { + if offsets.is_empty() { + return Vec::new(); + } + let before_header = &contents[..header_start]; + let header_line = before_header.matches('\n').count() + 1; + let line_start = before_header.rfind('\n').map_or(0, |index| index + 1); + let header_column = before_header[line_start..].chars().count() + 1; + let header: Vec = contents[header_start..header_end].chars().collect(); + relative_locations(&header, offsets) + .into_iter() + .map(|(name, relative_line, relative_column, length)| { + let line = header_line + relative_line - 1; + let column = relative_column + + if relative_line == 1 { + header_column - 1 + } else { + 0 + }; + ( + name, + LineColumnSpan { + line, + column, + end_line: line, + end_column: column + length, + }, + ) + }) + .collect() +} + +/// Python's `_header_column_relative_locations`, including its forward-only cursor. +fn relative_locations( + header: &[char], + offsets: &[(String, usize, usize)], +) -> Vec<(String, usize, usize, usize)> { + let mut locations = Vec::with_capacity(offsets.len()); + let (mut line, mut column, mut cursor) = (1, 1, 0); + for (name, position, length) in offsets { + while cursor < *position { + let newline = header[cursor..*position.min(&header.len())] + .iter() + .position(|character| *character == '\n'); + match newline { + None => { + column += position - cursor; + cursor = *position; + } + Some(offset) => { + line += 1; + column = 1; + cursor += offset + 1; + } + } + } + locations.push((name.clone(), line, column, *length)); + } + locations +} + +/// Code-point offsets where each line starts. +pub(crate) fn line_starts(contents: &[char]) -> Vec { + let mut starts = vec![0]; + starts.extend( + contents + .iter() + .enumerate() + .filter(|(_, character)| **character == '\n') + .map(|(index, _)| index + 1), + ); + starts +} + +/// Python's `_location_for_absolute_span` over code-point offsets. +pub(crate) fn absolute_span(start: usize, end: usize, starts: &[usize]) -> LineColumnSpan { + let end = end.max(start); + let end_position = start.max(end.saturating_sub(1)); + let line_index = starts.partition_point(|line_start| *line_start <= start) - 1; + let end_line_index = starts.partition_point(|line_start| *line_start <= end_position) - 1; + LineColumnSpan { + line: line_index + 1, + column: start - starts[line_index] + 1, + end_line: end_line_index + 1, + end_column: end_position - starts[end_line_index] + 2, + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs new file mode 100644 index 0000000000..de1cf65fb0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod header_keys; +pub(crate) mod locations; +pub(crate) mod output_columns; +pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs new file mode 100644 index 0000000000..fe0ad8c2d9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/output_columns.rs @@ -0,0 +1,335 @@ +//! Python's lexical scan of the top-level SELECT list for authored output column locations. + +use crate::model_files::_helpers::locations::{absolute_span, line_starts}; +use crate::models::LineColumnSpan; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; + +const SELECT_KEYWORD: &str = "SELECT"; +const UNION_KEYWORD: &str = "UNION"; +const FROM_KEYWORD: &str = "FROM"; +const ESCAPE: char = '\\'; + +/// Python's `matched_model_output_column_locations` from the SQL start (a byte offset). +pub(crate) fn output_column_locations( + contents: &str, + sql_start: usize, + extract_implicit_alias_columns: bool, +) -> Vec<(String, LineColumnSpan)> { + let characters: Vec = contents.chars().collect(); + let sql_offset = contents[..sql_start].chars().count(); + let sql = &characters[sql_offset..]; + let Some((list_start, list_end)) = top_level_select_list_bounds(sql) else { + return Vec::new(); + }; + let starts = line_starts(&characters); + let mut locations: Vec<(String, LineColumnSpan)> = Vec::new(); + for (item_start, item_end) in split_top_level_items(sql, list_start, list_end) { + let Some(name) = + item_output_name(&sql[item_start..item_end], extract_implicit_alias_columns) + else { + continue; + }; + let mut start = item_start; + while start < item_end && is_python_space(sql[start]) { + start += 1; + } + let mut end = item_end; + while end > start && is_python_space(sql[end - 1]) { + end -= 1; + } + if start >= end { + continue; + } + locations.push(( + name, + absolute_span(sql_offset + start, sql_offset + end, &starts), + )); + } + locations +} + +fn is_scan_special(character: char) -> bool { + matches!( + character, + '\'' | '"' | '`' | '(' | ')' | 's' | 'S' | 'f' | 'F' | 'u' | 'U' + ) +} + +fn top_level_select_list_bounds(sql: &[char]) -> Option<(usize, usize)> { + let length = sql.len(); + let has_union_candidate = contains_lowercase_union(sql); + let (mut depth, mut index) = (0_usize, 0_usize); + let (mut list_start, mut list_end): (Option, Option) = (None, None); + let mut in_quote: Option = None; + while index < length { + if in_quote.is_none() { + match sql[index..] + .iter() + .position(|character| is_scan_special(*character)) + { + Some(offset) => index += offset, + None => break, + } + } + let character = sql[index]; + if let Some(quote) = in_quote { + if character == ESCAPE { + index += 2; + continue; + } + if character == quote { + in_quote = None; + } + index += 1; + continue; + } + if matches!(character, '\'' | '"') { + in_quote = Some(character); + index += 1; + continue; + } + if character == '(' { + depth += 1; + index += 1; + continue; + } + if character == ')' { + depth = depth.saturating_sub(1); + index += 1; + continue; + } + if depth != 0 { + index += 1; + continue; + } + let upper = character.to_ascii_uppercase(); + match list_start { + None => { + if upper == 'S' && keyword_at(sql, SELECT_KEYWORD, index) { + index += SELECT_KEYWORD.len(); + list_start = Some(index); + continue; + } + } + Some(start) => { + if upper == 'U' && keyword_at(sql, UNION_KEYWORD, index) { + return None; + } + if list_end.is_none() && upper == 'F' && keyword_at(sql, FROM_KEYWORD, index) { + list_end = Some(index); + if !has_union_candidate { + return Some((start, index)); + } + } + } + } + index += 1; + } + list_start.map(|start| (start, list_end.unwrap_or(length))) +} + +/// Python's `"union" in sql.lower()`; only `İ` and the Kelvin sign lower to ASCII letters. +fn contains_lowercase_union(sql: &[char]) -> bool { + let mut lowered: Vec = Vec::with_capacity(sql.len()); + for character in sql { + match character { + '\u{130}' => lowered.extend(['i', '\u{307}']), + '\u{212a}' => lowered.push('k'), + other if other.is_ascii() => lowered.push(other.to_ascii_lowercase()), + _ => lowered.push('\u{fffd}'), + } + } + lowered + .windows(5) + .any(|window| window == ['u', 'n', 'i', 'o', 'n']) +} + +/// Python's `str.upper()` of one character when that is a single ASCII character. +fn ascii_upper(character: char) -> Option { + match character { + '\u{131}' => Some('I'), + '\u{17f}' => Some('S'), + other if other.is_ascii() => Some(other.to_ascii_uppercase()), + _ => None, + } +} + +fn keyword_at(sql: &[char], keyword: &str, index: usize) -> bool { + let end = index + keyword.len(); + let slice = &sql[index..end.min(sql.len())]; + if slice.len() != keyword.len() + || !slice + .iter() + .zip(keyword.chars()) + .all(|(character, expected)| ascii_upper(*character) == Some(expected)) + { + return false; + } + let before = index.checked_sub(1).map_or(' ', |previous| sql[previous]); + let after = sql.get(end).copied().unwrap_or(' '); + !is_python_word(before) && !is_python_word(after) +} + +fn split_top_level_items(sql: &[char], start: usize, end: usize) -> Vec<(usize, usize)> { + let mut items: Vec<(usize, usize)> = Vec::new(); + let mut depth = 0_usize; + let mut item_start = start; + let mut index = start; + let mut in_quote: Option = None; + while index < end { + let character = sql[index]; + if let Some(quote) = in_quote { + if character == ESCAPE { + index += 2; + continue; + } + if character == quote { + in_quote = None; + } + index += 1; + continue; + } + match character { + '\'' | '"' => in_quote = Some(character), + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ',' if depth == 0 => { + items.push((item_start, index)); + item_start = index + 1; + } + _ => {} + } + index += 1; + } + items.push((item_start, end)); + items +} + +/// Python's `_select_item_output_name`. +fn item_output_name(item: &[char], extract_implicit_alias_columns: bool) -> Option { + if let Some(name) = explicit_alias(item) { + return Some(name); + } + if let Some(name) = bare_reference(item) { + return Some(name); + } + if extract_implicit_alias_columns { + return implicit_alias(item); + } + None +} + +/// `[A-Za-z_]`, which also matches `İ`, `ı`, `ſ` and the Kelvin sign under `re.IGNORECASE`. +fn is_identifier_start(character: char, ignore_case: bool) -> bool { + character.is_ascii_alphabetic() + || character == '_' + || (ignore_case && matches!(character, '\u{130}' | '\u{131}' | '\u{17f}' | '\u{212a}')) +} + +fn is_identifier_continue(character: char, ignore_case: bool) -> bool { + is_identifier_start(character, ignore_case) || character.is_ascii_digit() +} + +/// `(?P[A-Za-z_][A-Za-z0-9_]*|"[^"]+")` at `start`: the end of the name. +fn name_end(item: &[char], start: usize, ignore_case: bool) -> Option { + let first = *item.get(start)?; + if is_identifier_start(first, ignore_case) { + let mut end = start + 1; + while end < item.len() && is_identifier_continue(item[end], ignore_case) { + end += 1; + } + return Some(end); + } + if first == '"' + && item + .get(start + 1) + .is_some_and(|character| *character != '"') + { + let close = item[start + 1..] + .iter() + .position(|character| *character == '"')?; + return Some(start + 1 + close + 1); + } + None +} + +fn rest_is_space(item: &[char], start: usize) -> bool { + item[start..] + .iter() + .all(|character| is_python_space(*character)) +} + +fn space_run_end(item: &[char], start: usize) -> usize { + let mut end = start; + while end < item.len() && is_python_space(item[end]) { + end += 1; + } + end +} + +fn captured_name(item: &[char], start: usize, end: usize) -> String { + item[start..end] + .iter() + .collect::() + .trim_matches('"') + .to_owned() +} + +/// `re.search(r"\s+AS\s+NAME\s*\Z", item, re.IGNORECASE)`. +fn explicit_alias(item: &[char]) -> Option { + for start in 0..item.len() { + if !is_python_space(item[start]) || (start > 0 && is_python_space(item[start - 1])) { + continue; + } + let keyword = space_run_end(item, start); + if !matches!(item.get(keyword), Some('a' | 'A')) + || !matches!(item.get(keyword + 1), Some('s' | 'S' | '\u{17f}')) + { + continue; + } + let name_start = space_run_end(item, keyword + 2); + if name_start == keyword + 2 { + continue; + } + if let Some(end) = name_end(item, name_start, true) + && rest_is_space(item, end) + { + return Some(captured_name(item, name_start, end)); + } + } + None +} + +/// `re.match(r"\s*(?:(?:IDENT|QUOTED)\.)?NAME\s*\Z", item)`. +fn bare_reference(item: &[char]) -> Option { + let start = space_run_end(item, 0); + if let Some(qualifier_end) = name_end(item, start, false) + && item.get(qualifier_end) == Some(&'.') + && let Some(end) = name_end(item, qualifier_end + 1, false) + && rest_is_space(item, end) + { + return Some(captured_name(item, qualifier_end + 1, end)); + } + let end = name_end(item, start, false)?; + rest_is_space(item, end).then(|| captured_name(item, start, end)) +} + +/// `re.search(r"\)\s+NAME\s*\Z", item)`. +fn implicit_alias(item: &[char]) -> Option { + for (index, character) in item.iter().enumerate() { + if *character != ')' { + continue; + } + let name_start = space_run_end(item, index + 1); + if name_start == index + 1 { + continue; + } + if let Some(end) = name_end(item, name_start, false) + && rest_is_space(item, end) + { + return Some(captured_name(item, name_start, end)); + } + } + None +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs new file mode 100644 index 0000000000..c23cd28716 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -0,0 +1,99 @@ +//! Parse one model file with the checks and messages of Python's `parse_matched_model_sql`. + +use crate::model_files::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; +use crate::model_files::_helpers::locations::header_column_locations; +use crate::model_files::_helpers::output_columns::output_column_locations; +use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_sqltext::compiler::main::model_header_byte_matching::match_one_bytes; +use sqlbuild_sqltext::compiler::main::model_header_single_parsing::parse_one; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +const MODEL_STATEMENT: &str = "MODEL()"; + +fn failure(message: String) -> DiscoveryFailure { + DiscoveryFailure::new(FailureKind::ModelSql, message) +} + +pub(crate) fn parse_model_file( + file_path: &str, + contents: String, + options: &ModelFileOptions, +) -> Result { + let Some((header_start, header_end, sql_start)) = match_one_bytes(&contents) else { + return Err(failure(format!( + "SQL model '{file_path}' must start with a MODEL(...) header as the first \ + non-whitespace content" + ))); + }; + let header = &contents[header_start..header_end]; + let (header_values, column_offsets) = match parse_one(header) { + (_, _, Some(error)) => return Err(syntax_failure(file_path, &error)), + (Some(AuthoredValue::Map(values)), Some(offsets), None) => (values, offsets), + _ => { + return Err(syntax_failure( + file_path, + "Native MODEL header parser returned neither values nor an error", + )); + } + }; + let mut removed: Vec<&str> = header_keys(&header_values) + .filter(|key| contains(&options.removed_keys, key)) + .collect(); + removed.sort_unstable(); + if !removed.is_empty() { + return Err(failure(format!( + "MODEL() option(s) {} in '{file_path}' were removed with virtual environments; \ + projects run in direct mode", + removed.join(", ") + ))); + } + let unsupported: Vec<&str> = header_keys(&header_values) + .filter(|key| !contains(&options.supported_keys, key)) + .collect(); + if !unsupported.is_empty() { + return Err(unsupported_keys_failure(&UnsupportedKeys { + statement: MODEL_STATEMENT, + file_path, + header, + header_line: contents[..header_start].matches('\n').count() + 1, + keys: &unsupported, + supported_keys: &options.supported_keys, + })); + } + let query_sql: String = python_strip(&contents[sql_start..]).to_owned(); + if query_sql.is_empty() { + return Err(failure(format!( + "SQL model '{file_path}' must contain SQL after MODEL(...)" + ))); + } + let header_column_locations = + header_column_locations(&contents, (header_start, header_end), &column_offsets); + let output_column_locations = if options.extract_output_column_locations { + output_column_locations(&contents, sql_start, options.extract_implicit_alias_columns) + } else { + Vec::new() + }; + Ok(DiscoveredModelFile { + contents, + header_values, + header_column_locations, + output_column_locations, + query_sql, + }) +} + +fn syntax_failure(file_path: &str, error: &str) -> DiscoveryFailure { + failure(format!( + "MODEL(...) in '{file_path}' contains invalid SQLBuild header syntax: {error}" + )) +} + +fn header_keys(values: &[(String, AuthoredValue)]) -> impl Iterator { + values.iter().map(|(key, _)| key.as_str()) +} + +fn contains(keys: &[String], key: &str) -> bool { + keys.iter().any(|candidate| candidate == key) +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs new file mode 100644 index 0000000000..0255001251 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs @@ -0,0 +1,49 @@ +//! Discover, read and parse the SQL model files under `models/`. + +use crate::_helpers::pool::discovery_pool; +use crate::_helpers::reading::read_authored_text; +use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; +use crate::constants::{MODELS_DIRECTORY, SQL_FILE_SUFFIX}; +use crate::model_files::_helpers::parsing::parse_model_file; +use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageDeferral}; +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +/// Every model file in Python's discovery order, each parsed or carrying its first failure. +pub fn discover_model_files( + root: &ProjectRoot, + tree: &ProjectTree, + options: &ModelFileOptions, +) -> Result>, StageDeferral> { + let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + pool.install(|| { + let paths: Vec = rglob(tree, MODELS_DIRECTORY, |entry| { + entry.name.ends_with(SQL_FILE_SUFFIX) + })?; + Ok(paths + .into_par_iter() + .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) + .map(|relative_path| DiscoveredFile { + outcome: discover_one(root, tree, &relative_path, options), + relative_path, + }) + .collect()) + }) +} + +fn discover_one( + root: &ProjectRoot, + tree: &ProjectTree, + relative_path: &str, + options: &ModelFileOptions, +) -> FileOutcome { + let Ok(contents) = read_authored_text(&tree.absolute(relative_path)) else { + return FileOutcome::Unreadable; + }; + match parse_model_file(&root.display_path(relative_path), contents, options) { + Ok(model_file) => FileOutcome::Parsed(model_file), + Err(failure) => FileOutcome::Failed(failure), + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs new file mode 100644 index 0000000000..4d9ade76d4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -0,0 +1 @@ +pub mod discover_model_files; diff --git a/crates/sqlbuild-discovery/src/model_files/mod.rs b/crates/sqlbuild-discovery/src/model_files/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs new file mode 100644 index 0000000000..f23ad90701 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -0,0 +1,24 @@ +//! Native results for SQL model files. + +use crate::models::LineColumnSpan; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +/// What the Python facade asked for, with the header key sets Python owns during dual running. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelFileOptions { + pub supported_keys: Vec, + pub removed_keys: Vec, + pub extract_implicit_alias_columns: bool, + pub extract_output_column_locations: bool, +} + +/// One parsed model file; values mirror the fields of Python's `DiscoveredSqlModelFile`. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredModelFile { + pub contents: String, + pub header_values: Vec<(String, AuthoredValue)>, + pub header_column_locations: Vec<(String, LineColumnSpan)>, + /// In Python's assignment order; a repeated name keeps its first position and last value. + pub output_column_locations: Vec<(String, LineColumnSpan)>, + pub query_sql: String, +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests.rs b/crates/sqlbuild-discovery/src/model_files/tests.rs new file mode 100644 index 0000000000..ab952d899c --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_discover_model_files.rs"] +mod discover_model_files; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs new file mode 100644 index 0000000000..aa5ad43815 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -0,0 +1,75 @@ +use crate::model_files::_helpers::parsing::parse_model_file; +use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; +use crate::models::LineColumnSpan; + +const FILE_PATH: &str = "/project/models/orders.sql"; + +fn options() -> ModelFileOptions { + ModelFileOptions { + supported_keys: [ + "audits", + "columns", + "constants", + "description", + "enums", + "materialized", + "tags", + "unique_key", + ] + .into_iter() + .map(str::to_owned) + .collect(), + removed_keys: vec!["run_despite_unchanged".to_owned()], + extract_implicit_alias_columns: true, + extract_output_column_locations: true, + } +} + +fn located(locations: &[(String, LineColumnSpan)]) -> Vec { + locations + .iter() + .map(|(name, span)| { + ( + name.clone(), + (span.line, span.column, span.end_line, span.end_column), + ) + }) + .collect() +} + +fn summary(model: &DiscoveredModelFile) -> (String, Vec, Vec) { + ( + model.query_sql.clone(), + located(&model.header_column_locations), + located(&model.output_column_locations), + ) +} + +/// Parse one model file as `/project/models/orders.sql` and summarise the outcome. +pub(super) fn parsed_summary(contents: &str) -> ModelSummary { + parse_model_file(FILE_PATH, contents.to_owned(), &options()) + .map(|model| summary(&model)) + .map_err(|failure| (failure.message, failure.help)) +} + +/// The expected successful summary. +pub(super) fn parsed( + query: &str, + header: &[(&str, Span)], + output: &[(&str, Span)], +) -> ModelSummary { + Ok((query.to_owned(), owned(header), owned(output))) +} + +fn owned(locations: &[(&str, Span)]) -> Vec { + locations + .iter() + .map(|(name, span)| ((*name).to_owned(), *span)) + .collect() +} + +/// The expected failure summary. +pub(super) fn failed(message: &str, help: Option<&str>) -> ModelSummary { + Err((message.to_owned(), help.map(str::to_owned))) +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_discover_model_files.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_discover_model_files.rs new file mode 100644 index 0000000000..56e30c12ed --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_discover_model_files.rs @@ -0,0 +1,119 @@ +use crate::model_files::tests::helpers::{failed, parsed, parsed_summary}; +use crate::model_files::tests::test_types::ModelFileTestCase; + +const SUPPORTED_KEYS_HELP: &str = "supported keys: audits, columns, constants, description, enums, materialized, tags, unique_key"; + +#[test] +fn given_model_files_when_parsing_then_values_locations_and_failures_match_python() { + let test_cases = [ + ModelFileTestCase { + description: "columns and simple projections", + contents: "MODEL (\n columns (\n id (type INT),\n ),\n);\nSELECT o.id, total AS amount, sum(x) s FROM t", + expected_summary: parsed( + "SELECT o.id, total AS amount, sum(x) s FROM t", + &[("id", (3, 5, 3, 7))], + &[ + ("id", (6, 8, 6, 12)), + ("amount", (6, 14, 6, 29)), + ("s", (6, 31, 6, 39)), + ], + ), + }, + ModelFileTestCase { + description: "non-ASCII before the header shifts code-point columns", + contents: "MODEL (description 'é', columns (id (type INT)));\nSELECT \"Ünit\" FROM t", + expected_summary: parsed( + "SELECT \"Ünit\" FROM t", + &[("id", (1, 34, 1, 36))], + &[("Ünit", (2, 8, 2, 14))], + ), + }, + ModelFileTestCase { + description: "a UNION hides the projection list", + contents: "MODEL ();\nSELECT a FROM t UNION ALL SELECT a FROM u", + expected_summary: parsed("SELECT a FROM t UNION ALL SELECT a FROM u", &[], &[]), + }, + ModelFileTestCase { + description: "dotless i spells UNION for Python's upper()", + contents: "MODEL ();\nSELECT a FROM t unıon SELECT a FROM u", + expected_summary: parsed( + "SELECT a FROM t unıon SELECT a FROM u", + &[], + &[("a", (2, 8, 2, 9))], + ), + }, + ModelFileTestCase { + description: "missing header", + contents: "SELECT 1", + expected_summary: failed( + "SQL model '/project/models/orders.sql' must start with a MODEL(...) header as the first non-whitespace content", + None, + ), + }, + ModelFileTestCase { + description: "byte order mark is not whitespace", + contents: "\u{feff}MODEL ();\nSELECT 1", + expected_summary: failed( + "SQL model '/project/models/orders.sql' must start with a MODEL(...) header as the first non-whitespace content", + None, + ), + }, + ModelFileTestCase { + description: "header syntax", + contents: "MODEL (materialized: table);\nSELECT 1", + expected_summary: failed( + "MODEL(...) in '/project/models/orders.sql' contains invalid SQLBuild header syntax: unexpected ':' after key 'materialized'; use SQLBuild syntax 'materialized value'", + None, + ), + }, + ModelFileTestCase { + description: "removed key", + contents: "MODEL (run_despite_unchanged true, bogus 1);\nSELECT 1", + expected_summary: failed( + "MODEL() option(s) run_despite_unchanged in '/project/models/orders.sql' were removed with virtual environments; projects run in direct mode", + None, + ), + }, + ModelFileTestCase { + description: "unsupported key on a later line with a suggestion", + contents: "MODEL (\n materialized table,\n /* note */ tagz [a],\n);\nSELECT 1", + expected_summary: failed( + "MODEL() in '/project/models/orders.sql:3' has unsupported keys: tagz", + Some("did you mean 'tags'?"), + ), + }, + ModelFileTestCase { + description: "unsupported key after a line comment", + contents: "MODEL (materialized table, -- c\n col_x 1);\nSELECT 1", + expected_summary: failed( + "MODEL() in '/project/models/orders.sql:2' has unsupported keys: col_x", + Some(SUPPORTED_KEYS_HELP), + ), + }, + ModelFileTestCase { + description: "two unsupported keys without suggestions", + contents: "MODEL (zzz 1, qqq 2);\nSELECT 1", + expected_summary: failed( + "MODEL() in '/project/models/orders.sql:1' has unsupported keys: zzz, qqq", + Some(SUPPORTED_KEYS_HELP), + ), + }, + ModelFileTestCase { + description: "empty query", + contents: "MODEL ();\n \n", + expected_summary: failed( + "SQL model '/project/models/orders.sql' must contain SQL after MODEL(...)", + None, + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + parsed_summary(test_case.contents), + test_case.expected_summary, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs new file mode 100644 index 0000000000..ab03edff85 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs @@ -0,0 +1,15 @@ +/// Line, column, end line and end column. +pub(super) type Span = (usize, usize, usize, usize); + +/// A located name. +pub(super) type Located = (String, Span); + +/// The parsed query and locations, or the failure message and help. +pub(super) type ModelSummary = + Result<(String, Vec, Vec), (String, Option)>; + +pub(super) struct ModelFileTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_summary: ModelSummary, +} diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs new file mode 100644 index 0000000000..9a933a7b46 --- /dev/null +++ b/crates/sqlbuild-discovery/src/models.rs @@ -0,0 +1,91 @@ +//! Results native discovery hands to the Python facade. + +use sqlbuild_core::text::errors::TextDecodeError; +use std::path::PathBuf; + +/// The Python `DiscoveryError` subclass a native failure is raised as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FailureKind { + /// `ModelSqlParseError` (D002). + ModelSql, +} + +impl FailureKind { + /// The stable name the facade maps to an exception class. + pub fn as_str(self) -> &'static str { + match self { + Self::ModelSql => "model_sql", + } + } +} + +/// One discovery failure with Python's exact message and optional help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiscoveryFailure { + pub kind: FailureKind, + pub message: String, + pub help: Option, +} + +impl DiscoveryFailure { + /// A failure without help text. + pub fn new(kind: FailureKind, message: String) -> Self { + Self { + kind, + message, + help: None, + } + } +} + +/// Native discovery cannot reproduce Python for this project; the Python stage runs instead. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StageDeferral { + pub reason: String, +} + +/// The outcome of reading and parsing one authored file. +#[derive(Clone, Debug, PartialEq)] +pub enum FileOutcome { + Parsed(T), + Failed(DiscoveryFailure), + /// The bytes could not be read or decoded; Python re-reads the file to raise its error. + Unreadable, +} + +/// One discovered file, by `/`-separated path relative to the project directory. +#[derive(Clone, Debug, PartialEq)] +pub struct DiscoveredFile { + pub relative_path: String, + pub outcome: FileOutcome, +} + +/// A one-based line and code-point column range, as Python's `SourceLocation` stores it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LineColumnSpan { + pub line: usize, + pub column: usize, + pub end_line: usize, + pub end_column: usize, +} + +/// The project directory and the text Python prints before a relative path (`str(project_dir / x)`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectRoot { + pub directory: PathBuf, + pub display_prefix: String, +} + +impl ProjectRoot { + /// Python's `str(project_dir / relative_path)` for a `/`-separated relative path. + pub fn display_path(&self, relative_path: &str) -> String { + format!("{}{relative_path}", self.display_prefix) + } +} + +/// Why an authored file could not be read the way Python's `read_text` reads it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReadFailure { + Io(std::io::ErrorKind), + Decode(TextDecodeError), +} diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs new file mode 100644 index 0000000000..5946236045 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs @@ -0,0 +1,56 @@ +//! One directory listing with the entry type facts Python's `os.scandir` reports. + +use crate::models::StageDeferral; +use crate::tree::models::TreeEntry; +use std::fs::DirEntry; +use std::path::Path; + +/// List a directory; an unreadable directory lists as empty, as in Python's snapshot. +pub(crate) fn list_directory(directory: &Path) -> Result, StageDeferral> { + let Ok(entries) = std::fs::read_dir(directory) else { + return Ok(Vec::new()); + }; + let mut listing: Vec = Vec::new(); + for entry in entries { + let Ok(entry) = entry else { + return Ok(Vec::new()); + }; + listing.push(tree_entry(&entry)?); + } + Ok(listing) +} + +fn tree_entry(entry: &DirEntry) -> Result { + let name = entry + .file_name() + .into_string() + .map_err(|name| StageDeferral { + reason: format!("file name is not UTF-8: {}", name.to_string_lossy()), + })?; + let (is_dir, is_walkable_dir) = match entry.file_type() { + Ok(file_type) if file_type.is_symlink() => (link_target_is_dir(entry), false), + Ok(file_type) => (file_type.is_dir(), file_type.is_dir()), + Err(_unreadable) => (false, false), + }; + Ok(TreeEntry { + name, + is_dir, + is_walkable_dir, + }) +} + +fn link_target_is_dir(entry: &DirEntry) -> bool { + match std::fs::metadata(entry.path()) { + Ok(metadata) => metadata.is_dir(), + Err(_dangling) => false, + } +} + +/// Join a `/`-separated relative directory and an entry name. +pub(crate) fn join_relative(directory: &str, name: &str) -> String { + if directory.is_empty() { + name.to_owned() + } else { + format!("{directory}/{name}") + } +} diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs b/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs new file mode 100644 index 0000000000..c484bf675a --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod listing; +pub(crate) mod ordering; +pub(crate) mod walking; diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs b/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs new file mode 100644 index 0000000000..defef4fcc7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs @@ -0,0 +1,7 @@ +//! Python's `sorted()` order of paths: part by part, each part by code point. + +use std::cmp::Ordering; + +pub(crate) fn compare_relative_paths(left: &str, right: &str) -> Ordering { + left.split('/').cmp(right.split('/')) +} diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs new file mode 100644 index 0000000000..7fba2181b3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs @@ -0,0 +1,55 @@ +//! The parallel walk below one root, listing each directory once per snapshot. + +use crate::models::StageDeferral; +use crate::tree::_helpers::listing::{join_relative, list_directory}; +use crate::tree::models::{ProjectTree, TreeEntry}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use std::sync::Arc; + +pub(crate) type WalkedDirectory = (String, Arc>); + +/// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. +pub(crate) fn is_dir(tree: &ProjectTree, relative_path: &str) -> bool { + match std::fs::metadata(tree.absolute(relative_path)) { + Ok(metadata) => metadata.is_dir(), + Err(_unreadable) => false, + } +} + +/// Every directory below `directory`, with its listing, descending only into real directories. +pub(crate) fn walk( + tree: &ProjectTree, + directory: &str, +) -> Result, StageDeferral> { + let listing: Arc> = listing(tree, directory)?; + let children: Vec = listing + .iter() + .filter(|entry| entry.is_walkable_dir) + .map(|entry| join_relative(directory, &entry.name)) + .collect(); + let nested: Vec> = children + .into_par_iter() + .map(|child| walk(tree, &child)) + .collect::>()?; + let mut walked: Vec = vec![(directory.to_owned(), listing)]; + walked.extend(nested.into_iter().flatten()); + Ok(walked) +} + +fn listing(tree: &ProjectTree, directory: &str) -> Result>, StageDeferral> { + if let Some(cached) = cached_listing(tree, directory) { + return Ok(cached); + } + let listing: Arc> = Arc::new(list_directory(&tree.absolute(directory))?); + if let Ok(mut listings) = tree.listings.lock() { + listings.insert(directory.to_owned(), Arc::clone(&listing)); + } + Ok(listing) +} + +fn cached_listing(tree: &ProjectTree, directory: &str) -> Option>> { + match tree.listings.lock() { + Ok(listings) => listings.get(directory).cloned(), + Err(_poisoned) => None, + } +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs new file mode 100644 index 0000000000..7f1b91e6d5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -0,0 +1 @@ +pub mod rglob; diff --git a/crates/sqlbuild-discovery/src/tree/main/rglob.rs b/crates/sqlbuild-discovery/src/tree/main/rglob.rs new file mode 100644 index 0000000000..223cc056c1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/rglob.rs @@ -0,0 +1,33 @@ +//! `sorted(root.rglob(pattern))` over the shared snapshot. + +use crate::models::StageDeferral; +use crate::tree::_helpers::listing::join_relative; +use crate::tree::_helpers::ordering::compare_relative_paths; +use crate::tree::_helpers::walking::{WalkedDirectory, is_dir, walk}; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// Every entry below `root` that `accept` keeps, in Python's sorted path order. +pub fn rglob( + tree: &ProjectTree, + root: &str, + accept: impl Fn(&TreeEntry) -> bool, +) -> Result, StageDeferral> { + if !is_dir(tree, root) { + return Ok(Vec::new()); + } + let mut selected: Vec = Vec::new(); + for walked in walk(tree, root)? { + selected.extend(accepted_paths(&walked, &accept)); + } + selected.sort_by(|left, right| compare_relative_paths(left, right)); + Ok(selected) +} + +fn accepted_paths(walked: &WalkedDirectory, accept: &impl Fn(&TreeEntry) -> bool) -> Vec { + let (directory, listing) = walked; + listing + .iter() + .filter(|entry| accept(entry)) + .map(|entry| join_relative(directory, &entry.name)) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/tree/mod.rs b/crates/sqlbuild-discovery/src/tree/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/tree/models.rs b/crates/sqlbuild-discovery/src/tree/models.rs new file mode 100644 index 0000000000..96d24a7eca --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/models.rs @@ -0,0 +1,40 @@ +//! One shared directory snapshot for a native discovery pass. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +/// One directory entry, with the same type facts as Python's `DirectorySnapshotEntry`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeEntry { + pub name: String, + /// `entry.is_dir()`, following symbolic links. + pub is_dir: bool, + /// `entry.is_dir(follow_symlinks=False)`: a real directory the walk descends into. + pub is_walkable_dir: bool, +} + +/// Directory listings read once per pass and shared by every query, keyed by relative path. +#[derive(Debug)] +pub struct ProjectTree { + pub(crate) directory: PathBuf, + pub(crate) listings: Mutex>>>, +} + +impl ProjectTree { + /// An empty snapshot of `directory`; listings are read on first use. + pub fn new(directory: &Path) -> Self { + Self { + directory: directory.to_path_buf(), + listings: Mutex::new(HashMap::new()), + } + } + + /// The absolute path of a `/`-separated project-relative path. + pub fn absolute(&self, relative_path: &str) -> PathBuf { + relative_path + .split('/') + .filter(|part| !part.is_empty()) + .fold(self.directory.clone(), |path, part| path.join(part)) + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests.rs b/crates/sqlbuild-discovery/src/tree/tests.rs new file mode 100644 index 0000000000..6993fa3916 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_project_tree.rs"] +mod project_tree; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/tree/tests/helpers.rs b/crates/sqlbuild-discovery/src/tree/tests/helpers.rs new file mode 100644 index 0000000000..fa88ccc6d6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/helpers.rs @@ -0,0 +1,38 @@ +use crate::tree::main::rglob::rglob; +use crate::tree::models::ProjectTree; +use crate::tree::tests::test_types::GlobTestCase; +use std::fs; +use std::path::Path; + +fn write_file(project: &Path, relative_path: &str) { + let path = project.join(relative_path); + fs::create_dir_all(path.parent().expect("parent directory")).expect("directories"); + fs::write(path, "MODEL ();\nSELECT 1").expect("file"); +} + +#[cfg(unix)] +fn link(project: &Path, (link, target): (&str, &str)) { + std::os::unix::fs::symlink(project.join(target), project.join(link)).expect("symlink"); +} + +/// Build the case's project and glob `models/` for its suffix. +pub(super) fn globbed_paths(test_case: &GlobTestCase) -> Vec { + let project = tempfile::tempdir().expect("temporary project"); + test_case + .files + .iter() + .for_each(|relative_path| write_file(project.path(), relative_path)); + test_case.directories.iter().for_each(|relative_path| { + fs::create_dir_all(project.path().join(relative_path)).expect("directory") + }); + #[cfg(unix)] + test_case + .links + .iter() + .for_each(|pair| link(project.path(), *pair)); + let tree = ProjectTree::new(project.path()); + rglob(&tree, "models", |entry| { + entry.name.ends_with(test_case.suffix) + }) + .expect("walk") +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs b/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs new file mode 100644 index 0000000000..11b214bcb6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs @@ -0,0 +1,51 @@ +use crate::tree::tests::helpers::globbed_paths; +use crate::tree::tests::test_types::GlobTestCase; + +#[test] +fn given_project_layouts_when_globbing_then_paths_match_python_rglob_and_sort() { + let test_cases = [ + GlobTestCase { + description: "paths sort part by part, hidden files and directories match", + files: &[ + "models/a-c.sql", + "models/a/b.sql", + "models/b.sql", + "models/a/.x.sql", + ], + directories: &["models/dir.sql"], + links: &[], + suffix: ".sql", + expected_paths: &[ + "models/a/.x.sql", + "models/a/b.sql", + "models/a-c.sql", + "models/b.sql", + "models/dir.sql", + ], + }, + GlobTestCase { + description: "missing root finds nothing", + files: &["seeds/a.sql"], + directories: &[], + links: &[], + suffix: ".sql", + expected_paths: &[], + }, + GlobTestCase { + description: "a directory link matches by name but is not walked", + files: &["elsewhere/inner.sql", "models/x.txt"], + directories: &[], + links: &[("models/linked.sql", "elsewhere")], + suffix: ".sql", + expected_paths: &["models/linked.sql"], + }, + ]; + for test_case in test_cases { + assert_eq!( + globbed_paths(&test_case), + test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_types.rs b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs new file mode 100644 index 0000000000..f5f6278f16 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs @@ -0,0 +1,9 @@ +pub(super) struct GlobTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) directories: &'static [&'static str], + /// Symbolic links as `(link, target)`, both relative to the project. + pub(super) links: &'static [(&'static str, &'static str)], + pub(super) suffix: &'static str, + pub(super) expected_paths: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index bf813b9f9f..0b4f3a971a 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -23,6 +23,7 @@ serde_json.workspace = true sqlbuild-analysis.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-discovery.workspace = true sqlbuild-rules.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs new file mode 100644 index 0000000000..19f8190e09 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -0,0 +1,117 @@ +//! Native discovery results as the plain data the Python discovery facade materialises. + +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use sqlbuild_discovery::models::{ + DiscoveredFile, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, +}; +use sqlbuild_discovery::tree::models::ProjectTree; +use std::path::PathBuf; + +use crate::bindings::_helpers::functions::map_to_python; +use crate::bindings::models::ModelDiscoveryRequest; +use crate::bindings::types::CompilerDetach; + +type PyObject = Py; +type Located = (String, usize, usize, usize, usize); + +fn located(locations: Vec<(String, LineColumnSpan)>) -> Vec { + locations + .into_iter() + .map(|(name, span)| (name, span.line, span.column, span.end_line, span.end_column)) + .collect() +} + +fn tuple_object<'py>(py: Python<'py>, items: Vec) -> PyResult { + Ok(PyTuple::new(py, items)?.unbind().into_any()) +} + +fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult { + value.into_py_any(py) +} + +fn outcome_object( + py: Python<'_>, + outcome: FileOutcome, + parsed: impl FnOnce(Python<'_>, T) -> PyResult, +) -> PyResult { + match outcome { + FileOutcome::Parsed(value) => parsed(py, value), + FileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), + FileOutcome::Failed(failure) => tuple_object( + py, + vec![ + object(py, "error")?, + object(py, failure.kind.as_str())?, + object(py, failure.message)?, + object(py, failure.help)?, + ], + ), + } +} + +fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult { + tuple_object( + py, + vec![ + object(py, "ok")?, + object(py, model.contents)?, + map_to_python(py, model.header_values)?, + object(py, located(model.header_column_locations))?, + object(py, located(model.output_column_locations))?, + object(py, model.query_sql)?, + ], + ) +} + +fn files_object( + py: Python<'_>, + files: Vec>, + parsed: impl Fn(Python<'_>, T) -> PyResult, +) -> PyResult> { + files + .into_iter() + .map(|file| { + Ok(( + file.relative_path, + outcome_object(py, file.outcome, &parsed)?, + )) + }) + .collect() +} + +/// Discover the model files natively, or return `None` when Python must run the stage. +#[pyfunction] +fn discover_model_files( + py: Python<'_>, + request: ModelDiscoveryRequest, +) -> PyResult>> { + let root = ProjectRoot { + directory: PathBuf::from(request.project_dir), + display_prefix: request.display_prefix, + }; + let options = ModelFileOptions { + supported_keys: request.supported_keys, + removed_keys: request.removed_keys, + extract_implicit_alias_columns: request.extract_implicit_alias_columns, + extract_output_column_locations: request.extract_output_column_locations, + }; + let discovered: Result>, StageDeferral> = py + .compiler_detach(|| { + let tree = ProjectTree::new(&root.directory); + Ok(discover_models(&root, &tree, &options)) + }) + .map_err(crate::bindings::_helpers::panics::compiler_error)?; + match discovered { + Ok(files) => Ok(Some(files_object(py, files, model_object)?)), + Err(_deferral) => Ok(None), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs index 95f89aec8a..50af8a6f98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs @@ -376,7 +376,7 @@ fn authored_value_to_python( } } -fn map_to_python( +pub(crate) fn map_to_python( py: Python<'_>, values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, ) -> PyResult> { @@ -594,6 +594,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; crate::bindings::_helpers::oracles::register(module)?; + crate::bindings::_helpers::discovery::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index d1e72c2cfb..5e8b1378a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,6 +1,7 @@ mod allocator; mod catalog; pub(crate) mod conversions; +mod discovery; pub(crate) mod functions; mod normalization_results; mod oracles; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index 1c422613db..05a6f267c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -14,7 +14,10 @@ use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{ JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, }; +use sqlbuild_core::text::main::close_matches::close_matches; use sqlbuild_core::text::main::decode_python_text::decode_python_text; +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; use sqlbuild_core::text::models::LineIndex; use std::path::Path; @@ -226,7 +229,31 @@ fn _oracle_project_config(project_dir: &str) -> String { })) } +#[pyfunction] +fn _oracle_python_alnum(code_points: Vec) -> (String, Vec) { + ( + python_alnum_unicode_version().to_owned(), + code_points + .into_iter() + .map(|code_point| char::from_u32(code_point).is_some_and(is_python_alnum)) + .collect(), + ) +} + +#[pyfunction] +fn _oracle_close_matches( + word: &str, + possibilities: Vec, + count: usize, + cutoff: f64, +) -> Vec { + let candidates: Vec<&str> = possibilities.iter().map(String::as_str).collect(); + close_matches(word, &candidates, count, cutoff) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_python_alnum, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_close_matches, module)?)?; module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; module.add_function(wrap_pyfunction!(_oracle_yaml_load, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 26fc63d576..b9982f238c 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -96,3 +96,15 @@ impl From for validation::PositionInput { } } } + +/// One native model discovery request from the Python discovery facade. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct ModelDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) supported_keys: Vec, + pub(crate) removed_keys: Vec, + pub(crate) extract_implicit_alias_columns: bool, + pub(crate) extract_output_column_locations: bool, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs index df18220cc1..cd07d750a3 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs @@ -1,11 +1,20 @@ //! Exact native equivalent of the possessive Python `MODEL(...)` header regular expression. -/// Code-point offsets of one matched header: header start, header end, and SQL start. +use sqlbuild_core::text::main::is_python_space::is_python_space; + +/// Offsets of one matched header: header start, header end, and SQL start. pub(crate) type HeaderMatchOffsets = (usize, usize, usize); const MODEL_KEYWORD: &[u8] = b"MODEL"; pub(crate) fn match_one(text: &str) -> Option { + match_one_bytes(text).map(|(header_start, header_end, sql_start)| { + code_point_offsets(text, header_start, header_end, sql_start) + }) +} + +/// UTF-8 byte offsets of one matched header: header start, header end, and SQL start. +pub(crate) fn match_one_bytes(text: &str) -> Option { let bytes = text.as_bytes(); let mut index = skip_whitespace(text, 0); if !bytes[index..].starts_with(MODEL_KEYWORD) { @@ -25,12 +34,7 @@ pub(crate) fn match_one(text: &str) -> Option { return None; } let sql_start = skip_whitespace(text, index + 1); - Some(code_point_offsets( - text, - header_start, - header_end, - sql_start, - )) + Some((header_start, header_end, sql_start)) } /// Consume the possessive header body and return the byte offset where it stops. @@ -103,7 +107,7 @@ fn escape_end(text: &str, backslash: usize) -> Option { fn skip_whitespace(text: &str, start: usize) -> usize { let mut index = start; for character in text[start..].chars() { - if !is_python_whitespace(character) { + if !is_python_space(character) { break; } index += character.len_utf8(); @@ -111,24 +115,6 @@ fn skip_whitespace(text: &str, start: usize) -> usize { index } -/// Python's `str.isspace()`, which `\s` uses for `str` patterns. -fn is_python_whitespace(character: char) -> bool { - matches!( - character, - '\t'..='\r' - | '\u{1c}'..='\u{20}' - | '\u{85}' - | '\u{a0}' - | '\u{1680}' - | '\u{2000}'..='\u{200a}' - | '\u{2028}' - | '\u{2029}' - | '\u{202f}' - | '\u{205f}' - | '\u{3000}' - ) -} - fn code_point_offsets( text: &str, header_start: usize, diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs index f9a2f8bfa5..06224c0bc9 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs @@ -27,17 +27,14 @@ pub(crate) type HeaderParseResult = ( pub(crate) fn parse_batch(headers: &[String]) -> Result, String> { let pool = build_tokenizer_pool(headers.len())?; - Ok(pool.install(|| { - headers - .par_iter() - .map( - |header| match tokenize(header).and_then(HeaderParser::parse) { - Ok((value, offsets)) => (Some(value), Some(offsets), None), - Err(error) => (None, None, Some(error)), - }, - ) - .collect() - })) + Ok(pool.install(|| headers.par_iter().map(|header| parse_one(header)).collect())) +} + +pub(crate) fn parse_one(header: &str) -> HeaderParseResult { + match tokenize(header).and_then(HeaderParser::parse) { + Ok((value, offsets)) => (Some(value), Some(offsets), None), + Err(error) => (None, None, Some(error)), + } } pub(crate) fn tokenize_one(header: &str) -> Result, String> { diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index b06270fd75..e4d9f28ab3 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -1,5 +1,7 @@ +pub mod model_header_byte_matching; pub mod model_header_matching; pub mod model_header_parsing; +pub mod model_header_single_parsing; pub mod model_header_tokenizing; pub mod sql_interpolation; pub mod sql_references; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_header_byte_matching.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_byte_matching.rs new file mode 100644 index 0000000000..7ed07b85f1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_byte_matching.rs @@ -0,0 +1,6 @@ +use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; + +/// UTF-8 byte offsets of one matched header: header start, header end, and SQL start. +pub fn match_one_bytes(text: &str) -> Option { + matching::match_one_bytes(text) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_header_single_parsing.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_single_parsing.rs new file mode 100644 index 0000000000..c3af4f4bac --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_single_parsing.rs @@ -0,0 +1,6 @@ +use crate::compiler::_helpers::model_headers::tokenization::{self, HeaderParseResult}; + +/// Parse one header on the calling thread; deep headers need a large worker stack. +pub fn parse_one(header: &str) -> HeaderParseResult { + tokenization::parse_one(header) +} diff --git a/fensu.toml b/fensu.toml index de66a371f9..ae8ef366a2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -307,6 +307,7 @@ crate_names = [ "sqlbuild-analysis", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-discovery", "sqlbuild-python", "sqlbuild-rules", "sqlbuild-sqltext", diff --git a/scripts/generate_python_char_table.py b/scripts/generate_python_char_table.py new file mode 100644 index 0000000000..3d5d6ee034 --- /dev/null +++ b/scripts/generate_python_char_table.py @@ -0,0 +1,15 @@ +"""Direct wrapper for regenerating the native `str.isalnum()` table; run it with the CI Python.""" + +from __future__ import annotations + +from scripts.python_char_table.main.generate import generate_python_char_table + + +def main() -> int: + """Regenerate the native Python character table.""" + + return generate_python_char_table() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/python_char_table/__init__.py b/scripts/python_char_table/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/python_char_table/_helpers/__init__.py b/scripts/python_char_table/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/python_char_table/_helpers/table.py b/scripts/python_char_table/_helpers/table.py new file mode 100644 index 0000000000..29c505aa17 --- /dev/null +++ b/scripts/python_char_table/_helpers/table.py @@ -0,0 +1,41 @@ +"""Range computation and Rust rendering of Python's `str.isalnum()` table.""" + +from __future__ import annotations + +import unicodedata + +from scripts.python_char_table.constants import MAX_CODE_POINT + + +def alnum_ranges() -> list[tuple[int, int]]: + """Return the inclusive code point ranges where `str.isalnum()` is true.""" + + ranges: list[tuple[int, int]] = [] + start: int | None = None + for code_point in range(MAX_CODE_POINT + 1): + alnum: bool = chr(code_point).isalnum() + if alnum and start is None: + start = code_point + elif not alnum and start is not None: + ranges.append((start, code_point - 1)) + start = None + if start is not None: + ranges.append((start, MAX_CODE_POINT)) + return ranges + + +def render_table(ranges: list[tuple[int, int]]) -> str: + """Render the Rust source of the range table.""" + + lines: list[str] = [ + "//! Generated by `scripts/generate_python_char_table.py`; do not edit.", + "", + "/// The Unicode version of the Python that generated the table.", + f'pub(crate) const PYTHON_ALNUM_UNICODE_VERSION: &str = "{unicodedata.unidata_version}";', + "", + "/// Inclusive code point ranges where Python's `str.isalnum()` is true, in order.", + "pub(crate) const PYTHON_ALNUM_RANGES: &[(u32, u32)] = &[", + ] + lines.extend(f" (0x{start:04X}, 0x{end:04X})," for start, end in ranges) + lines.append("];") + return "\n".join(lines) + "\n" diff --git a/scripts/python_char_table/constants.py b/scripts/python_char_table/constants.py new file mode 100644 index 0000000000..46faf65387 --- /dev/null +++ b/scripts/python_char_table/constants.py @@ -0,0 +1,14 @@ +"""Paths and limits of the native Python character table generator.""" + +from pathlib import Path + +TABLE_PATH: Path = ( + Path(__file__).resolve().parents[2] + / "crates" + / "sqlbuild-core" + / "src" + / "text" + / "_helpers" + / "alnum_ranges.rs" +) +MAX_CODE_POINT: int = 0x10FFFF diff --git a/scripts/python_char_table/main/__init__.py b/scripts/python_char_table/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/python_char_table/main/generate.py b/scripts/python_char_table/main/generate.py new file mode 100644 index 0000000000..b0e9113c64 --- /dev/null +++ b/scripts/python_char_table/main/generate.py @@ -0,0 +1,17 @@ +"""Focused entry for regenerating the native `str.isalnum()` table with the CI Python.""" + +from __future__ import annotations + +import sys +import unicodedata + +from scripts.python_char_table._helpers.table import alnum_ranges, render_table +from scripts.python_char_table.constants import TABLE_PATH + + +def generate_python_char_table() -> int: + """Write the table next to the native text helpers and report its Unicode version.""" + + _ = TABLE_PATH.write_text(render_table(alnum_ranges()), encoding="utf-8") + print(f"wrote {TABLE_PATH} (Unicode {unicodedata.unidata_version})", file=sys.stderr) + return 0 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 532bca8ae0..24a7c622a9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -43,6 +43,9 @@ def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... +def discover_model_files( + request: dict[str, object], +) -> list[tuple[str, tuple[object, ...]]] | None: ... def substitute_static_project_vars( sqls: list[str], variables: list[tuple[str, str]] ) -> list[tuple[int, str | None]]: ... @@ -58,3 +61,7 @@ def _oracle_text_positions( def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... def _oracle_project_config(project_dir: str) -> str: ... +def _oracle_python_alnum(code_points: list[int]) -> tuple[str, list[bool]]: ... +def _oracle_close_matches( + word: str, possibilities: list[str], count: int, cutoff: float +) -> list[str]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index c5ad6b944b..6de7587bd5 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -34,6 +34,9 @@ from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( build_integration_loader_functions, ) +from sqlbuild.compiler.discovery._helpers.native.model_files import ( + discover_native_model_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( @@ -74,17 +77,24 @@ def build_discovered_project_inputs( extract_output_column_locations: bool = True, fact_cache: FactCacheStore | None = None, declaration_reuse: DeclarationFilesReuse | None = None, + native: bool = False, ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" with OperationLifecycle( operation_kind="project", operation_name="discovery_declaration_parse" ) as declaration_lifecycle: - discover: Callable[[], DiscoveredDeclarationFiles] = partial( - _discover_declaration_files, + discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]] = partial( + _discover_model_files, project_dir=project_dir, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, + native=native, + ) + discover: Callable[[], DiscoveredDeclarationFiles] = partial( + _discover_declaration_files, + project_dir=project_dir, + discover_models=discover_models, fact_cache=fact_cache, ) declarations: DiscoveredDeclarationFiles = ( @@ -93,12 +103,7 @@ def build_discovered_project_inputs( else declaration_reuse.declaration_files( variant=f"{int(sql_analysis_enabled)}{int(extract_output_column_locations)}", discover=discover, - discover_models=partial( - _discover_model_files, - project_dir=project_dir, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ), + discover_models=discover_models, ) ) declaration_lifecycle.completed( @@ -207,18 +212,13 @@ def build_discovered_project_inputs( def _discover_declaration_files( *, project_dir: Path, - sql_analysis_enabled: bool, - extract_output_column_locations: bool, + discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], fact_cache: FactCacheStore | None, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): return DiscoveredDeclarationFiles( source_files=discover_source_files(project_dir=project_dir, fact_cache=fact_cache), - model_files=discover_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ), + model_files=discover_models(), enum_files=discover_enum_files(project_dir=project_dir), constant_files=discover_constant_files(project_dir=project_dir), model_schema_files=discover_model_schema_files(project_dir=project_dir), @@ -236,10 +236,17 @@ def _discover_declaration_files( def _discover_model_files( - *, project_dir: Path, sql_analysis_enabled: bool, extract_output_column_locations: bool + *, + project_dir: Path, + sql_analysis_enabled: bool, + extract_output_column_locations: bool, + native: bool, ) -> tuple[DiscoveredSqlModelFile, ...]: + discover: Callable[..., tuple[DiscoveredSqlModelFile, ...]] = ( + discover_native_model_files if native else discover_model_files + ) with DirectorySnapshot.scope(project_dir=project_dir): - return discover_model_files( + return discover( project_dir=project_dir, extract_implicit_alias_columns=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py new file mode 100644 index 0000000000..bbf0cb8e8d --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -0,0 +1,126 @@ +"""SQL model files discovered by the native engine, materialised as Python discovery records.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files +from sqlbuild.compiler.discovery._helpers.filesystem.model_files import ( + discover_matched_model_file, +) +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_display_prefix, + native_failure, + native_locations, +) +from sqlbuild.compiler.discovery._helpers.sql.declarations import ( + parse_model_constant_declarations, + parse_model_enum_declarations, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import ( + match_model_header, + project_native_header_values, +) +from sqlbuild.compiler.discovery.constants import ( + NATIVE_FAILED_TAG, + NATIVE_UNREADABLE_TAG, + REMOVED_SQL_MODEL_HEADER_KEYS, + SQL_MODEL_HEADER_KEYS, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.discovery.types import NativeLocation + +_WINDOWS_OS_NAME: str = "nt" + + +def discover_native_model_files( + *, + project_dir: Path, + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, +) -> tuple[DiscoveredSqlModelFile, ...]: + """Discover model files natively; Python runs when the native walk defers the project.""" + + discovered: list[tuple[str, tuple[object, ...]]] | None = ( + None + if os.name == _WINDOWS_OS_NAME + else _native.discover_model_files( + { + "project_dir": str(project_dir), + "display_prefix": native_display_prefix(project_dir), + "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), + "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), + "extract_implicit_alias_columns": extract_implicit_alias_columns, + "extract_output_column_locations": extract_output_column_locations, + } + ) + ) + if discovered is None: + return discover_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=extract_implicit_alias_columns, + extract_output_column_locations=extract_output_column_locations, + ) + return tuple( + _model_file( + project_dir=project_dir, + relative_path=Path(relative_path), + payload=payload, + extract_implicit_alias_columns=extract_implicit_alias_columns, + extract_output_column_locations=extract_output_column_locations, + ) + for relative_path, payload in discovered + ) + + +def _model_file( + *, + project_dir: Path, + relative_path: Path, + payload: tuple[object, ...], + extract_implicit_alias_columns: bool, + extract_output_column_locations: bool, +) -> DiscoveredSqlModelFile: + file_path: Path = project_dir / relative_path + tag: object = payload[0] + if tag == NATIVE_UNREADABLE_TAG: + contents: str = file_path.read_text(encoding="utf-8") + return discover_matched_model_file( + file_path=file_path, + relative_path=relative_path, + contents=contents, + header_match=match_model_header(contents), + extract_implicit_alias_columns=extract_implicit_alias_columns, + extract_output_column_locations=extract_output_column_locations, + ) + if tag == NATIVE_FAILED_TAG: + raise native_failure(payload) + _tag, contents, values, header_locations, output_locations, query_sql = payload + header_values: dict[str, object] = project_native_header_values(cast(dict[str, object], values)) + return DiscoveredSqlModelFile( + file_path=file_path, + relative_path=relative_path, + contents=str(contents), + header_values=header_values, + header_column_locations=native_locations( + locations=cast(list[NativeLocation], header_locations), relative_path=relative_path + ), + output_column_locations=native_locations( + locations=cast(list[NativeLocation], output_locations), relative_path=relative_path + ), + query_sql=str(query_sql), + enum_declarations=parse_model_enum_declarations( + raw_value=header_values.get("enums"), + model_name=file_path.stem, + relative_path=relative_path, + ), + constant_declarations=parse_model_constant_declarations( + raw_value=header_values.get("constants"), + model_name=file_path.stem, + relative_path=relative_path, + ), + extract_implicit_alias_columns=extract_implicit_alias_columns, + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py new file mode 100644 index 0000000000..523d746dba --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -0,0 +1,43 @@ +"""Shared decoding of the plain-data payloads native discovery returns.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError +from sqlbuild.compiler.discovery.types import NativeLocation +from sqlbuild.spec.contracts.models import SourceLocation + +_DISPLAY_PROBE: str = "_" +_FAILURE_CLASSES: dict[str, type[DiscoveryError]] = {"model_sql": ModelSqlParseError} + + +def native_display_prefix(project_dir: Path) -> str: + """Return the text `str(project_dir / name)` prints before `name`.""" + + return str(project_dir / _DISPLAY_PROBE)[: -len(_DISPLAY_PROBE)] + + +def native_failure(payload: tuple[object, ...]) -> DiscoveryError: + """Build the discovery error a native failure payload describes.""" + + _tag, kind, message, help_text = payload + error_class: type[DiscoveryError] = _FAILURE_CLASSES[str(kind)] + return error_class(str(message), help=None if help_text is None else str(help_text)) + + +def native_locations( + *, locations: list[NativeLocation], relative_path: Path +) -> dict[str, SourceLocation]: + """Build authored locations; a repeated name keeps its first position and last value.""" + + return { + name: SourceLocation( + path=relative_path, + line=line, + column=column, + end_line=end_line, + end_column=end_column, + ) + for name, line, column, end_line, end_column in locations + } diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index d103451a2a..9a44737f44 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -734,6 +734,12 @@ def prepare_matched_model_file_headers(header_matches: list[ModelHeaderMatch | N ) +def project_native_header_values(values: dict[str, object]) -> dict[str, object]: + """Project native header parser values, with their marker tuples, to Python header values.""" + + return _project_native_header_map(values) + + def _project_native_header_map(values: dict[str, object]) -> dict[str, object]: return {key: _project_native_header_value(value) for key, value in values.items()} diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 2d2396b48f..451aeaea91 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -180,3 +180,5 @@ DISCOVERY_FACT_CACHE_ALGORITHM: str = "discovered-files-v1" DISCOVERY_SOURCE_FACT_KIND: str = "source" DISCOVERY_SQL_TEST_FACT_KIND: str = "sql_test" +NATIVE_FAILED_TAG: str = "error" +NATIVE_UNREADABLE_TAG: str = "read" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index 6154330a7f..c603cfce2f 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -53,6 +53,16 @@ def discover_project_inputs( extract_output_column_locations=extract_output_column_locations, cache_request=cache_request, declaration_reuse=declaration_reuse, + native=False, + ), + native_stage=partial( + _discover_project_inputs, + project_dir=project_dir, + sql_analysis_enabled_override=sql_analysis_enabled_override, + extract_output_column_locations=extract_output_column_locations, + cache_request=cache_request, + declaration_reuse=declaration_reuse, + native=True, ), ) @@ -64,6 +74,7 @@ def _discover_project_inputs( extract_output_column_locations: bool, cache_request: DiscoveryCacheRequest | None, declaration_reuse: DeclarationFilesReuse | None, + native: bool, ) -> DiscoveredProjectInputs: with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): return _assemble_discovered_project_inputs( @@ -72,6 +83,7 @@ def _discover_project_inputs( extract_output_column_locations=extract_output_column_locations, cache_request=cache_request, declaration_reuse=declaration_reuse, + native=native, ) @@ -82,6 +94,7 @@ def _assemble_discovered_project_inputs( extract_output_column_locations: bool, cache_request: DiscoveryCacheRequest | None, declaration_reuse: DeclarationFilesReuse | None, + native: bool, ) -> DiscoveredProjectInputs: project_config: ProjectConfig = load_project_config(project_dir=project_dir) local_config: LocalConfig = load_local_config(project_dir=project_dir) @@ -101,25 +114,36 @@ def _assemble_discovered_project_inputs( else project_config.settings.sql_analysis ) ) - with FactCacheStore( - root=discovery_cache_root( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - cache_request=cache_request, - ), - namespace=DISCOVERY_FACT_CACHE_NAMESPACE, - algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, - ) as fact_cache: + if native: discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( project_dir=project_dir, project_config=project_config, local_config=local_config, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, - fact_cache=fact_cache, declaration_reuse=declaration_reuse, + native=True, ) + else: + with FactCacheStore( + root=discovery_cache_root( + project_dir=project_dir, + project_config=project_config, + local_config=local_config, + cache_request=cache_request, + ), + namespace=DISCOVERY_FACT_CACHE_NAMESPACE, + algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, + ) as fact_cache: + discovered_inputs = build_discovered_project_inputs( + project_dir=project_dir, + project_config=project_config, + local_config=local_config, + sql_analysis_enabled=sql_analysis_enabled, + extract_output_column_locations=extract_output_column_locations, + fact_cache=fact_cache, + declaration_reuse=declaration_reuse, + ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( configure_discovered_event_exporters, diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index f033bf0592..cd1a8be413 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -18,6 +18,7 @@ from sqlbuild.provider.classes.provider import Provider type ProjectProvider = Provider +type NativeLocation = tuple[str, int, int, int, int] class LoaderConnectionMode(StrEnum): diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py index 4c8aebc06a..13927c6920 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -5,8 +5,13 @@ from sqlbuild.compiler.frontier.types import CompilerStage -def native_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: - """Produce the frontier object for `until` natively; no stage is native yet.""" +def native_frontier[T]( + *, + until: CompilerStage, + python_stage: Callable[[], T], + native_stage: Callable[[], T] | None, +) -> T: + """Produce the frontier object for `until` natively; stages not yet native run in Python.""" _ = until - return python_stage() + return python_stage() if native_stage is None else native_stage() diff --git a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py index a872314978..9dee821b4e 100644 --- a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py +++ b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py @@ -8,13 +8,18 @@ from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage -def compile_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: +def compile_frontier[T]( + *, + until: CompilerStage, + python_stage: Callable[[], T], + native_stage: Callable[[], T] | None = None, +) -> T: """Produce the frontier object for `until` on the active engine, capturing it on request.""" result: T = ( python_stage() if active_compiler_engine() is CompilerEngine.PYTHON - else native_frontier(until=until, python_stage=python_stage) + else native_frontier(until=until, python_stage=python_stage, native_stage=native_stage) ) write_stage_capture(stage=until, value=result) return result diff --git a/tests/integration/src/sqlbuild/compiler/_test_types.py b/tests/integration/src/sqlbuild/compiler/_test_types.py index f17956cecf..4c549cca28 100644 --- a/tests/integration/src/sqlbuild/compiler/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/_test_types.py @@ -37,3 +37,25 @@ class OrjsonFloatLayoutTestCase: description: str value: float expected_text: str + + +@dataclass(frozen=True) +class CloseMatchesOracleTestCase: + """Seeded words compared between the native port and `difflib.get_close_matches`.""" + + description: str + seed: int + case_count: int + count: int + cutoff: float + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class CharacterClassOracleTestCase: + """A code point range classified natively and by Python's `str` methods.""" + + description: str + first: int + last: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 445a81334c..e0a3208398 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -43,3 +43,32 @@ class LargeDocumentOracleTestCase: description: str document: str expected_maximum_seconds: float = 2.0 + + +@dataclass(frozen=True) +class ModelDiscoveryParityTestCase: + """Authored model files discovered by both compiler engines.""" + + description: str + files: tuple[tuple[str, bytes], ...] + directories: tuple[str, ...] = () + expected_identical: bool = True + + +@dataclass(frozen=True) +class GeneratedModelParityTestCase: + """Seeded model files discovered by both compiler engines, one file at a time.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class EngineSwitchParityTestCase: + """A project discovered through `discover_project_inputs` under each engine.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_identical: bool = True diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 86242681ed..c842a846da 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -17,7 +17,14 @@ import yaml from sqlbuild import _native +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files +from sqlbuild.compiler.discovery._helpers.native.model_files import ( + discover_native_model_files, +) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text @@ -579,3 +586,121 @@ def harmful_mismatches( return list( compress(zip(inputs, expected, actual, strict=True), map(_is_harmful, expected, actual)) )[:3] + + +PROJECT_CONFIG: str = 'name = "orders"\nadapter = "duckdb"\n' +HEADER_ENTRIES: tuple[str, ...] = ( + "materialized table", + "description 'Orders by customer'", + "description \"é ☕ 'quoted'\"", + "tags [core, finance]", + "unique_key [order_id]", + 'columns (order_id (type INTEGER), "Total" (type DECIMAL(10, 2)))', + "columns (\n customer_id (type BIGINT, audits [not_null]),\n )", + "enums (_status [OPEN, CLOSED])", + "enums (status [open])", + "constants (_limit_rows 10, _ratio 1.5)", + 'pre_hooks ["SELECT 1"]', + "tagz [a]", + "materialised table", + "run_despite_unchanged true", + "materialized: table", + "description 'unterminated", + "/* note */ colums (id (type INT))", + "-- trailing comment\n audits [unique]", +) +SELECT_ITEMS: tuple[str, ...] = ( + "order_id", + "o.customer_id", + "total AS amount", + "total as Amount", + "total aſ amount", + "sum(total) total_sum", + "COUNT(*)", + '"Quoted Name"', + 'o."Ünit"', + "'a,b' AS text_value", + "'it\\'s' AS escaped", + "CASE WHEN a THEN 1 END AS flag", + "(SELECT 1) sub", + "café", + "1 AS ünder", +) +TAILS: tuple[str, ...] = ( + "", + " FROM orders o", + " FROM orders\nUNION ALL\nSELECT 1 FROM b", + " FROM orders unıon SELECT 1", + " FROM (SELECT 1 FROM x) t", + " fRoM orders", + "\n-- FROM comment", +) +SEPARATORS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", " ", "\u3000", "\n\n") + + +class FailureCapture: + """Swallow and keep the discovery failure a block raises.""" + + def __init__(self) -> None: + self.failure: BaseException | None = None + + def __enter__(self) -> FailureCapture: + return self + + def __exit__(self, error_type: object, error: BaseException | None, traceback: object) -> bool: + self.failure = error + return isinstance(error, OSError | ValueError) + + +def model_discovery_outcome(*, project_dir: Path, native: bool) -> object: + """Return the rendered model files, or the error type, message and help.""" + + discover: Callable[..., tuple[DiscoveredSqlModelFile, ...]] = { + True: discover_native_model_files, + False: discover_model_files, + }[native] + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with DirectorySnapshot.scope(project_dir=project_dir), capture: + rendered[0] = render_stage_capture( + discover( + project_dir=project_dir, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + ) + ) + failure: BaseException | None = capture.failure + return { + True: rendered[0], + False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), + }[failure is None] + + +def write_project(*, project_dir: Path, files: tuple[tuple[str, bytes], ...]) -> None: + """Write a project config and authored files.""" + + project_dir.mkdir(parents=True, exist_ok=True) + _ = (project_dir / "sqlbuild_project.toml").write_text(PROJECT_CONFIG, encoding="utf-8") + for relative_path, data in files: + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(data) + + +def generated_model_bytes(*, rng: random.Random) -> bytes: + """Return one model file mixing valid and invalid headers, projections and newlines.""" + + separator: str = rng.choice(SEPARATORS) + entries: list[str] = rng.sample(HEADER_ENTRIES, k=rng.randint(0, 2)) + header: str = ("," + separator + " ").join(entries) + items: list[str] = rng.sample(SELECT_ITEMS, k=rng.randint(1, 4)) + body: str = "SELECT " + ("," + rng.choice(SEPARATORS)).join(items) + rng.choice(TAILS) + prefix: str = rng.choice(("",) * 8 + (" \n", "\ufeff", "-- lead\n")) + text: str = rng.choices( + ( + f"{prefix}MODEL ({separator} {header}{separator});{separator}{body}", + f"MODEL ({header});", + ), + weights=(19, 1), + )[0] + return text.encode("utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py new file mode 100644 index 0000000000..5f60bb47f3 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -0,0 +1,146 @@ +"""Both compiler engines discover identical SQL model files, failures and project inputs.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + EngineSwitchParityTestCase, + GeneratedModelParityTestCase, + ModelDiscoveryParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + generated_model_bytes, + model_discovery_outcome, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_VALID_MODEL: bytes = b"MODEL (materialized table);\nSELECT order_id, total AS amount FROM orders" + + +@pytest.mark.parametrize( + "test_case", + [ + ModelDiscoveryParityTestCase( + description="sorted part by part, scoped declaration trees skipped", + files=( + ("models/a-c.sql", _VALID_MODEL), + ("models/a/b.sql", _VALID_MODEL), + ("models/.hidden.sql", _VALID_MODEL), + ("models/core/_sqlbuild/schemas/x.sql", b"not a model"), + ("models/core/macros/helpers.sql", b"not a model"), + ("models/notes.txt", b"ignored"), + ), + ), + ModelDiscoveryParityTestCase( + description="CRLF, CR and non-ASCII locations", + files=( + ( + "models/orders.sql", + "MODEL (\r\n description 'é',\r columns (id (type INT)),\r\n);\r\n" + 'SELECT id, "Ünit" AS ünit FROM t'.encode(), + ), + ), + ), + ModelDiscoveryParityTestCase( + description="byte order mark before the header", + files=(("models/orders.sql", b"\xef\xbb\xbf" + _VALID_MODEL),), + ), + ModelDiscoveryParityTestCase( + description="invalid UTF-8 raises Python's decoding error", + files=(("models/a.sql", _VALID_MODEL), ("models/b.sql", b"MODEL ();\nSELECT '\xff'")), + ), + ModelDiscoveryParityTestCase( + description="a directory named like a model is read and fails", + files=(("models/a.sql", _VALID_MODEL),), + directories=("models/b.sql",), + ), + ModelDiscoveryParityTestCase( + description="the first failing file in order wins", + files=( + ("models/a.sql", _VALID_MODEL), + ("models/b.sql", b"MODEL (tagz [x]);\nSELECT 1"), + ("models/c.sql", b"SELECT 1"), + ), + ), + ModelDiscoveryParityTestCase( + description="model-local enum declarations fail after the header checks", + files=(("models/a.sql", b"MODEL (enums [a]);\nSELECT 1"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: ModelDiscoveryParityTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + for directory in test_case.directories: + (tmp_path / directory).mkdir(parents=True) + + python: object = model_discovery_outcome(project_dir=tmp_path, native=False) + native: object = model_discovery_outcome(project_dir=tmp_path, native=True) + + assert (native == python) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedModelParityTestCase( + description="mixed headers and projections", seed=61, case_count=400 + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_model_files_when_discovering_with_each_engine_then_outcomes_match( + test_case: GeneratedModelParityTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + contents: list[bytes] = [generated_model_bytes(rng=rng) for _ in range(test_case.case_count)] + expected: list[object] = [] + actual: list[object] = [] + for index, data in enumerate(contents): + project_dir: Path = tmp_path / f"case_{index}" + write_project(project_dir=project_dir, files=(("models/orders.sql", data),)) + expected.append(model_discovery_outcome(project_dir=project_dir, native=False)) + actual.append(model_discovery_outcome(project_dir=project_dir, native=True)) + + assert mismatches(inputs=list(contents), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineSwitchParityTestCase( + description="models, model-local declarations and a macro", + files=( + ("models/orders.sql", _VALID_MODEL), + ( + "models/marts/customers.sql", + b"MODEL (enums (_tier [GOLD, SILVER]));\nSELECT 1 AS id", + ), + ("macros/money.py", b"def cents(value):\n return f'{value} * 100'\n"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_when_discovering_through_the_engine_switch_then_inputs_match( + test_case: EngineSwitchParityTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + python: str = render_stage_capture(discover_project_inputs(project_dir=tmp_path)) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "native") + native: str = render_stage_capture(discover_project_inputs(project_dir=tmp_path)) + + assert (native == python) is test_case.expected_identical diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py index 67f935b588..8e8eb90868 100644 --- a/tests/integration/src/sqlbuild/compiler/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -299,3 +299,31 @@ def python_read_text(*, path: Path, data: bytes) -> str: path.write_bytes(data) return path.read_text(encoding="utf-8") + + +MUTATION_ALPHABET: str = "abcdefghijklmnopqrstuvwxyz_0é" + + +def _insert(*, rng: random.Random, word: list[str], position: int) -> None: + word.insert(position, rng.choice(MUTATION_ALPHABET)) + + +def _delete(*, rng: random.Random, word: list[str], position: int) -> None: + _ = rng + del word[position : position + 1] + + +def _substitute(*, rng: random.Random, word: list[str], position: int) -> None: + word[position : position + 1] = [rng.choice(MUTATION_ALPHABET)] + + +MUTATIONS: tuple[Callable[..., None], ...] = (_insert, _delete, _substitute) + + +def mutated_word(*, rng: random.Random, candidates: list[str]) -> str: + """Return a candidate with a few random insertions, deletions and substitutions.""" + + word: list[str] = list(rng.choice(candidates)) + for _ in range(rng.randint(0, 4)): + rng.choice(MUTATIONS)(rng=rng, word=word, position=rng.randint(0, len(word))) + return "".join(word) diff --git a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py new file mode 100644 index 0000000000..e7f22c465d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py @@ -0,0 +1,83 @@ +"""Compare native ports of Python string helpers with the Python standard library.""" + +from __future__ import annotations + +import difflib +import random +import sys +import unicodedata + +import pytest + +from sqlbuild import _native +from sqlbuild.compiler.discovery.constants import SQL_MODEL_HEADER_KEYS +from tests.integration.src.sqlbuild.compiler._test_types import ( + CharacterClassOracleTestCase, + CloseMatchesOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches, mutated_word + + +@pytest.mark.skipif( + _native._oracle_python_alnum([])[0] != unicodedata.unidata_version, + reason="the native table was generated from a Python with another Unicode database", +) +@pytest.mark.parametrize( + "test_case", + [CharacterClassOracleTestCase(description="every code point", first=0, last=sys.maxunicode)], + ids=lambda case: case.description, +) +def test_given_code_points_when_classifying_natively_then_isalnum_matches_python( + test_case: CharacterClassOracleTestCase, +) -> None: + code_points: list[int] = list(range(test_case.first, test_case.last + 1)) + + expected: list[object] = [chr(code_point).isalnum() for code_point in code_points] + actual: list[object] = list(_native._oracle_python_alnum(code_points)[1]) + + assert mismatches(inputs=list(code_points), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CloseMatchesOracleTestCase( + description="mutated header keys, best match only", + seed=51, + case_count=600, + count=1, + cutoff=0.6, + ), + CloseMatchesOracleTestCase( + description="mutated header keys, ranked with a low cutoff", + seed=52, + case_count=300, + count=4, + cutoff=0.2, + ), + ], + ids=lambda case: case.description, +) +def test_given_words_when_finding_close_matches_natively_then_difflib_agrees( + test_case: CloseMatchesOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + candidates: list[str] = sorted(SQL_MODEL_HEADER_KEYS) + words: list[str] = [ + mutated_word(rng=rng, candidates=candidates) for _ in range(test_case.case_count) + ] + + expected: list[object] = [ + difflib.get_close_matches(word, candidates, n=test_case.count, cutoff=test_case.cutoff) + for word in words + ] + actual: list[object] = [ + _native._oracle_close_matches(word, candidates, test_case.count, test_case.cutoff) + for word in words + ] + + assert mismatches(inputs=list(words), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) From 997514a66ef3cf441dd9502261c44826e7e0abd6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:08:22 +0100 Subject: [PATCH 2/3] fix(native): defer model discovery to Python when it cannot match exactly Run the Python path when the running Python's Unicode database differs from the native character table or the project root is not UTF-8, and share the native models walk with the pass's directory snapshot. --- .../src/tree/main/listings.rs | 15 ++++ .../sqlbuild-discovery/src/tree/main/mod.rs | 1 + .../src/bindings/_helpers/discovery.rs | 39 ++++++++-- src/sqlbuild/_native.pyi | 8 +- .../discovery/_helpers/native/model_files.py | 24 +++--- .../discovery/_helpers/native/payloads.py | 48 +++++++++++- .../discovery/classes/directory_snapshot.py | 8 ++ src/sqlbuild/compiler/discovery/types.py | 2 + .../compiler/discovery/_test_types.py | 24 +++++- .../sqlbuild/compiler/discovery/helpers.py | 12 +++ .../discovery/test_native_model_discovery.py | 78 +++++++++++++++++++ .../src/sqlbuild/compiler/helpers.py | 1 + .../test_native_python_compat_oracle.py | 11 ++- 13 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/tree/main/listings.rs diff --git a/crates/sqlbuild-discovery/src/tree/main/listings.rs b/crates/sqlbuild-discovery/src/tree/main/listings.rs new file mode 100644 index 0000000000..2a335fdd4e --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/listings.rs @@ -0,0 +1,15 @@ +//! The directory listings a pass has read, for the Python snapshot to share. + +use crate::tree::models::{ProjectTree, TreeEntry}; +use std::sync::Arc; + +/// Every listing read so far as `(relative directory, entries)`, in no particular order. +pub fn read_listings(tree: &ProjectTree) -> Vec<(String, Arc>)> { + match tree.listings.lock() { + Ok(listings) => listings + .iter() + .map(|(directory, entries)| (directory.clone(), Arc::clone(entries))) + .collect(), + Err(_poisoned) => Vec::new(), + } +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs index 7f1b91e6d5..cb7cc0dc9a 100644 --- a/crates/sqlbuild-discovery/src/tree/main/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -1 +1,2 @@ +pub mod listings; pub mod rglob; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs index 19f8190e09..30ddb6d43a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery.rs @@ -3,12 +3,14 @@ use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_alnum_unicode_version::python_alnum_unicode_version; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{ DiscoveredFile, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, }; -use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_discovery::tree::main::listings::read_listings; +use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; use std::path::PathBuf; use crate::bindings::_helpers::functions::map_to_python; @@ -83,12 +85,29 @@ fn files_object( .collect() } -/// Discover the model files natively, or return `None` when Python must run the stage. +type Listing = (String, Vec<(String, bool, bool)>); +type ModelDiscovery = (Vec<(String, PyObject)>, Vec); + +fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { + entries + .iter() + .map(|entry| (entry.name.clone(), entry.is_dir, entry.is_walkable_dir)) + .collect() +} + +fn listing_rows(tree: &ProjectTree) -> Vec { + read_listings(tree) + .into_iter() + .map(|(directory, entries)| (directory, entry_rows(&entries))) + .collect() +} + +/// Discover the model files natively with the listings the walk read, or `None` to defer. #[pyfunction] fn discover_model_files( py: Python<'_>, request: ModelDiscoveryRequest, -) -> PyResult>> { +) -> PyResult> { let root = ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -99,19 +118,27 @@ fn discover_model_files( extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, }; - let discovered: Result>, StageDeferral> = py + let (discovered, listings): ( + Result>, StageDeferral>, + Vec, + ) = py .compiler_detach(|| { let tree = ProjectTree::new(&root.directory); - Ok(discover_models(&root, &tree, &options)) + let discovered = discover_models(&root, &tree, &options); + Ok((discovered, listing_rows(&tree))) }) .map_err(crate::bindings::_helpers::panics::compiler_error)?; match discovered { - Ok(files) => Ok(Some(files_object(py, files, model_object)?)), + Ok(files) => Ok(Some((files_object(py, files, model_object)?, listings))), Err(_deferral) => Ok(None), } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; + module.add( + "PYTHON_ALNUM_UNICODE_VERSION", + python_alnum_unicode_version(), + )?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 24a7c622a9..70e969ca26 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -43,9 +43,15 @@ def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... + +PYTHON_ALNUM_UNICODE_VERSION: str + def discover_model_files( request: dict[str, object], -) -> list[tuple[str, tuple[object, ...]]] | None: ... +) -> ( + tuple[list[tuple[str, tuple[object, ...]]], list[tuple[str, list[tuple[str, bool, bool]]]]] + | None +): ... def substitute_static_project_vars( sqls: list[str], variables: list[tuple[str, str]] ) -> list[tuple[int, str | None]]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index bbf0cb8e8d..9061a7e797 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from pathlib import Path from typing import cast @@ -12,9 +11,11 @@ discover_matched_model_file, ) from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_discovery_supported, native_display_prefix, native_failure, native_locations, + seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.declarations import ( parse_model_constant_declarations, @@ -31,9 +32,7 @@ SQL_MODEL_HEADER_KEYS, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.discovery.types import NativeLocation - -_WINDOWS_OS_NAME: str = "nt" +from sqlbuild.compiler.discovery.types import NativeLocation, NativeModelDiscovery def discover_native_model_files( @@ -42,21 +41,22 @@ def discover_native_model_files( extract_implicit_alias_columns: bool, extract_output_column_locations: bool, ) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover model files natively; Python runs when the native walk defers the project.""" + """Discover model files natively; Python runs when native discovery cannot match it.""" - discovered: list[tuple[str, tuple[object, ...]]] | None = ( - None - if os.name == _WINDOWS_OS_NAME - else _native.discover_model_files( + display_prefix: str = native_display_prefix(project_dir) + discovered: NativeModelDiscovery | None = ( + _native.discover_model_files( { "project_dir": str(project_dir), - "display_prefix": native_display_prefix(project_dir), + "display_prefix": display_prefix, "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), "extract_implicit_alias_columns": extract_implicit_alias_columns, "extract_output_column_locations": extract_output_column_locations, } ) + if native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix) + else None ) if discovered is None: return discover_model_files( @@ -64,6 +64,8 @@ def discover_native_model_files( extract_implicit_alias_columns=extract_implicit_alias_columns, extract_output_column_locations=extract_output_column_locations, ) + files, listings = discovered + seed_snapshot_listings(project_dir=project_dir, listings=listings) return tuple( _model_file( project_dir=project_dir, @@ -72,7 +74,7 @@ def discover_native_model_files( extract_implicit_alias_columns=extract_implicit_alias_columns, extract_output_column_locations=extract_output_column_locations, ) - for relative_path, payload in discovered + for relative_path, payload in files ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 523d746dba..23d13f459a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -2,13 +2,23 @@ from __future__ import annotations +import os +import unicodedata from pathlib import Path +import sqlbuild._native as _native +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError -from sqlbuild.compiler.discovery.types import NativeLocation +from sqlbuild.compiler.discovery.types import ( + DirectorySnapshotEntry, + NativeListing, + NativeLocation, +) from sqlbuild.spec.contracts.models import SourceLocation _DISPLAY_PROBE: str = "_" +_WINDOWS_OS_NAME: str = "nt" +_PATH_ENCODING: str = "utf-8" _FAILURE_CLASSES: dict[str, type[DiscoveryError]] = {"model_sql": ModelSqlParseError} @@ -41,3 +51,39 @@ def native_locations( ) for name, line, column, end_line, end_column in locations } + + +def native_discovery_supported(*, project_dir: Path, display_prefix: str) -> bool: + """Whether native discovery reproduces Python here: same Unicode data, UTF-8 paths, POSIX.""" + + return ( + os.name != _WINDOWS_OS_NAME + and unicodedata.unidata_version == _native.PYTHON_ALNUM_UNICODE_VERSION + and _is_utf8_text(str(project_dir)) + and _is_utf8_text(display_prefix) + ) + + +def seed_snapshot_listings(*, project_dir: Path, listings: list[NativeListing]) -> None: + """Share the native walk's listings with the pass's Python directory snapshot.""" + + DirectorySnapshot.current(project_dir=project_dir).seed_listings( + {project_dir / directory: _snapshot_entries(entries) for directory, entries in listings} + ) + + +def _snapshot_entries( + entries: list[tuple[str, bool, bool]], +) -> tuple[DirectorySnapshotEntry, ...]: + return tuple( + DirectorySnapshotEntry(name=name, is_dir=is_dir, is_walkable_dir=walkable) + for name, is_dir, walkable in entries + ) + + +def _is_utf8_text(text: str) -> bool: + try: + _ = text.encode(_PATH_ENCODING) + except UnicodeEncodeError: + return False + return True diff --git a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py index fe030ff14d..81e06847a7 100644 --- a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py +++ b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py @@ -56,6 +56,14 @@ def current(cls, *, project_dir: Path) -> DirectorySnapshot: return active return DirectorySnapshot(project_dir=project_dir) + def seed_listings(self, listings: dict[Path, tuple[DirectorySnapshotEntry, ...]]) -> None: + """Share listings another walk of this pass read; listings already read are kept.""" + + directory: Path + entries: tuple[DirectorySnapshotEntry, ...] + for directory, entries in listings.items(): + _ = self._listings.setdefault(directory, entries) + def rglob(self, *, root: Path, pattern: str) -> tuple[Path, ...]: """Return the paths ``root.rglob(pattern)`` yields for a single-name pattern, unordered.""" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index cd1a8be413..2c91d5429a 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -19,6 +19,8 @@ type ProjectProvider = Provider type NativeLocation = tuple[str, int, int, int, int] +type NativeListing = tuple[str, list[tuple[str, bool, bool]]] +type NativeModelDiscovery = tuple[list[tuple[str, tuple[object, ...]]], list[NativeListing]] class LoaderConnectionMode(StrEnum): diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index e0a3208398..92f2242ec9 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -1,6 +1,8 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field + +import sqlbuild._native as _native @dataclass(frozen=True) @@ -72,3 +74,23 @@ class EngineSwitchParityTestCase: description: str files: tuple[tuple[str, bytes], ...] expected_identical: bool = True + + +@dataclass(frozen=True) +class NativeDeferralTestCase: + """A project and runtime under which native model discovery must or must not run.""" + + description: str + project_name: str + expected_native_calls: int + unidata_version: str = field(default_factory=lambda: _native.PYTHON_ALNUM_UNICODE_VERSION) + + +@dataclass(frozen=True) +class SharedSnapshotTestCase: + """A file created after native model discovery, inside the same discovery pass.""" + + description: str + created_file: str + pattern: str + expected_matches: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index c842a846da..69df632906 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -704,3 +704,15 @@ def generated_model_bytes(*, rng: random.Random) -> bytes: weights=(19, 1), )[0] return text.encode("utf-8") + + +class CallCounter: + """Count calls and delegate to the wrapped function.""" + + def __init__(self, function: Callable[..., object]) -> None: + self.function: Callable[..., object] = function + self.calls: int = 0 + + def __call__(self, *args: object) -> object: + self.calls += 1 + return self.function(*args) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index 5f60bb47f3..13d227cd94 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -2,11 +2,18 @@ from __future__ import annotations +import os import random +import unicodedata from pathlib import Path import pytest +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.native.model_files import ( + discover_native_model_files, +) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR @@ -14,8 +21,11 @@ EngineSwitchParityTestCase, GeneratedModelParityTestCase, ModelDiscoveryParityTestCase, + NativeDeferralTestCase, + SharedSnapshotTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + CallCounter, generated_model_bytes, model_discovery_outcome, write_project, @@ -144,3 +154,71 @@ def test_given_project_when_discovering_through_the_engine_switch_then_inputs_ma native: str = render_stage_capture(discover_project_inputs(project_dir=tmp_path)) assert (native == python) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + NativeDeferralTestCase( + description="matching Unicode data and a UTF-8 root run natively", + project_name="orders", + expected_native_calls=1, + ), + NativeDeferralTestCase( + description="different Unicode data defers to Python", + project_name="orders", + unidata_version="0.0.0", + expected_native_calls=0, + ), + NativeDeferralTestCase( + description="a non-UTF-8 project root defers to Python", + project_name=os.fsdecode(b"orders\xff"), + expected_native_calls=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_runtime_and_root_when_discovering_natively_then_python_runs_where_native_cannot_match( + test_case: NativeDeferralTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = tmp_path / test_case.project_name + write_project(project_dir=project_dir, files=(("models/orders.sql", _VALID_MODEL),)) + python: object = model_discovery_outcome(project_dir=project_dir, native=False) + counter: CallCounter = CallCounter(_native.discover_model_files) + monkeypatch.setattr(_native, "discover_model_files", counter) + monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) + + native: object = model_discovery_outcome(project_dir=project_dir, native=True) + + assert (native, counter.calls) == (python, test_case.expected_native_calls) + + +@pytest.mark.parametrize( + "test_case", + [ + SharedSnapshotTestCase( + description="a schema file created after the model walk stays unseen", + created_file="models/marts/schema.yml", + pattern="schema.yml", + ), + ], + ids=lambda case: case.description, +) +def test_given_native_model_walk_when_globbing_models_later_in_the_pass_then_the_walk_is_shared( + test_case: SharedSnapshotTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=(("models/marts/orders.sql", _VALID_MODEL),)) + with DirectorySnapshot.scope(project_dir=tmp_path) as snapshot: + _ = discover_native_model_files( + project_dir=tmp_path, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + ) + _ = (tmp_path / test_case.created_file).write_text("models: []\n", encoding="utf-8") + matches: tuple[Path, ...] = snapshot.rglob( + root=tmp_path / "models", pattern=test_case.pattern + ) + + assert tuple(path.relative_to(tmp_path).as_posix() for path in matches) == ( + test_case.expected_matches + ) diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py index 8e8eb90868..0d34d92e5f 100644 --- a/tests/integration/src/sqlbuild/compiler/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -301,6 +301,7 @@ def python_read_text(*, path: Path, data: bytes) -> str: return path.read_text(encoding="utf-8") +TABLE_PYTHON_VERSION: tuple[int, int] = (3, 12) MUTATION_ALPHABET: str = "abcdefghijklmnopqrstuvwxyz_0é" diff --git a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py index e7f22c465d..b8f59494e3 100644 --- a/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py @@ -15,12 +15,16 @@ CharacterClassOracleTestCase, CloseMatchesOracleTestCase, ) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches, mutated_word +from tests.integration.src.sqlbuild.compiler.helpers import ( + TABLE_PYTHON_VERSION, + mismatches, + mutated_word, +) @pytest.mark.skipif( - _native._oracle_python_alnum([])[0] != unicodedata.unidata_version, - reason="the native table was generated from a Python with another Unicode database", + sys.version_info[:2] != TABLE_PYTHON_VERSION, + reason="the native table reproduces the CI Python; other versions defer to Python discovery", ) @pytest.mark.parametrize( "test_case", @@ -31,6 +35,7 @@ def test_given_code_points_when_classifying_natively_then_isalnum_matches_python test_case: CharacterClassOracleTestCase, ) -> None: code_points: list[int] = list(range(test_case.first, test_case.last + 1)) + assert _native.PYTHON_ALNUM_UNICODE_VERSION == unicodedata.unidata_version expected: list[object] = [chr(code_point).isalnum() for code_point in code_points] actual: list[object] = list(_native._oracle_python_alnum(code_points)[1]) From 8f0e11661bd24bd80a07afd768836897205aea29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:33:45 +0100 Subject: [PATCH 3/3] test(compiler): pass every frontier stage through the perturbation hooks --- tests/e2e/scripts/compiler_differential/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index efadb94b07..e28a9accab 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -18,8 +18,8 @@ _original = frontier.native_frontier -def _perturbed(*, until, python_stage): - result = _original(until=until, python_stage=python_stage) +def _perturbed(*, until, **stages): + result = _original(until=until, **stages) if until is not CompilerStage.COMPILED_PROJECT or not result.models: return result first = result.models[0] @@ -42,8 +42,8 @@ def _perturbed(*, until, python_stage): _original = frontier.native_frontier -def _perturbed(*, until, python_stage): - result = _original(until=until, python_stage=python_stage) +def _perturbed(*, until, **stages): + result = _original(until=until, **stages) if until is not CompilerStage.DISCOVERED_PROJECT_INPUTS or len(result.model_files) < 2: return result return dataclasses.replace(result, model_files=tuple(reversed(result.model_files)))