diff --git a/AGENTS.md b/AGENTS.md index 79f0386051..6d5c006c03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,6 +201,53 @@ - Members marked `[forced]` are adapter contract overrides required by `test_strict_adapter.py`. Record other intentional mirrors in `[dupes]` in `fensu.toml` with a reason. - Public adapter contract methods are duplicated per adapter on purpose: `test_strict_adapter.py` requires every first-class adapter to define them, and `[dupes]` exempts them. The private helpers behind them must not be copied. Keep them in shared builders, such as the snapshot SQL builders in `src/sqlbuild/adapter/contract/classes/`, and let each contract method delegate. Express real dialect differences as explicit parameters, such as a `SnapshotSqlDialect` field, not as a forked copy of the helper. +## Rust and Native Builds + +- Python imports the native extension built from `crates/sqlbuild-python`. After changing any + Rust, rebuild it incrementally with `uv run --no-sync maturin develop --release --skip-install` + before running Python tests. Use one profile consistently within a worktree. +- Always run Python commands with `uv run --no-sync`. A syncing `uv run` or `uv sync` can replace + the locally built extension with a packaged one; if that happens, rebuild with maturin rather + than debugging stale native behaviour. +- Keep one persistent `CARGO_TARGET_DIR` per worktree, outside the repository, and reuse it for + the whole task. Never run `cargo clean`, delete a target directory, or switch target + directories mid-task. Do not override `RUSTC_WRAPPER`; the compiler cache is shared. +- An incremental rebuild after a focused change takes minutes. A build that is unexpectedly cold + or much slower than that is a signal to stop and investigate, not to wait or start over. +- Get fast feedback first: `cargo check -p ` while editing, `cargo test -p ` for + the crate you touched, then one maturin build and the focused Python tests. Run clippy with the + CI toolchain for the touched crates only. +- Fix compiler errors by reading them and making the smallest correct change. Never delete and + rewrite a working module, or restart a port from scratch, because a build failed. +- Do not run two native builds against the same target directory at once, and do not run + benchmarks while a build or test suite is running. +- Every new or changed binding updates `src/sqlbuild/_native.pyi` in the same change, and Rust + follows the `fensu.toml` layout rules (`uv run --no-sync fensu check`). + +## Implementation Subagents + +- Implementation subagents work only with explicit user approval, each on a bounded assignment in + its own worktree and branch. The primary agent owns the plan, integration, verification gates, + pushes and pull requests. +- A port's specification is the Python it replaces. Behaviour must match exactly, including error + types, messages, ordering and Python string semantics; check the native result against the + Python on realistic inputs before deleting the Python. +- Stop and hand back to the primary agent with a short report (what you tried, what blocks you, + the options you see) instead of continuing alone when: + - the next step will take an unusually long time, such as a cold rebuild, a broad or full test + suite, or any single command expected to run longer than about ten minutes; + - you are about to restart, rewrite from scratch, or abandon an approach you already built; + - the same build or test has failed twice with the same approach; + - the change needs files, crates, dependencies, `stays` entries or behaviour outside the + assignment; + - the Python specification is ambiguous, or native and Python results differ and the cause is + not obvious. +- Never run `cargo clean`, delete target directories, reset, rebase or force-push shared branches, + commit outside the assigned branch, run full suites, or edit `AGENTS.md` or + `python-boundary.toml` `stays` entries. +- Finish with local commits on the assigned branch and a report listing the files changed, the + Python deleted, the checks run with their results, and any residual risk. + ## Rust-Hosted Migration - Rust is moving to host compile, planning and non-warehouse CLI work. Python stays only for the modules listed in `python-boundary.toml`: user Python code, adapters, the executor and public Python APIs. diff --git a/Cargo.lock b/Cargo.lock index 41007b15cc..5cce8efcf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,36 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "attribute-derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 2.0.119", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -83,9 +113,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", + "regex-automata", "serde_core", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.4.4" @@ -108,6 +148,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "command-group" version = "5.0.1" @@ -118,6 +164,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -188,6 +248,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "derive-where" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "digest" version = "0.10.7" @@ -267,6 +338,51 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" +dependencies = [ + "attribute-derive", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "get-size2" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown", + "ordermap", + "smallvec", + "thin-vec", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -305,9 +421,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -322,6 +438,33 @@ dependencies = [ "rustversion", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + +[[package]] +name = "is-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8267aa6001e25494f3015f9663bbd88a18240c74483afa5f0934a1b3e4c388e9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -365,6 +508,29 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "memchr" version = "2.8.3" @@ -442,6 +608,53 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -478,6 +691,26 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -567,12 +800,64 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rayon" version = "1.12.0" @@ -642,6 +927,87 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ruff_python_ast" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12d40253033e38020db1bdb91a2cf5d0bee1e755c1938a26d43fa3c73d2b426" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "154345bd168f768a36f620180eabc9a5bae844d8c853182b1a24bb1a984c9b00" +dependencies = [ + "bitflags", + "bstr", + "compact_str", + "get-size2", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", + "unicode_names2", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc22c8e07f58267a6c517ab87df57f503ef92927cd0fc46c8445ff318a39e54" +dependencies = [ + "itertools", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7f467940ad19aead60f1a0d7ddbb08aa69f3bbfa49ab271886fb0d98459952" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dca33c148ef1d49a0dd430cdc05125e9c63a3c93028a2cbe7f6ffb3e5bbc4c7" +dependencies = [ + "get-size2", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.4" @@ -661,6 +1027,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -756,6 +1128,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + [[package]] name = "sqlbuild-analysis" version = "0.156.1" @@ -802,7 +1186,7 @@ version = "0.156.1" dependencies = [ "blake3", "ctrlc", - "getrandom", + "getrandom 0.4.3", "rustix", "serde", "serde_json", @@ -847,6 +1231,17 @@ dependencies = [ "sqlbuild-core", ] +[[package]] +name = "sqlbuild-pysource" +version = "0.156.1" +dependencies = [ + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", + "sha2", + "sqlbuild-core", +] + [[package]] name = "sqlbuild-python" version = "0.156.1" @@ -863,6 +1258,7 @@ dependencies = [ "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", + "sqlbuild-pysource", "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", @@ -973,6 +1369,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "2.0.119" @@ -1008,12 +1410,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -1054,6 +1462,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -1126,12 +1540,49 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_names2" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf", + "unicode_names2_generator", +] + +[[package]] +name = "unicode_names2_generator" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" +dependencies = [ + "getopts", + "log", + "phf_codegen", + "rand", +] + [[package]] name = "unindent" version = "0.2.4" @@ -1154,6 +1605,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "winapi" version = "0.3.9" @@ -1215,6 +1672,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index d142b6768f..b4f5016969 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/sqlbuild-core", + "crates/sqlbuild-pysource", "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", @@ -23,6 +24,7 @@ resolver = "2" [workspace.metadata.native-layers] order = [ "sqlbuild-core", + "sqlbuild-pysource", "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", @@ -73,6 +75,10 @@ blake3 = { version = "1", features = ["rayon"] } ctrlc = "3" quote = "1" rayon = "1" +# Static reading of project Python source; the parser fensu uses. +ruff_python_ast = "=0.0.4" +ruff_python_parser = "=0.0.4" +ruff_text_size = "=0.0.4" # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } saphyr-parser = "0.1" @@ -91,6 +97,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } stacker = "0.1" diff --git a/crates/README.md b/crates/README.md index 90e82ec284..38b90d60fb 100644 --- a/crates/README.md +++ b/crates/README.md @@ -7,6 +7,7 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | | `sqlbuild-config` | Configuration file reading without Python: TOML, YAML 1.2 with the core schema and merge keys, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | @@ -29,7 +30,7 @@ Dependencies point one way, from the top of this graph to the bottom: sqlbuild-python -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config - -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-core + -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-core sqlbuild-cli -> sqlbuild-cache, sqlbuild-core ``` diff --git a/crates/sqlbuild-pysource/Cargo.toml b/crates/sqlbuild-pysource/Cargo.toml new file mode 100644 index 0000000000..7855c87dea --- /dev/null +++ b/crates/sqlbuild-pysource/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqlbuild-pysource" +version.workspace = true +description = "Static analysis of project Python source with the ruff parser; never imports or runs it." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +ruff_python_ast.workspace = true +ruff_python_parser.workspace = true +ruff_text_size.workspace = true +sha2.workspace = true +sqlbuild-core.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs new file mode 100644 index 0000000000..da8c2d78f5 --- /dev/null +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -0,0 +1,6 @@ +//! Static reading of project Python source; CPython stays the authority on syntax validity. + +#![forbid(unsafe_code)] + +pub mod macro_modules; +pub mod module_parse; diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs new file mode 100644 index 0000000000..4a1551516a --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/declarations.rs @@ -0,0 +1,117 @@ +//! Top-level declarations of a macro module: private-name rules, exports and their signatures. + +use ruff_python_ast::{Expr, ModModule, Stmt, StmtFunctionDef}; +use ruff_text_size::Ranged; +use std::collections::HashSet; + +use crate::macro_modules::_helpers::python_nodes::PyNode; +use crate::macro_modules::constants::PRIVATE_PREFIX; + +/// The first public name a non-function top-level statement declares, if any. +pub(crate) fn public_non_function_declaration(statement: &Stmt) -> Option { + let names: Vec = match statement { + Stmt::ClassDef(class) => vec![class.name.as_str().to_owned()], + Stmt::TypeAlias(alias) => match alias.name.as_ref() { + Expr::Name(name) => vec![name.id.as_str().to_owned()], + _ => Vec::new(), + }, + Stmt::Assign(assign) => assign + .targets + .iter() + .flat_map(|target| PyNode::Expr(target).walk()) + .filter_map(PyNode::name) + .map(|name| name.id.as_str().to_owned()) + .collect(), + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(name) => vec![name.id.as_str().to_owned()], + _ => Vec::new(), + }, + _ => Vec::new(), + }; + names + .into_iter() + .find(|name| !name.starts_with(PRIVATE_PREFIX)) +} + +/// The top-level function definitions, public or not, in source order. +pub(crate) fn top_level_functions(module: &ModModule) -> impl Iterator { + module.body.iter().filter_map(|statement| match statement { + Stmt::FunctionDef(function) => Some(function), + _ => None, + }) +} + +/// Whether a function name is exported. +pub(crate) fn is_public(name: &str) -> bool { + !name.starts_with(PRIVATE_PREFIX) +} + +/// Names of the public top-level functions. +pub(crate) fn module_exports(module: &ModModule) -> HashSet { + top_level_functions(module) + .map(|function| function.name.as_str()) + .filter(|name| is_public(name)) + .map(str::to_owned) + .collect() +} + +/// Parameter names in `posonlyargs, args, vararg, kwonlyargs, kwarg` order. +pub(crate) fn parameter_names(function: &StmtFunctionDef) -> Vec { + let parameters = &function.parameters; + parameters + .posonlyargs + .iter() + .chain(¶meters.args) + .map(|parameter| ¶meter.parameter) + .chain(parameters.vararg.as_deref()) + .chain( + parameters + .kwonlyargs + .iter() + .map(|parameter| ¶meter.parameter), + ) + .chain(parameters.kwarg.as_deref()) + .map(|parameter| parameter.name.as_str().to_owned()) + .collect() +} + +/// CPython's `lineno` of a function: the line of its `def`, or of `async` for a coroutine. +pub(crate) fn definition_line(source: &str, function: &StmtFunctionDef) -> usize { + let mut start: usize = keyword_start(source, function.name.start().to_usize(), "def"); + if function.is_async { + start = keyword_start(source, start, "async"); + } + line_number(&source[..start]) +} + +/// Where `keyword` starts, reading back from `end` over blanks and line continuations. +fn keyword_start(source: &str, end: usize, keyword: &str) -> usize { + let mut text: &str = &source[..end]; + loop { + let trimmed: &str = text.trim_end_matches([' ', '\t', '\x0c']); + let continued: Option<&str> = trimmed + .strip_suffix("\\\r\n") + .or_else(|| trimmed.strip_suffix("\\\n")) + .or_else(|| trimmed.strip_suffix("\\\r")); + match continued { + Some(rest) => text = rest, + None => { + text = trimmed; + break; + } + } + } + text.strip_suffix(keyword).map_or(text.len(), str::len) +} + +/// The 1-based line of the text's end, counting `\n`, `\r\n` and `\r` as line breaks. +fn line_number(text: &str) -> usize { + let bytes: &[u8] = text.as_bytes(); + let mut line: usize = 1; + for (index, byte) in bytes.iter().enumerate() { + if *byte == b'\n' || (*byte == b'\r' && bytes.get(index + 1) != Some(&b'\n')) { + line += 1; + } + } + line +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs new file mode 100644 index 0000000000..03e1061fc8 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/function_calls.rs @@ -0,0 +1,188 @@ +//! Macro calls inside top-level function bodies, and names used outside them. + +use ruff_python_ast::{Expr, ExprContext, ModModule, StmtFunctionDef}; +use std::collections::{HashMap, HashSet}; + +use crate::macro_modules::_helpers::python_nodes::PyNode; + +/// The identity of an `ast.Name` node: its address in the parsed tree. +pub(crate) type NodeId = *const Expr; + +/// The imported macro bindings and local function names one module's calls resolve against. +pub(crate) struct CallNames<'a> { + /// Local binding to imported macro name. + pub(crate) imported: &'a HashMap, + pub(crate) functions: &'a HashSet, + pub(crate) display: &'a str, +} + +/// One function's dependencies on imported macros and the local functions it calls. +pub(crate) struct FunctionCalls { + pub(crate) dependencies: Vec, + pub(crate) helpers: Vec, +} + +/// The nodes of a function body, depth first, without entering nested scopes. +pub(crate) fn runtime_nodes(function: &StmtFunctionDef) -> Vec> { + let mut nodes: Vec> = Vec::new(); + let mut pending: Vec> = function.body.iter().rev().map(PyNode::Stmt).collect(); + while let Some(node) = pending.pop() { + nodes.push(node); + if node.opens_scope() { + continue; + } + pending.extend(node.children().into_iter().rev()); + } + nodes +} + +fn bound_names(function: &StmtFunctionDef, nodes: &[PyNode<'_>]) -> HashSet { + let parameters = &function.parameters; + let mut names: HashSet = parameters + .posonlyargs + .iter() + .chain(¶meters.args) + .chain(¶meters.kwonlyargs) + .map(|parameter| parameter.parameter.name.as_str().to_owned()) + .collect(); + names.extend( + parameters + .vararg + .iter() + .chain(¶meters.kwarg) + .map(|parameter| parameter.name.as_str().to_owned()), + ); + names.extend( + nodes + .iter() + .filter_map(|node| node.name()) + .filter(|name| name.ctx == ExprContext::Store) + .map(|name| name.id.as_str().to_owned()), + ); + names +} + +fn first_sorted<'a>(names: impl Iterator) -> Option<&'a String> { + names.min() +} + +fn direct_call_name(node: PyNode<'_>) -> Option<&Expr> { + match node { + PyNode::Expr(Expr::Call(call)) if call.func.is_name_expr() => Some(&call.func), + _ => None, + } +} + +/// The calls of one function, or the message of the first misuse Python reports. +pub(crate) fn function_calls( + function: &StmtFunctionDef, + nodes: &[PyNode<'_>], + names: &CallNames<'_>, +) -> Result { + let display: &str = names.display; + let bound: HashSet = bound_names(function, nodes); + if let Some(shadowed) = first_sorted( + bound + .iter() + .filter(|name| names.imported.contains_key(*name)), + ) { + return Err(format!( + "Macro module '{display}' shadows imported macro '{shadowed}'; imported macros must \ + be called directly without shadowing" + )); + } + let direct_calls: HashSet = nodes + .iter() + .filter_map(|node| direct_call_name(*node)) + .map(std::ptr::from_ref) + .collect(); + let mut dependencies: Vec = Vec::new(); + let mut helpers: Vec = Vec::new(); + for node in nodes { + if let PyNode::Expr(expression @ Expr::Name(name)) = node { + let direct: bool = direct_calls.contains(&std::ptr::from_ref(*expression)); + if !direct && names.imported.contains_key(name.id.as_str()) { + return Err(format!( + "Macro module '{display}' uses imported macro '{}' dynamically; call imported \ + macros directly", + name.id + )); + } + if !direct && names.functions.contains(name.id.as_str()) { + return Err(format!( + "Macro module '{display}' uses local helper '{}' dynamically; call local \ + macros and helpers directly", + name.id + )); + } + } + let Some(Expr::Name(called)) = direct_call_name(*node) else { + continue; + }; + let called: &str = called.id.as_str(); + if let Some(imported) = names.imported.get(called) { + dependencies.push(imported.clone()); + } else if names.functions.contains(called) { + if bound.contains(called) { + return Err(format!( + "Macro module '{display}' shadows local helper '{called}'; static macro \ + dependency analysis cannot resolve this call" + )); + } + helpers.push(called.to_owned()); + } + } + Ok(FunctionCalls { + dependencies: unique(dependencies), + helpers: unique(helpers), + }) +} + +/// The values without repeats, first occurrence first, as `dict.fromkeys` keeps them. +pub(crate) fn unique(values: Vec) -> Vec { + let mut seen: HashSet = HashSet::new(); + values + .into_iter() + .filter(|value| seen.insert(value.clone())) + .collect() +} + +/// Reject an imported macro or local function named outside an ordinary function body. +pub(crate) fn validate_use_locations( + module: &ModModule, + names: &CallNames<'_>, + runtime: &HashSet, +) -> Result<(), String> { + let display: &str = names.display; + let outside: Vec<(&str, NodeId)> = PyNode::Module(module) + .walk() + .into_iter() + .filter_map(|node| match node { + PyNode::Expr(expression @ Expr::Name(name)) => { + Some((name.id.as_str(), std::ptr::from_ref(expression))) + } + _ => None, + }) + .filter(|(_, id)| !runtime.contains(id)) + .collect(); + if let Some((name, _)) = outside + .iter() + .find(|(name, _)| names.imported.contains_key(*name)) + { + return Err(format!( + "Macro module '{display}' uses imported macro '{name}' outside an ordinary function \ + body; call imported macros directly from module-level macro or helper functions" + )); + } + if let Some((name, _)) = outside + .iter() + .find(|(name, _)| names.functions.contains(*name)) + { + return Err(format!( + "Macro module '{display}' uses local helper '{name}' outside an ordinary function \ + body; call local macros and helpers directly from module-level macro or helper \ + functions" + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs new file mode 100644 index 0000000000..0c53353130 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declarations; +pub(crate) mod function_calls; +pub(crate) mod module_analysis; +pub(crate) mod module_names; +pub(crate) mod project_analysis; +pub(crate) mod python_nodes; diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs new file mode 100644 index 0000000000..6163f88223 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_analysis.rs @@ -0,0 +1,357 @@ +//! One macro module's imports, calls and exported-macro dependencies. + +use ruff_python_ast::ExceptHandler; +use ruff_python_ast::{ModModule, Stmt, StmtFunctionDef, StmtImportFrom}; +use std::collections::{HashMap, HashSet}; +use std::iter::once; + +use crate::macro_modules::_helpers::declarations::module_exports; +use crate::macro_modules::_helpers::function_calls::{ + CallNames, NodeId, function_calls, runtime_nodes, unique, validate_use_locations, +}; +use crate::macro_modules::_helpers::module_names::{ + import_visible, imported_module_names, is_project_macro_import, resolve_import_from, +}; +use crate::macro_modules::_helpers::python_nodes::PyNode; +use crate::macro_modules::constants::{CYCLE_SEPARATOR, STAR_IMPORT_NAME}; +use crate::macro_modules::models::MacroModuleSource; + +/// One parsed module of the analysed set. +pub(crate) struct ParsedModule<'a> { + pub(crate) file_index: usize, + pub(crate) name: String, + pub(crate) source: &'a MacroModuleSource, + pub(crate) module: &'a ModModule, + pub(crate) exports: HashSet, +} + +/// Every parsed module of the analysed set, with the names imports resolve against. +pub(crate) struct ModuleSet<'a> { + pub(crate) modules: Vec>, + pub(crate) positions: HashMap, + pub(crate) module_names: Vec, + pub(crate) package_names: Vec, +} + +impl<'a> ModuleSet<'a> { + /// The set over `modules`, in discovery order. + pub(crate) fn new(modules: Vec>, package_names: Vec) -> Self { + let positions: HashMap = modules + .iter() + .enumerate() + .map(|(position, module)| (module.name.clone(), position)) + .collect(); + let module_names: Vec = modules.iter().map(|module| module.name.clone()).collect(); + Self { + modules, + positions, + module_names, + package_names, + } + } + + fn is_project_import(&self, imported: &str) -> bool { + is_project_macro_import(imported, (&self.module_names, &self.package_names)) + } +} + +/// A parsed module with its exports, ready for the set. +pub(crate) fn parsed_module<'a>( + file_index: usize, + (name, source, module): (String, &'a MacroModuleSource, &'a ModModule), +) -> ParsedModule<'a> { + ParsedModule { + file_index, + name, + source, + module, + exports: module_exports(module), + } +} + +/// What the analysis of one module found. +pub(crate) struct ModuleAnalysis { + pub(crate) file_index: usize, + pub(crate) name: String, + /// Each export's macro dependencies, through local helpers included. + pub(crate) dependencies: HashMap>, + /// Project macro modules imported, first import first. + pub(crate) module_dependencies: Vec, +} + +/// Analyse the module at `position`, or the message of the first error Python raised. +pub(crate) fn analyze_module( + set: &ModuleSet<'_>, + position: usize, +) -> Result { + let parsed: &ParsedModule<'_> = &set.modules[position]; + let display: &str = &parsed.source.relative_path.display; + validate_no_nested_project_imports(set, parsed)?; + let (imported, module_dependencies) = analyze_imports(set, parsed)?; + let mut function_order: Vec<&str> = Vec::new(); + let mut functions: HashMap<&str, &StmtFunctionDef> = HashMap::new(); + for statement in &parsed.module.body { + if let Stmt::FunctionDef(function) = statement + && functions.insert(function.name.as_str(), function).is_none() + { + function_order.push(function.name.as_str()); + } + } + if let Some(shadowed) = function_order + .iter() + .filter(|name| imported.contains_key(**name)) + .min() + { + return Err(format!( + "Macro module '{display}' defines function '{shadowed}' over an imported macro \ + binding; imported macros must not be replaced" + )); + } + let function_names: HashSet = function_order + .iter() + .map(|name| (*name).to_owned()) + .collect(); + let names: CallNames<'_> = CallNames { + imported: &imported, + functions: &function_names, + display, + }; + let mut direct_dependencies: HashMap> = HashMap::new(); + let mut helper_calls: HashMap> = HashMap::new(); + let mut runtime: HashSet = HashSet::new(); + for name in &function_order { + let function: &StmtFunctionDef = functions[name]; + let nodes: Vec> = runtime_nodes(function); + runtime.extend(nodes.iter().filter_map(|node| match node { + PyNode::Expr(expression) => Some(std::ptr::from_ref(*expression)), + _ => None, + })); + let calls = function_calls(function, &nodes, &names)?; + direct_dependencies.insert((*name).to_owned(), calls.dependencies); + helper_calls.insert((*name).to_owned(), calls.helpers); + } + validate_local_call_cycles(&parsed.exports, &helper_calls, display)?; + validate_use_locations(parsed.module, &names, &runtime)?; + Ok(ModuleAnalysis { + file_index: parsed.file_index, + name: parsed.name.clone(), + dependencies: export_dependencies(&parsed.exports, &direct_dependencies, &helper_calls), + module_dependencies, + }) +} + +/// Every statement inside `statement`'s compound bodies, at any depth. +fn nested_statements(statement: &Stmt) -> Vec<&Stmt> { + let bodies: Vec<&[Stmt]> = match statement { + Stmt::FunctionDef(node) => vec![&node.body], + Stmt::ClassDef(node) => vec![&node.body], + Stmt::For(node) => vec![&node.body, &node.orelse], + Stmt::While(node) => vec![&node.body, &node.orelse], + Stmt::If(node) => once(node.body.as_slice()) + .chain( + node.elif_else_clauses + .iter() + .map(|clause| clause.body.as_slice()), + ) + .collect(), + Stmt::With(node) => vec![&node.body], + Stmt::Match(node) => node.cases.iter().map(|case| case.body.as_slice()).collect(), + Stmt::Try(node) => once(node.body.as_slice()) + .chain( + node.handlers + .iter() + .map(|ExceptHandler::ExceptHandler(handler)| handler.body.as_slice()), + ) + .chain([node.orelse.as_slice(), node.finalbody.as_slice()]) + .collect(), + _ => Vec::new(), + }; + bodies + .into_iter() + .flatten() + .flat_map(|inner| once(inner).chain(nested_statements(inner))) + .collect() +} + +fn validate_no_nested_project_imports( + set: &ModuleSet<'_>, + parsed: &ParsedModule<'_>, +) -> Result<(), String> { + let mut project_import: bool = false; + for statement in parsed.module.body.iter().flat_map(nested_statements) { + if !matches!(statement, Stmt::Import(_) | Stmt::ImportFrom(_)) { + continue; + } + let names: Vec = imported_module_names(statement, &parsed.name); + project_import |= names.iter().any(|name| set.is_project_import(name)); + } + if project_import { + return Err(format!( + "Macro module '{}' contains a nested project macro import; project macro imports \ + must be static and module-level", + parsed.source.relative_path.display + )); + } + Ok(()) +} + +type ImportedBindings = (HashMap, Vec); + +fn analyze_imports( + set: &ModuleSet<'_>, + parsed: &ParsedModule<'_>, +) -> Result { + let display: &str = &parsed.source.relative_path.display; + let mut imported: HashMap = HashMap::new(); + let mut module_dependencies: Vec = Vec::new(); + for statement in &parsed.module.body { + if let Stmt::Import(_) = statement { + if imported_module_names(statement, &parsed.name) + .iter() + .any(|name| set.is_project_import(name)) + { + return Err(format!( + "Macro module '{display}' must use 'from ... import ...' for project macro \ + imports; module imports are not supported" + )); + } + continue; + } + let Stmt::ImportFrom(import) = statement else { + continue; + }; + let Some(resolved) = resolve_import_from(import, &parsed.name) else { + return Err(format!( + "Relative project macro import in '{}' escapes its top-level package", + parsed.name + )); + }; + let Some(position) = set.positions.get(&resolved) else { + if set.is_project_import(&resolved) { + return Err(format!( + "Macro module '{display}' imports project package '{resolved}' instead of a \ + macro module" + )); + } + continue; + }; + module_dependencies.push(resolved.clone()); + imported = record_imported_bindings( + import, + (parsed, &set.modules[*position]), + &resolved, + imported, + )?; + } + Ok((imported, unique(module_dependencies))) +} + +fn record_imported_bindings( + import: &StmtImportFrom, + (importer, imported_module): (&ParsedModule<'_>, &ParsedModule<'_>), + resolved: &str, + imported: HashMap, +) -> Result, String> { + let mut imported: HashMap = imported; + let display: &str = &importer.source.relative_path.display; + for alias in &import.names { + let name: &str = alias.name.as_str(); + if name == STAR_IMPORT_NAME { + return Err(format!( + "Macro module '{display}' must not use star imports for project macros" + )); + } + if !imported_module.exports.contains(name) { + return Err(format!( + "Macro module '{display}' cannot import '{name}' from project macro module \ + '{resolved}'; the name is not an exported macro" + )); + } + if !import_visible(importer.source, imported_module.source) { + return Err(format!( + "Macro module '{display}' cannot import macro '{name}' from '{}': the imported \ + macro is not visible from the importer scope", + imported_module.source.relative_path.display + )); + } + let binding: &str = alias.asname.as_ref().map_or(name, |asname| asname.as_str()); + if imported.contains_key(binding) { + return Err(format!( + "Macro module '{display}' binds project macro import '{binding}' more than once" + )); + } + imported.insert(binding.to_owned(), name.to_owned()); + } + Ok(imported) +} + +fn export_dependencies( + exports: &HashSet, + direct_dependencies: &HashMap>, + helper_calls: &HashMap>, +) -> HashMap> { + let mut dependencies: HashMap> = HashMap::new(); + for export in exports { + let mut found: Vec = Vec::new(); + let mut pending: std::collections::VecDeque<&str> = + std::collections::VecDeque::from([export.as_str()]); + let mut visited: HashSet<&str> = HashSet::new(); + while let Some(function) = pending.pop_front() { + if !visited.insert(function) { + continue; + } + found.extend(direct_dependencies[function].iter().cloned()); + for called in &helper_calls[function] { + if exports.contains(called) { + found.push(called.clone()); + } + pending.push_back(called); + } + } + dependencies.insert(export.clone(), unique(found)); + } + dependencies +} + +fn validate_local_call_cycles( + exports: &HashSet, + helper_calls: &HashMap>, + display: &str, +) -> Result<(), String> { + let mut sorted: Vec<&String> = exports.iter().collect(); + sorted.sort(); + let mut completed: HashSet = HashSet::new(); + for export in sorted { + completed = visit_cycle(export, (helper_calls, &[]), completed).map_err(|cycle| { + format!( + "Macro call cycle in '{display}': {}", + cycle.join(CYCLE_SEPARATOR) + ) + })?; + } + Ok(()) +} + +/// Depth-first cycle search returning the completed names; the error is the cycle path. +pub(crate) fn visit_cycle( + name: &str, + (edges, active): (&HashMap>, &[String]), + completed: HashSet, +) -> Result, Vec> { + if let Some(start) = active.iter().position(|item| item == name) { + let mut cycle: Vec = active[start..].to_vec(); + cycle.push(name.to_owned()); + return Err(cycle); + } + if completed.contains(name) { + return Ok(completed); + } + let mut path: Vec = active.to_vec(); + path.push(name.to_owned()); + let mut updated: HashSet = completed; + for dependency in &edges[name] { + updated = visit_cycle(dependency, (edges, &path), updated)?; + } + updated.insert(name.to_owned()); + Ok(updated) +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_names.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_names.rs new file mode 100644 index 0000000000..eda3e0a7fd --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/module_names.rs @@ -0,0 +1,153 @@ +//! Dotted macro module and package names, import resolution and scope visibility. + +use ruff_python_ast::{Stmt, StmtImportFrom}; + +use crate::macro_modules::constants::{ + MODULE_SEPARATOR, PATH_ROOTS, STAR_IMPORT_NAME, SUFFIX_MARK, +}; +use crate::macro_modules::models::MacroModuleSource; +use crate::macro_modules::types::MacroScopeKind; + +/// `".".join(relative_path.with_suffix("").parts)`. +pub(crate) fn module_name(source: &MacroModuleSource) -> String { + let parts: &[String] = &source.relative_path.parts; + let mut names: Vec<&str> = parts.iter().map(String::as_str).collect(); + if let Some(last) = names.last_mut() + && let Some(dot) = last.rfind(SUFFIX_MARK) + && dot > 0 + && dot < last.len() - SUFFIX_MARK.len_utf8() + { + *last = &last[..dot]; + } + names.join(MODULE_SEPARATOR) +} + +/// The dotted declaration root, or the module's first path part. +pub(crate) fn package_name(source: &MacroModuleSource) -> String { + match &source.declaration_root { + Some(parts) => parts.join(MODULE_SEPARATOR), + None => source + .relative_path + .parts + .first() + .cloned() + .unwrap_or_default(), + } +} + +/// Whether `imported` names a project macro module, one of its packages, or a project package. +pub(crate) fn is_project_macro_import( + imported: &str, + (module_names, package_names): (&[String], &[String]), +) -> bool { + module_names + .iter() + .any(|module| module == imported || is_dotted_child(module, imported)) + || package_names + .iter() + .any(|package| imported == package || is_dotted_child(imported, package)) +} + +fn is_dotted_child(name: &str, parent: &str) -> bool { + name.strip_prefix(parent) + .is_some_and(|rest| rest.starts_with(MODULE_SEPARATOR)) +} + +/// The package parts a relative import of `level` keeps from `current_module`, then `base`. +fn relative_parts(current_module: &str, level: usize, base: &str) -> Vec { + let mut package: Vec<&str> = current_module.split(MODULE_SEPARATOR).collect(); + package.pop(); + let retained: usize = (package.len() + 1).saturating_sub(level); + let mut parts: Vec = package + .iter() + .take(retained) + .map(|part| (*part).to_owned()) + .collect(); + if !base.is_empty() { + parts.extend(base.split(MODULE_SEPARATOR).map(str::to_owned)); + } + parts +} + +/// The absolute module a `from ... import` reads, or `None` when it escapes the top package. +pub(crate) fn resolve_import_from( + statement: &StmtImportFrom, + current_module: &str, +) -> Option { + let base: &str = statement + .module + .as_ref() + .map_or("", |module| module.as_str()); + let level: usize = statement.level as usize; + if level == 0 { + return Some(base.to_owned()); + } + let package_length: usize = current_module.split(MODULE_SEPARATOR).count() - 1; + if level > package_length { + return None; + } + Some(relative_parts(current_module, level, base).join(MODULE_SEPARATOR)) +} + +/// Every dotted name an import statement may load: the module, then each imported member. +pub(crate) fn imported_module_names(statement: &Stmt, current_module: &str) -> Vec { + match statement { + Stmt::Import(import) => import + .names + .iter() + .map(|alias| alias.name.as_str().to_owned()) + .collect(), + Stmt::ImportFrom(import) => { + let base: &str = import.module.as_ref().map_or("", |module| module.as_str()); + let base: String = if import.level == 0 { + base.to_owned() + } else { + relative_parts(current_module, import.level as usize, base).join(MODULE_SEPARATOR) + }; + let mut names: Vec = Vec::new(); + if !base.is_empty() { + names.push(base.clone()); + } + for alias in &import.names { + if alias.name.as_str() == STAR_IMPORT_NAME { + continue; + } + names.push(if base.is_empty() { + alias.name.as_str().to_owned() + } else { + format!("{base}{MODULE_SEPARATOR}{}", alias.name.as_str()) + }); + } + names + } + _ => Vec::new(), + } +} + +/// Whether a module importing from `imported` can see its macros, by path parts. +pub(crate) fn import_visible(importer: &MacroModuleSource, imported: &MacroModuleSource) -> bool { + let relative_parent: &[String] = importer + .relative_path + .parts + .split_last() + .map_or(&[], |(_, parent)| parent); + let importer_parent: &[String] = importer.owning_path.as_deref().unwrap_or(relative_parent); + let owner: &[String] = imported.owning_path.as_deref().unwrap_or(&[]); + match imported.scope_kind { + MacroScopeKind::Global => true, + MacroScopeKind::Local => importer_parent == owner, + MacroScopeKind::Inherited => { + importer_parent == owner || is_parent_path(owner, importer_parent) + } + MacroScopeKind::Private => false, + } +} + +/// Whether `owner` is one of `path.parents`. +fn is_parent_path(owner: &[String], path: &[String]) -> bool { + let anchored: bool = path + .first() + .is_some_and(|first| first.ends_with(PATH_ROOTS)); + let shortest: usize = usize::from(anchored); + owner.len() >= shortest && owner.len() < path.len() && path.starts_with(owner) +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/project_analysis.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/project_analysis.rs new file mode 100644 index 0000000000..5d60896932 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/project_analysis.rs @@ -0,0 +1,343 @@ +//! Strict analysis of a set of macro modules, and the fault-tolerant inventory built on it. + +use ruff_python_ast::ModModule; +use sha2::{Digest, Sha256}; +use std::cell::OnceCell; +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; + +use crate::macro_modules::_helpers::declarations::{ + definition_line, is_public, parameter_names, public_non_function_declaration, + top_level_functions, +}; +use crate::macro_modules::_helpers::module_analysis::{ + ModuleAnalysis, ModuleSet, ParsedModule, analyze_module, parsed_module, visit_cycle, +}; +use crate::macro_modules::_helpers::module_names::{module_name, package_name}; +use crate::macro_modules::constants::CYCLE_SEPARATOR; +use crate::macro_modules::errors::{MacroAnalysisFailure, MacroModuleParseError}; +use crate::macro_modules::models::{ + MacroExport, MacroFault, MacroModuleInventory, MacroModuleLoad, MacroModuleSource, +}; +use crate::module_parse::main::parse_python_module::parse_python_module; + +/// A parsed module, or the message of the error its parse or declaration check raised. +type ParsedSource = Result; + +/// The sources and their parses, each read at most once and only when analysis reaches it. +pub(crate) struct ProjectSources<'a> { + sources: &'a [MacroModuleSource], + parsed: Vec>>, +} + +impl<'a> ProjectSources<'a> { + /// Unparsed sources. + pub(crate) fn new(sources: &'a [MacroModuleSource]) -> Self { + Self { + sources, + parsed: sources.iter().map(|_| OnceCell::new()).collect(), + } + } + + fn parsed(&self, index: usize) -> Result<&ParsedSource, MacroModuleParseError> { + self.parsed[index] + .get_or_init(|| parse_source(&self.sources[index])) + .as_ref() + .map_err(Clone::clone) + } +} + +fn parse_source(source: &MacroModuleSource) -> Result { + if let Some(message) = &source.syntax_error { + return Ok(Err(message.clone())); + } + let module: ModModule = + parse_python_module(&source.contents).map_err(|message| MacroModuleParseError { + relative_path: source.relative_path.display.clone(), + message, + })?; + if let Some(name) = module.body.iter().find_map(public_non_function_declaration) { + return Ok(Err(format!( + "Macro module '{}' declaration '{name}' must be underscore-private", + source.relative_path.display + ))); + } + Ok(Ok(module)) +} + +/// The modules of `subset` analysed in discovery order, or why analysis stopped first. +pub(crate) fn analyze_project( + project: &ProjectSources<'_>, + subset: &[usize], +) -> Result, MacroAnalysisFailure> { + let mut names: Vec<(usize, String)> = Vec::new(); + let mut seen: HashMap = HashMap::new(); + for index in subset { + let name: String = module_name(&project.sources[*index]); + if let Some(existing) = seen.get(&name) { + return Err(MacroAnalysisFailure::Compile(format!( + "Macro module path collision for '{name}': {} and {}", + project.sources[*existing].relative_path.display, + project.sources[*index].relative_path.display + ))); + } + seen.insert(name.clone(), *index); + names.push((*index, name)); + } + let mut modules: Vec> = Vec::new(); + for (index, name) in names { + let parsed: &ModModule = project + .parsed(index) + .map_err(MacroAnalysisFailure::Native)? + .as_ref() + .map_err(|message| MacroAnalysisFailure::Compile(message.clone()))?; + modules.push(parsed_module( + index, + (name, &project.sources[index], parsed), + )); + } + let package_names: Vec = subset + .iter() + .map(|index| package_name(&project.sources[*index])) + .collect(); + let set: ModuleSet<'_> = ModuleSet::new(modules, package_names); + let analyses: Vec = (0..set.modules.len()) + .map(|position| analyze_module(&set, position)) + .collect::>() + .map_err(MacroAnalysisFailure::Compile)?; + validate_import_cycles(project, &analyses).map_err(MacroAnalysisFailure::Compile)?; + Ok(analyses) +} + +fn validate_import_cycles( + project: &ProjectSources<'_>, + analyses: &[ModuleAnalysis], +) -> Result<(), String> { + let edges: HashMap> = analyses + .iter() + .map(|analysis| (analysis.name.clone(), analysis.module_dependencies.clone())) + .collect(); + let files: HashMap<&str, usize> = analyses + .iter() + .map(|analysis| (analysis.name.as_str(), analysis.file_index)) + .collect(); + let mut sorted: Vec<&String> = edges.keys().collect(); + sorted.sort(); + let mut completed: HashSet = HashSet::new(); + for name in sorted { + completed = visit_cycle(name, (&edges, &[]), completed).map_err(|cycle| { + let repeated: &str = cycle.last().map_or("", String::as_str); + format!( + "Macro import cycle in '{}': {}", + project.sources[files[repeated]].relative_path.display, + cycle.join(CYCLE_SEPARATOR) + ) + })?; + } + Ok(()) +} + +fn source_digest(contents: &str) -> String { + let digest = Sha256::digest(contents.as_bytes()); + let mut text: String = String::with_capacity(digest.len() * 2); + for byte in digest { + let _ = write!(text, "{byte:02x}"); + } + text +} + +/// Exports of cleanly analysed modules, with a fault for every repeated export name. +fn inventory_from_analyses( + project: &ProjectSources<'_>, + analyses: &[ModuleAnalysis], +) -> (Vec, Vec) { + let mut exports: HashMap = HashMap::new(); + let mut collided: HashSet = HashSet::new(); + let mut faults: Vec = Vec::new(); + for analysis in analyses { + let source: &MacroModuleSource = &project.sources[analysis.file_index]; + let Ok(Ok(module)) = project.parsed(analysis.file_index) else { + continue; + }; + let digest: String = source_digest(&source.contents); + for function in top_level_functions(module) { + let name: &str = function.name.as_str(); + if !is_public(name) { + continue; + } + let item: MacroExport = MacroExport { + name: name.to_owned(), + file_index: analysis.file_index, + parameters: parameter_names(function), + line: definition_line(&source.contents, function), + source_digest: digest.clone(), + dependencies: analysis.dependencies[name].clone(), + }; + let display: &str = &source.relative_path.display; + if let Some(existing) = exports.remove(name) { + collided.insert(name.to_owned()); + faults.push(MacroFault { + file_index: analysis.file_index, + message: format!( + "Macro name collision for '{name}': {} and {display}", + project.sources[existing.file_index].relative_path.display + ), + }); + } else if collided.contains(name) { + faults.push(MacroFault { + file_index: analysis.file_index, + message: format!("Macro name collision for '{name}' at {display}"), + }); + } else { + exports.insert(name.to_owned(), item); + } + } + } + let mut exports: Vec = exports.into_values().collect(); + exports.sort_by(|left, right| { + ( + &left.name, + &project.sources[left.file_index].relative_path.posix, + ) + .cmp(&( + &right.name, + &project.sources[right.file_index].relative_path.posix, + )) + }); + (exports, faults) +} + +/// Groups of modules in `subset` sharing a module name, by name, each in discovery order. +fn module_path_collisions( + project: &ProjectSources<'_>, + subset: &[usize], +) -> Vec<(String, Vec)> { + let mut grouped: HashMap> = HashMap::new(); + for index in subset { + grouped + .entry(module_name(&project.sources[*index])) + .or_default() + .push(*index); + } + let mut collisions: Vec<(String, Vec)> = grouped + .into_iter() + .filter(|(_, files)| files.len() > 1) + .collect(); + collisions.sort_by(|left, right| left.0.cmp(&right.0)); + collisions +} + +fn deduplicated(project: &ProjectSources<'_>, faults: Vec) -> Vec { + let mut seen: HashSet<(String, String)> = HashSet::new(); + faults + .into_iter() + .filter(|fault| { + seen.insert(( + project.sources[fault.file_index] + .relative_path + .posix + .clone(), + fault.message.clone(), + )) + }) + .collect() +} + +/// The module a failure is attributed to: the first whose path the message names. +fn fault_file(project: &ProjectSources<'_>, subset: &[usize], message: &str) -> usize { + subset + .iter() + .copied() + .find(|index| message.contains(project.sources[*index].relative_path.posix.as_str())) + .unwrap_or(subset[0]) +} + +/// Re-analyse without each failing module until the rest analyse cleanly. +fn tolerant_inventory( + project: &ProjectSources<'_>, + first_failure: String, +) -> Result { + let mut remaining: Vec = (0..project.sources.len()).collect(); + let mut faults: Vec = Vec::new(); + let mut failure: Option = Some(first_failure); + while !remaining.is_empty() { + let outcome: Result, MacroAnalysisFailure> = match failure.take() { + Some(message) => Err(MacroAnalysisFailure::Compile(message)), + None => analyze_project(project, &remaining), + }; + let message: String = match outcome { + Ok(analyses) => { + let (exports, inventory_faults) = inventory_from_analyses(project, &analyses); + faults.extend(inventory_faults); + return Ok(MacroModuleInventory { + exports, + faults: deduplicated(project, faults), + modules: None, + }); + } + Err(MacroAnalysisFailure::Native(error)) => return Err(error), + Err(MacroAnalysisFailure::Compile(message)) => message, + }; + let collisions: Vec<(String, Vec)> = module_path_collisions(project, &remaining); + if collisions.is_empty() { + let file_index: usize = fault_file(project, &remaining, &message); + let posix: &str = &project.sources[file_index].relative_path.posix; + faults.push(MacroFault { + file_index, + message, + }); + remaining.retain(|index| project.sources[*index].relative_path.posix != posix); + continue; + } + let mut collided: HashSet<&str> = HashSet::new(); + for (name, files) in &collisions { + let paths: Vec<&str> = files + .iter() + .map(|index| project.sources[*index].relative_path.display.as_str()) + .collect(); + let paths: String = paths.join(" and "); + for index in files { + collided.insert(&project.sources[*index].relative_path.posix); + faults.push(MacroFault { + file_index: *index, + message: format!("Macro module path collision for '{name}': {paths}"), + }); + } + } + remaining.retain(|index| { + !collided.contains(project.sources[*index].relative_path.posix.as_str()) + }); + } + Ok(MacroModuleInventory { + exports: Vec::new(), + faults: deduplicated(project, faults), + modules: None, + }) +} + +/// The inventory of every module, with the load plan when the strict analysis passes. +pub(crate) fn project_inventory( + project: &ProjectSources<'_>, +) -> Result { + let all: Vec = (0..project.sources.len()).collect(); + match analyze_project(project, &all) { + Ok(analyses) => { + let (exports, faults) = inventory_from_analyses(project, &analyses); + let modules: Vec = analyses + .into_iter() + .map(|analysis| MacroModuleLoad { + file_index: analysis.file_index, + module_name: analysis.name, + module_dependencies: analysis.module_dependencies, + }) + .collect(); + Ok(MacroModuleInventory { + exports, + faults, + modules: Some(modules), + }) + } + Err(MacroAnalysisFailure::Native(error)) => Err(error), + Err(MacroAnalysisFailure::Compile(message)) => tolerant_inventory(project, message), + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs b/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs new file mode 100644 index 0000000000..3f08569e4b --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/_helpers/python_nodes.rs @@ -0,0 +1,397 @@ +//! The ruff tree as CPython `ast` nodes in `iter_child_nodes` order, leaf nodes left out. + +use ruff_python_ast::{ + Arguments, Comprehension, Decorator, ElifElseClause, ExceptHandler, ExceptHandlerExceptHandler, + Expr, ExprName, FStringPart, InterpolatedElement, InterpolatedStringElement, + InterpolatedStringElements, InterpolatedStringFormatSpec, Keyword, MatchCase, ModModule, + Parameter, Parameters, Pattern, Stmt, TypeParam, TypeParams, WithItem, +}; +use std::collections::VecDeque; +use std::iter::once; + +/// One CPython `ast` node over the ruff tree. +#[derive(Clone, Copy)] +pub(crate) enum PyNode<'a> { + Module(&'a ModModule), + Stmt(&'a Stmt), + Expr(&'a Expr), + /// An `If` whose `orelse` CPython nests from ruff's `elif`/`else` clauses. + IfChain(&'a Expr, &'a [Stmt], &'a [ElifElseClause]), + Arguments(&'a Parameters), + Arg(&'a Parameter), + Keyword(&'a Keyword), + Comprehension(&'a Comprehension), + WithItem(&'a WithItem), + MatchCase(&'a MatchCase), + Pattern(&'a Pattern), + Handler(&'a ExceptHandlerExceptHandler), + TypeParam(&'a TypeParam), + /// A `FormattedValue` or `Interpolation`. + Formatted(&'a InterpolatedElement), + /// The `JoinedStr` of a format spec. + FormatSpec(&'a InterpolatedStringFormatSpec), +} + +impl<'a> PyNode<'a> { + /// The `ExprName` this node is, if it is a `Name`. + pub(crate) fn name(self) -> Option<&'a ExprName> { + match self { + Self::Expr(Expr::Name(name)) => Some(name), + _ => None, + } + } + + /// Whether a depth-first walk of runtime nodes stops here: nested functions, lambdas, classes. + pub(crate) fn opens_scope(self) -> bool { + matches!( + self, + Self::Stmt(Stmt::FunctionDef(_) | Stmt::ClassDef(_)) | Self::Expr(Expr::Lambda(_)) + ) + } + + /// The children `ast.iter_child_nodes` yields, in order, without leaf nodes. + pub(crate) fn children(self) -> Vec> { + match self { + Self::Module(module) => statements(&module.body).collect(), + Self::Stmt(statement) => statement_children(statement), + Self::Expr(expression) => expression_children(expression), + Self::IfChain(test, body, clauses) => if_children((test, body, clauses)), + Self::Arguments(parameters) => argument_children(parameters), + Self::Arg(parameter) => optional(parameter.annotation.as_deref()).collect(), + Self::Keyword(keyword) => vec![Self::Expr(&keyword.value)], + Self::Comprehension(comprehension) => [ + Self::Expr(&comprehension.target), + Self::Expr(&comprehension.iter), + ] + .into_iter() + .chain(expressions(&comprehension.ifs)) + .collect(), + Self::WithItem(item) => once(Self::Expr(&item.context_expr)) + .chain(optional(item.optional_vars.as_deref())) + .collect(), + Self::MatchCase(case) => once(Self::Pattern(&case.pattern)) + .chain(optional(case.guard.as_deref())) + .chain(statements(&case.body)) + .collect(), + Self::Pattern(pattern) => pattern_children(pattern), + Self::Handler(handler) => optional(handler.type_.as_deref()) + .chain(statements(&handler.body)) + .collect(), + Self::TypeParam(parameter) => type_param_children(parameter), + Self::Formatted(element) => once(Self::Expr(&element.expression)) + .chain(element.format_spec.as_deref().map(Self::FormatSpec)) + .collect(), + Self::FormatSpec(spec) => interpolations(&spec.elements).collect(), + } + } + + /// Every node `ast.walk` yields from this one, in its breadth-first order. + pub(crate) fn walk(self) -> Vec> { + let mut found: Vec> = Vec::new(); + let mut pending: VecDeque> = VecDeque::from([self]); + while let Some(node) = pending.pop_front() { + pending.extend(node.children()); + found.push(node); + } + found + } +} + +fn statements<'a>(body: &'a [Stmt]) -> impl Iterator> { + body.iter().map(PyNode::Stmt) +} + +fn expressions<'a>(values: &'a [Expr]) -> impl Iterator> { + values.iter().map(PyNode::Expr) +} + +fn optional(value: Option<&Expr>) -> impl Iterator> { + value.into_iter().map(PyNode::Expr) +} + +fn type_params(parameters: Option<&TypeParams>) -> impl Iterator> { + parameters + .into_iter() + .flat_map(|parameters| parameters.type_params.iter().map(PyNode::TypeParam)) +} + +fn decorators(decorators: &[Decorator]) -> impl Iterator> { + decorators + .iter() + .map(|decorator| PyNode::Expr(&decorator.expression)) +} + +fn handlers(handlers: &[ExceptHandler]) -> impl Iterator> { + handlers + .iter() + .map(|ExceptHandler::ExceptHandler(handler)| PyNode::Handler(handler)) +} + +fn class_bases(arguments: Option<&Arguments>) -> Vec> { + arguments.map_or_else(Vec::new, |arguments| { + expressions(&arguments.args) + .chain(arguments.keywords.iter().map(PyNode::Keyword)) + .collect() + }) +} + +fn statement_children(statement: &Stmt) -> Vec> { + match statement { + Stmt::FunctionDef(function) => once(PyNode::Arguments(&function.parameters)) + .chain(statements(&function.body)) + .chain(decorators(&function.decorator_list)) + .chain(optional(function.returns.as_deref())) + .chain(type_params(function.type_params.as_deref())) + .collect(), + Stmt::ClassDef(class) => class_bases(class.arguments.as_deref()) + .into_iter() + .chain(statements(&class.body)) + .chain(decorators(&class.decorator_list)) + .chain(type_params(class.type_params.as_deref())) + .collect(), + Stmt::Return(statement) => optional(statement.value.as_deref()).collect(), + Stmt::Delete(statement) => expressions(&statement.targets).collect(), + Stmt::TypeAlias(statement) => once(PyNode::Expr(&statement.name)) + .chain(type_params(statement.type_params.as_deref())) + .chain(once(PyNode::Expr(&statement.value))) + .collect(), + Stmt::Assign(statement) => expressions(&statement.targets) + .chain(once(PyNode::Expr(&statement.value))) + .collect(), + Stmt::AugAssign(statement) => vec![ + PyNode::Expr(&statement.target), + PyNode::Expr(&statement.value), + ], + Stmt::AnnAssign(statement) => [ + PyNode::Expr(&statement.target), + PyNode::Expr(&statement.annotation), + ] + .into_iter() + .chain(optional(statement.value.as_deref())) + .collect(), + Stmt::For(statement) => [ + PyNode::Expr(&statement.target), + PyNode::Expr(&statement.iter), + ] + .into_iter() + .chain(statements(&statement.body)) + .chain(statements(&statement.orelse)) + .collect(), + Stmt::While(statement) => once(PyNode::Expr(&statement.test)) + .chain(statements(&statement.body)) + .chain(statements(&statement.orelse)) + .collect(), + Stmt::If(statement) => if_children(( + &statement.test, + &statement.body, + &statement.elif_else_clauses, + )), + Stmt::With(statement) => statement + .items + .iter() + .map(PyNode::WithItem) + .chain(statements(&statement.body)) + .collect(), + Stmt::Match(statement) => once(PyNode::Expr(&statement.subject)) + .chain(statement.cases.iter().map(PyNode::MatchCase)) + .collect(), + Stmt::Raise(statement) => optional(statement.exc.as_deref()) + .chain(optional(statement.cause.as_deref())) + .collect(), + Stmt::Try(statement) => statements(&statement.body) + .chain(handlers(&statement.handlers)) + .chain(statements(&statement.orelse)) + .chain(statements(&statement.finalbody)) + .collect(), + Stmt::Assert(statement) => once(PyNode::Expr(&statement.test)) + .chain(optional(statement.msg.as_deref())) + .collect(), + Stmt::Expr(statement) => vec![PyNode::Expr(&statement.value)], + Stmt::Import(_) + | Stmt::ImportFrom(_) + | Stmt::Global(_) + | Stmt::Nonlocal(_) + | Stmt::Pass(_) + | Stmt::Break(_) + | Stmt::Continue(_) + | Stmt::IpyEscapeCommand(_) => Vec::new(), + } +} + +/// An `If`'s test and body, then its `orelse`: the next `elif` as a nested `If`, or `else` body. +fn if_children<'a>( + (test, body, clauses): (&'a Expr, &'a [Stmt], &'a [ElifElseClause]), +) -> Vec> { + let orelse: Vec> = clauses + .split_first() + .map_or_else(Vec::new, |(clause, rest)| match &clause.test { + Some(elif_test) => vec![PyNode::IfChain(elif_test, &clause.body, rest)], + None => statements(&clause.body).collect(), + }); + once(PyNode::Expr(test)) + .chain(statements(body)) + .chain(orelse) + .collect() +} + +/// `posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults`. +fn argument_children(parameters: &Parameters) -> Vec> { + let positional = || parameters.posonlyargs.iter().chain(¶meters.args); + positional() + .map(|parameter| PyNode::Arg(¶meter.parameter)) + .chain(parameters.vararg.as_deref().map(PyNode::Arg)) + .chain( + parameters + .kwonlyargs + .iter() + .map(|parameter| PyNode::Arg(¶meter.parameter)), + ) + .chain( + parameters + .kwonlyargs + .iter() + .filter_map(|parameter| parameter.default.as_deref().map(PyNode::Expr)), + ) + .chain(parameters.kwarg.as_deref().map(PyNode::Arg)) + .chain(positional().filter_map(|parameter| parameter.default.as_deref().map(PyNode::Expr))) + .collect() +} + +fn interpolations(elements: &InterpolatedStringElements) -> impl Iterator> { + elements.iter().filter_map(|element| match element { + InterpolatedStringElement::Interpolation(interpolation) => { + Some(PyNode::Formatted(interpolation)) + } + InterpolatedStringElement::Literal(_) => None, + }) +} + +fn fstring_parts(part: &FStringPart) -> Vec> { + match part { + FStringPart::FString(fstring) => interpolations(&fstring.elements).collect(), + FStringPart::Literal(_) => Vec::new(), + } +} + +fn comprehension_children<'a>( + leading: Vec>, + generators: &'a [Comprehension], +) -> Vec> { + leading + .into_iter() + .chain(generators.iter().map(PyNode::Comprehension)) + .collect() +} + +fn expression_children(expression: &Expr) -> Vec> { + match expression { + Expr::BoolOp(node) => expressions(&node.values).collect(), + Expr::Named(node) => vec![PyNode::Expr(&node.target), PyNode::Expr(&node.value)], + Expr::BinOp(node) => vec![PyNode::Expr(&node.left), PyNode::Expr(&node.right)], + Expr::UnaryOp(node) => vec![PyNode::Expr(&node.operand)], + Expr::Lambda(node) => node + .parameters + .as_deref() + .map(PyNode::Arguments) + .into_iter() + .chain(once(PyNode::Expr(&node.body))) + .collect(), + Expr::If(node) => vec![ + PyNode::Expr(&node.test), + PyNode::Expr(&node.body), + PyNode::Expr(&node.orelse), + ], + Expr::Dict(node) => node + .items + .iter() + .filter_map(|item| item.key.as_ref().map(PyNode::Expr)) + .chain(node.items.iter().map(|item| PyNode::Expr(&item.value))) + .collect(), + Expr::Set(node) => expressions(&node.elts).collect(), + Expr::ListComp(node) => { + comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) + } + Expr::SetComp(node) => { + comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) + } + Expr::DictComp(node) => comprehension_children( + optional(node.key.as_deref()) + .chain(once(PyNode::Expr(&node.value))) + .collect(), + &node.generators, + ), + Expr::Generator(node) => { + comprehension_children(vec![PyNode::Expr(&node.elt)], &node.generators) + } + Expr::Await(node) => vec![PyNode::Expr(&node.value)], + Expr::Yield(node) => optional(node.value.as_deref()).collect(), + Expr::YieldFrom(node) => vec![PyNode::Expr(&node.value)], + Expr::Compare(node) => once(PyNode::Expr(&node.left)) + .chain(expressions(&node.comparators)) + .collect(), + Expr::Call(node) => once(PyNode::Expr(&node.func)) + .chain(expressions(&node.arguments.args)) + .chain(node.arguments.keywords.iter().map(PyNode::Keyword)) + .collect(), + Expr::FString(node) => node.value.iter().flat_map(fstring_parts).collect(), + Expr::TString(node) => node + .value + .iter() + .flat_map(|tstring| interpolations(&tstring.elements)) + .collect(), + Expr::Attribute(node) => vec![PyNode::Expr(&node.value)], + Expr::Subscript(node) => vec![PyNode::Expr(&node.value), PyNode::Expr(&node.slice)], + Expr::Starred(node) => vec![PyNode::Expr(&node.value)], + Expr::List(node) => expressions(&node.elts).collect(), + Expr::Tuple(node) => expressions(&node.elts).collect(), + Expr::Slice(node) => optional(node.lower.as_deref()) + .chain(optional(node.upper.as_deref())) + .chain(optional(node.step.as_deref())) + .collect(), + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::Name(_) + | Expr::IpyEscapeCommand(_) => Vec::new(), + } +} + +fn pattern_children(pattern: &Pattern) -> Vec> { + match pattern { + Pattern::MatchValue(node) => vec![PyNode::Expr(&node.value)], + Pattern::MatchSequence(node) => node.patterns.iter().map(PyNode::Pattern).collect(), + Pattern::MatchMapping(node) => expressions(&node.keys) + .chain(node.patterns.iter().map(PyNode::Pattern)) + .collect(), + Pattern::MatchClass(node) => once(PyNode::Expr(&node.cls)) + .chain(node.arguments.patterns.iter().map(PyNode::Pattern)) + .chain( + node.arguments + .keywords + .iter() + .map(|keyword| PyNode::Pattern(&keyword.pattern)), + ) + .collect(), + Pattern::MatchAs(node) => node + .pattern + .as_deref() + .map(PyNode::Pattern) + .into_iter() + .collect(), + Pattern::MatchOr(node) => node.patterns.iter().map(PyNode::Pattern).collect(), + Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => Vec::new(), + } +} + +fn type_param_children(parameter: &TypeParam) -> Vec> { + match parameter { + TypeParam::TypeVar(node) => optional(node.bound.as_deref()) + .chain(optional(node.default.as_deref())) + .collect(), + TypeParam::TypeVarTuple(node) => optional(node.default.as_deref()).collect(), + TypeParam::ParamSpec(node) => optional(node.default.as_deref()).collect(), + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/constants.rs b/crates/sqlbuild-pysource/src/macro_modules/constants.rs new file mode 100644 index 0000000000..2fdb0bd3ab --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/constants.rs @@ -0,0 +1,14 @@ +//! Names and separators of project macro module analysis. + +/// Separator of dotted module names. +pub(crate) const MODULE_SEPARATOR: &str = "."; +/// The mark that starts a path suffix. +pub(crate) const SUFFIX_MARK: char = '.'; +/// The alias name of a star import. +pub(crate) const STAR_IMPORT_NAME: &str = "*"; +/// The prefix of private names. +pub(crate) const PRIVATE_PREFIX: &str = "_"; +/// Characters that end the anchor part of an absolute path. +pub(crate) const PATH_ROOTS: [char; 2] = ['/', '\\']; +/// Separator of the names in a reported call or import cycle. +pub(crate) const CYCLE_SEPARATOR: &str = " -> "; diff --git a/crates/sqlbuild-pysource/src/macro_modules/errors.rs b/crates/sqlbuild-pysource/src/macro_modules/errors.rs new file mode 100644 index 0000000000..683618e995 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/errors.rs @@ -0,0 +1,30 @@ +//! The native failure of reading a macro module CPython accepted. + +use std::fmt; + +/// The ruff parser rejected a module that CPython parsed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroModuleParseError { + pub relative_path: String, + pub message: String, +} + +impl fmt::Display for MacroModuleParseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "Native macro module analysis cannot parse '{}', which CPython accepted: {}", + self.relative_path, self.message + ) + } +} + +impl std::error::Error for MacroModuleParseError {} + +/// Why one strict analysis of a set of macro modules stopped. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum MacroAnalysisFailure { + /// The message of the `CompileInputError` Python's analysis raised first. + Compile(String), + Native(MacroModuleParseError), +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/inventory_macro_modules.rs b/crates/sqlbuild-pysource/src/macro_modules/main/inventory_macro_modules.rs new file mode 100644 index 0000000000..4d23544fc7 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/main/inventory_macro_modules.rs @@ -0,0 +1,12 @@ +//! The static inventory of project macro modules, as Python's AST analysis produced it. + +use crate::macro_modules::_helpers::project_analysis::{ProjectSources, project_inventory}; +use crate::macro_modules::errors::MacroModuleParseError; +use crate::macro_modules::models::{MacroModuleInventory, MacroModuleSource}; + +/// Exports, faults and load plan of the modules, or the ruff failure on a CPython-valid source. +pub fn inventory_macro_modules( + sources: &[MacroModuleSource], +) -> Result { + project_inventory(&ProjectSources::new(sources)) +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs new file mode 100644 index 0000000000..bf4f23865d --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/main/mod.rs @@ -0,0 +1 @@ +pub mod inventory_macro_modules; diff --git a/crates/sqlbuild-pysource/src/macro_modules/mod.rs b/crates/sqlbuild-pysource/src/macro_modules/mod.rs new file mode 100644 index 0000000000..38a9736573 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/macro_modules/models.rs b/crates/sqlbuild-pysource/src/macro_modules/models.rs new file mode 100644 index 0000000000..64f5eea25f --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/models.rs @@ -0,0 +1,66 @@ +//! Project macro modules in, and their static inventory and load order out. + +use crate::macro_modules::types::MacroScopeKind; + +/// A module's `relative_path` as `str()`, `as_posix()` and `parts` render it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroModulePath { + pub display: String, + pub posix: String, + pub parts: Vec, +} + +/// One discovered macro module and the outcome of CPython's parse of it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroModuleSource { + pub relative_path: MacroModulePath, + /// `declaration_root.parts`, when the module has one. + pub declaration_root: Option>, + /// `owning_path.parts`, when the module has one. + pub owning_path: Option>, + pub scope_kind: MacroScopeKind, + pub contents: String, + /// The compile error message of CPython's parse, when CPython rejected the source. + pub syntax_error: Option, +} + +/// One exported macro: a public top-level function of a module. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroExport { + pub name: String, + /// Index of the defining module in the analysed sources. + pub file_index: usize, + pub parameters: Vec, + /// CPython's `lineno` of the `def`. + pub line: usize, + /// SHA-256 hex digest of the module's UTF-8 source. + pub source_digest: String, + /// Names of every exported macro the export calls, directly or through helpers. + pub dependencies: Vec, +} + +/// One static fault and the module it is attributed to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroFault { + pub file_index: usize, + pub message: String, +} + +/// A module the loader executes, with the project modules it imports. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroModuleLoad { + pub file_index: usize, + pub module_name: String, + pub module_dependencies: Vec, +} + +/// Exports and faults of every module, and the load plan when the whole project analysed cleanly. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MacroModuleInventory { + /// Collision-free exports sorted by name, then path. + pub exports: Vec, + /// Faults in discovery order, the first being the error a load raises. + pub faults: Vec, + /// Every module in discovery order, present only when the strict analysis passed. + pub modules: Option>, +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/tests.rs b/crates/sqlbuild-pysource/src/macro_modules/tests.rs new file mode 100644 index 0000000000..aaad700890 --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_inventory_macro_modules.rs"] +mod inventory_macro_modules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/macro_modules/tests/helpers.rs b/crates/sqlbuild-pysource/src/macro_modules/tests/helpers.rs new file mode 100644 index 0000000000..91f3f7bc4c --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/tests/helpers.rs @@ -0,0 +1,94 @@ +use crate::macro_modules::main::inventory_macro_modules::inventory_macro_modules; +use crate::macro_modules::models::{MacroModulePath, MacroModuleSource}; +use crate::macro_modules::tests::test_types::{ExpectedExport, ExpectedModule, InventoryRows}; +use crate::macro_modules::types::MacroScopeKind; + +fn owned(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// A global-scope module at `path` with `contents` that CPython accepted. +fn source((path, contents): (&str, &str)) -> MacroModuleSource { + MacroModuleSource { + relative_path: MacroModulePath { + display: path.to_owned(), + posix: path.to_owned(), + parts: owned(&path.split('/').collect::>()), + }, + declaration_root: None, + owning_path: None, + scope_kind: MacroScopeKind::Global, + contents: contents.to_owned(), + syntax_error: None, + } +} + +/// The inventory of `sources` as plain rows, or the native parse failure's message. +pub(crate) fn inventory_rows(sources: &[(&str, &str)]) -> Result { + let sources: Vec = sources.iter().copied().map(source).collect(); + let inventory = inventory_macro_modules(&sources).map_err(|error| error.to_string())?; + Ok(( + inventory + .exports + .into_iter() + .map(|export| { + ( + export.name, + export.file_index, + export.parameters, + export.line, + export.dependencies, + ) + }) + .collect(), + inventory + .faults + .into_iter() + .map(|fault| (fault.file_index, fault.message)) + .collect(), + inventory.modules.map(|modules| { + modules + .into_iter() + .map(|module| { + ( + module.file_index, + module.module_name, + module.module_dependencies, + ) + }) + .collect() + }), + )) +} + +/// The expected rows of a test case. +pub(crate) fn expected_rows( + exports: &[ExpectedExport], + faults: &[(usize, &str)], + modules: Option<&[ExpectedModule]>, +) -> InventoryRows { + ( + exports + .iter() + .map(|(name, file, parameters, line, dependencies)| { + ( + (*name).to_owned(), + *file, + owned(parameters), + *line, + owned(dependencies), + ) + }) + .collect(), + faults + .iter() + .map(|(file, message)| (*file, (*message).to_owned())) + .collect(), + modules.map(|modules| { + modules + .iter() + .map(|(file, name, dependencies)| (*file, (*name).to_owned(), owned(dependencies))) + .collect() + }), + ) +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/tests/test_inventory_macro_modules.rs b/crates/sqlbuild-pysource/src/macro_modules/tests/test_inventory_macro_modules.rs new file mode 100644 index 0000000000..c7a2ffdd5d --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/tests/test_inventory_macro_modules.rs @@ -0,0 +1,216 @@ +use crate::macro_modules::tests::helpers::{expected_rows, inventory_rows}; +use crate::macro_modules::tests::test_types::{ + InventoryMacroModulesTestCase, NativeParseErrorTestCase, +}; + +const SHARED: (&str, &str) = ( + "macros/b.py", + "def b():\n pass\n\n\ndef c():\n pass\n", +); + +#[test] +fn given_macro_modules_when_inventoried_then_python_analysis_agrees() { + let test_cases = [ + InventoryMacroModulesTestCase { + description: "a clean project lists exports, signatures, dependencies and load order", + sources: &[ + ( + "macros/shared.py", + "def shared(value, /, other, *rest, key, **options):\n return value\n", + ), + ( + "macros/orders.py", + "from .shared import shared as render\n\n\ndef _helper():\n return \ + render(1)\n\n\n@decorated\nasync \\\n def orders():\n return _helper()\n", + ), + ], + expected_exports: &[ + ("orders", 1, &[], 9, &["shared"]), + ( + "shared", + 0, + &["value", "other", "rest", "key", "options"], + 1, + &[], + ), + ], + expected_faults: &[], + expected_modules: Some(&[ + (0, "macros.shared", &[]), + (1, "macros.orders", &["macros.shared"]), + ]), + }, + InventoryMacroModulesTestCase { + description: "module-level uses report the first name in breadth-first order", + sources: &[ + SHARED, + ( + "macros/a.py", + "from macros.b import b, c\n_Y = [[c], b]\n\n\ndef a():\n return 1\n", + ), + ], + expected_exports: &[("b", 0, &[], 1, &[]), ("c", 0, &[], 5, &[])], + expected_faults: &[( + 1, + "Macro module 'macros/a.py' uses imported macro 'b' outside an ordinary function \ + body; call imported macros directly from module-level macro or helper functions", + )], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "elif branches nest as CPython's orelse", + sources: &[ + SHARED, + ( + "macros/a.py", + "from macros.b import b, c\nif x:\n pass\nelif c:\n pass\nelse:\n \ + b\n\n\ndef a():\n return 1\n", + ), + ], + expected_exports: &[("b", 0, &[], 1, &[]), ("c", 0, &[], 5, &[])], + expected_faults: &[( + 1, + "Macro module 'macros/a.py' uses imported macro 'c' outside an ordinary function \ + body; call imported macros directly from module-level macro or helper functions", + )], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "function bodies report the first dynamic use in depth-first order", + sources: &[ + SHARED, + ( + "macros/a.py", + "from macros.b import b, c\n\n\ndef a():\n return f(b, [c])\n", + ), + ], + expected_exports: &[("b", 0, &[], 1, &[]), ("c", 0, &[], 5, &[])], + expected_faults: &[( + 1, + "Macro module 'macros/a.py' uses imported macro 'b' dynamically; call imported \ + macros directly", + )], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "a faulty module is dropped and the rest are still inventoried", + sources: &[ + ("macros/bad.py", "PUBLIC = 1\n"), + ("macros/good.py", "def good():\n return 1\n"), + ], + expected_exports: &[("good", 1, &[], 1, &[])], + expected_faults: &[( + 0, + "Macro module 'macros/bad.py' declaration 'PUBLIC' must be underscore-private", + )], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "an import cycle faults, then its importer fails once the cycle is gone", + sources: &[ + ( + "macros/a.py", + "from macros.b import b\n\n\ndef a():\n return b()\n", + ), + ( + "macros/b.py", + "from macros.a import a\n\n\ndef b():\n return a()\n", + ), + ], + expected_exports: &[], + expected_faults: &[ + ( + 0, + "Macro import cycle in 'macros/a.py': macros.a -> macros.b -> macros.a", + ), + ( + 1, + "Macro module 'macros/b.py' imports project package 'macros.a' instead of a \ + macro module", + ), + ], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "a local call cycle reachable from an export faults", + sources: &[( + "macros/a.py", + "def _h():\n return a()\n\n\ndef a():\n return _h()\n", + )], + expected_exports: &[], + expected_faults: &[(0, "Macro call cycle in 'macros/a.py': a -> _h -> a")], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "modules sharing a dotted name each fault", + sources: &[ + ("macros/a/b.py", "def first():\n pass\n"), + ("macros/a.b.py", "def second():\n pass\n"), + ("macros/c.py", "def third():\n pass\n"), + ], + expected_exports: &[("third", 2, &[], 1, &[])], + expected_faults: &[ + ( + 0, + "Macro module path collision for 'macros.a.b': macros/a/b.py and macros/a.b.py", + ), + ( + 1, + "Macro module path collision for 'macros.a.b': macros/a/b.py and macros/a.b.py", + ), + ], + expected_modules: None, + }, + InventoryMacroModulesTestCase { + description: "repeated export names fault without blocking the load plan", + sources: &[ + ("macros/a.py", "def same():\n pass\n"), + ("macros/b.py", "def same():\n pass\n"), + ("macros/c.py", "def same():\n pass\n"), + ], + expected_exports: &[], + expected_faults: &[ + ( + 1, + "Macro name collision for 'same': macros/a.py and macros/b.py", + ), + (2, "Macro name collision for 'same' at macros/c.py"), + ], + expected_modules: Some(&[ + (0, "macros.a", &[]), + (1, "macros.b", &[]), + (2, "macros.c", &[]), + ]), + }, + ]; + for test_case in test_cases { + assert_eq!( + inventory_rows(test_case.sources), + Ok(expected_rows( + test_case.expected_exports, + test_case.expected_faults, + test_case.expected_modules, + )), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_source_ruff_rejects_when_inventoried_then_native_error_names_module() { + let test_cases = [NativeParseErrorTestCase { + description: "a source CPython was said to accept but ruff rejects is a native error", + sources: &[("macros/a.py", "def a(:\n")], + expected_message_prefix: "Native macro module analysis cannot parse 'macros/a.py', which \ + CPython accepted: ", + }]; + for test_case in test_cases { + assert!( + inventory_rows(test_case.sources) + .is_err_and(|message| message.starts_with(test_case.expected_message_prefix)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/tests/test_types.rs b/crates/sqlbuild-pysource/src/macro_modules/tests/test_types.rs new file mode 100644 index 0000000000..411c46e67e --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/tests/test_types.rs @@ -0,0 +1,33 @@ +/// One expected export: name, file index, parameters, line and dependencies. +pub(crate) type ExpectedExport = ( + &'static str, + usize, + &'static [&'static str], + usize, + &'static [&'static str], +); + +/// One expected loadable module: file index, module name and imported project modules. +pub(crate) type ExpectedModule = (usize, &'static str, &'static [&'static str]); + +/// The inventory projected to comparable plain values. +pub(crate) type InventoryRows = ( + Vec<(String, usize, Vec, usize, Vec)>, + Vec<(usize, String)>, + Option)>>, +); + +pub(crate) struct InventoryMacroModulesTestCase { + pub(crate) description: &'static str, + /// `(relative path, contents)` of each module, in discovery order. + pub(crate) sources: &'static [(&'static str, &'static str)], + pub(crate) expected_exports: &'static [ExpectedExport], + pub(crate) expected_faults: &'static [(usize, &'static str)], + pub(crate) expected_modules: Option<&'static [ExpectedModule]>, +} + +pub(crate) struct NativeParseErrorTestCase { + pub(crate) description: &'static str, + pub(crate) sources: &'static [(&'static str, &'static str)], + pub(crate) expected_message_prefix: &'static str, +} diff --git a/crates/sqlbuild-pysource/src/macro_modules/types.rs b/crates/sqlbuild-pysource/src/macro_modules/types.rs new file mode 100644 index 0000000000..b5a9a3131f --- /dev/null +++ b/crates/sqlbuild-pysource/src/macro_modules/types.rs @@ -0,0 +1,10 @@ +//! The visibility tier of a macro module, as Python's `ScopeKind` names it. + +/// Who may import a macro module's exported macros. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MacroScopeKind { + Global, + Inherited, + Local, + Private, +} diff --git a/crates/sqlbuild-pysource/src/module_parse/main/mod.rs b/crates/sqlbuild-pysource/src/module_parse/main/mod.rs new file mode 100644 index 0000000000..569c278909 --- /dev/null +++ b/crates/sqlbuild-pysource/src/module_parse/main/mod.rs @@ -0,0 +1 @@ +pub mod parse_python_module; diff --git a/crates/sqlbuild-pysource/src/module_parse/main/parse_python_module.rs b/crates/sqlbuild-pysource/src/module_parse/main/parse_python_module.rs new file mode 100644 index 0000000000..a5a710c673 --- /dev/null +++ b/crates/sqlbuild-pysource/src/module_parse/main/parse_python_module.rs @@ -0,0 +1,8 @@ +use ruff_python_ast::ModModule; + +/// The module `source` parses to, or the parser's message. +pub fn parse_python_module(source: &str) -> Result { + ruff_python_parser::parse_module(source) + .map(ruff_python_parser::Parsed::into_syntax) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-pysource/src/module_parse/mod.rs b/crates/sqlbuild-pysource/src/module_parse/mod.rs new file mode 100644 index 0000000000..2a043412b4 --- /dev/null +++ b/crates/sqlbuild-pysource/src/module_parse/mod.rs @@ -0,0 +1 @@ +pub mod main; diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 52e5fe98d2..292ccdc015 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -28,6 +28,7 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-pysource.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true sqlbuild-rules.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs new file mode 100644 index 0000000000..3cb764a981 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -0,0 +1,553 @@ +//! The compile analysis sequence: discovery, graph, lineage, contracts, selection and rules. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + Status, call_with, imported, keywords, within, +}; + +const DISCOVER_MODULE: &str = "sqlbuild.compiler.discovery.main.discover"; +const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; +const LINEAGE_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.lineage"; +const CONTRACTS_MODULE: &str = "sqlbuild.compiler.contracts.main.validate"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; +const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; +const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; +const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const RULE_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const EARLY_LINT_MODULE: &str = "sqlbuild.rule_engine.classes.early_sql_lint"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild.cli.compile.models"; +const GENERIC_DIALECT: &str = "generic"; +const TARGET_DIRECTORY: &str = "target"; + +/// Milliseconds since `start`, truncated as Python's `int((monotonic() - start) * 1000)`. +fn elapsed_ms(start: Instant) -> u64 { + (start.elapsed().as_secs_f64() * 1000.0) as u64 +} + +/// The compile command's options, read by keyword as Python's signature named them. +struct AnalysisOptions<'py> { + project_dir: Bound<'py, PyAny>, + no_sql_validation: Bound<'py, PyAny>, + selected_target: Bound<'py, PyAny>, + lineage_mode: Bound<'py, PyAny>, + cli_vars: Bound<'py, PyAny>, + profile_flags: Bound<'py, PyAny>, + analysis_selection: Bound<'py, PyAny>, + status: Bound<'py, PyAny>, + prepared_artifacts: Bound<'py, PyAny>, +} + +impl<'py> AnalysisOptions<'py> { + fn read(py: Python<'py>, options: &Bound<'py, PyDict>) -> PyResult { + let required = |name: &str| -> PyResult> { + options.get_item(name)?.ok_or_else(|| { + pyo3::exceptions::PyTypeError::new_err(format!( + "analyze_compile_project() missing 1 required keyword-only argument: '{name}'" + )) + }) + }; + Ok(Self { + project_dir: required("project_dir")?, + no_sql_validation: required("no_sql_validation")?, + selected_target: required("selected_target")?, + lineage_mode: required("lineage_mode")?, + cli_vars: required("cli_vars")?, + profile_flags: required("profile_flags")?, + analysis_selection: required("analysis_selection")?, + status: required("status")?, + prepared_artifacts: options + .get_item("prepared_artifacts")? + .unwrap_or_else(|| py.None().into_bound(py)), + }) + } +} + +/// `{seconds:.2f}`, as Python formats elapsed milliseconds over a thousand. +fn seconds(milliseconds: u64) -> String { + format!("{:.2}", milliseconds as f64 / 1000.0) +} + +/// Whether any of `diagnostics` is an error. +fn any_error(diagnostics: &Bound<'_, PyAny>) -> PyResult { + for diagnostic in diagnostics.try_iter()? { + if diagnostic?.getattr("is_error")?.is_truthy()? { + return Ok(true); + } + } + Ok(false) +} + +/// Discover, compile and validate the project into one `CompileAnalysis`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn analyze_compile_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + analyze(py, &options.unwrap_or_else(|| PyDict::new(py))) +} + +/// `analyze_compile_project` with its keyword options. +pub(crate) fn analyze<'py>( + py: Python<'py>, + options: &Bound<'py, PyDict>, +) -> PyResult> { + let options: AnalysisOptions<'py> = AnalysisOptions::read(py, options)?; + let selection: &Bound<'py, PyAny> = &options.analysis_selection; + let enabled: bool = !(selection.getattr("select")?.is_truthy()? + || selection.getattr("exclude")?.is_truthy()?); + let early_lint_class: Bound<'py, PyAny> = imported(py, EARLY_LINT_MODULE, "EarlySqlLint")?; + let manager: Bound<'py, PyAny> = call_with( + &early_lint_class, + &[("enabled", PyBool::new(py, enabled).to_owned().into_any())], + )?; + within(&manager, |early_lint| analyzed(py, &options, early_lint)) +} + +/// The analysis phases sharing the command options, status line and early SQL lint. +struct Driver<'a, 'py> { + py: Python<'py>, + options: &'a AnalysisOptions<'py>, + status: Status<'py>, + early_lint: &'a Bound<'py, PyAny>, + select: Bound<'py, PyAny>, + exclude: Bound<'py, PyAny>, +} + +/// Contract validation's result with its wall and thread CPU milliseconds. +struct ContractPhase<'py> { + result: Bound<'py, PyAny>, + wall_ms: u64, + cpu_ms: i64, +} + +/// `_analyze_compile_project`. +fn analyzed<'py>( + py: Python<'py>, + options: &AnalysisOptions<'py>, + early_lint: &Bound<'py, PyAny>, +) -> PyResult> { + let driver: Driver<'_, 'py> = Driver { + py, + options, + status: Status(options.status.clone()), + early_lint, + select: options.analysis_selection.getattr("select")?, + exclude: options.analysis_selection.getattr("exclude")?, + }; + let (discovered, discover_ms): (Bound<'py, PyAny>, u64) = driver.discover()?; + let adapter: Bound<'py, PyAny> = driver.adapter(&discovered)?; + let dialect: Bound<'py, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let lint_dialect: Bound<'py, PyAny> = if dialect.is_truthy()? { + dialect.clone() + } else { + PyString::new(py, GENERIC_DIALECT).into_any() + }; + let (graph, graph_ms): (Bound<'py, PyAny>, u64) = + driver.graph((&discovered, &adapter), &lint_dialect)?; + let project: Bound<'py, PyAny> = graph.getattr("project")?; + let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; + let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); + if graph_failed { + early_lint_wait_ms = early_lint.call_method0("stop")?; + } + let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; + let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; + let selected_keys: Bound<'py, PyAny> = call_with( + &imported(py, SELECTION_MODULE, "select_project_graph")?, + &[ + ("graph", graph.clone()), + ("select", driver.select.clone()), + ("exclude", driver.exclude.clone()), + ], + )?; + let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + project + .getattr("diagnostics")? + .try_iter()? + .chain(contracts.result.getattr("diagnostics")?.try_iter()?) + .collect::>>()?, + )?; + let rules_result: Bound<'py, PyAny> = if graph_failed { + empty_rules_result(py)? + } else { + let result: Bound<'py, PyAny> = driver.rules( + (&graph, &discovered, &adapter), + (&core_diagnostics, &selected_keys), + &lint_dialect, + )?; + early_lint_wait_ms = early_lint.call_method0("stop")?; + result + }; + let diagnostics: Bound<'py, PyTuple> = + with_rule_diagnostics(py, &core_diagnostics, &rules_result)?; + call_with( + &imported(py, CLI_COMPILE_MODELS_MODULE, "CompileAnalysis")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter), + ("graph", graph), + ("selected_keys", selected_keys), + ("lineage", lineage), + ("diagnostics", diagnostics.into_any()), + ("discover_ms", discover_ms.into_pyobject(py)?.into_any()), + ("graph_ms", graph_ms.into_pyobject(py)?.into_any()), + ("lineage_ms", lineage_ms.into_pyobject(py)?.into_any()), + ( + "contract_ms", + contracts.wall_ms.into_pyobject(py)?.into_any(), + ), + ( + "contract_cpu_ms", + contracts.cpu_ms.into_pyobject(py)?.into_any(), + ), + ("built_in_rules_ms", rules_result.getattr("built_in_ms")?), + ("custom_rules_ms", rules_result.getattr("custom_ms")?), + ( + "custom_rules_cpu_ms", + rules_result.getattr("custom_cpu_ms")?, + ), + ("early_lint_wait_ms", early_lint_wait_ms), + ("rule_cache_hits", rules_result.getattr("cache_hits")?), + ("rule_cache_misses", rules_result.getattr("cache_misses")?), + ( + "skipped_type_proof_rules", + rules_result.getattr("skipped_type_proof_rules")?, + ), + ], + ) +} + +/// `RulesRunResult` of a compile whose rules did not run. +fn empty_rules_result(py: Python<'_>) -> PyResult> { + let zero: Bound<'_, PyAny> = 0_i64.into_pyobject(py)?.into_any(); + call_with( + &imported(py, RULE_MODELS_MODULE, "RulesRunResult")?, + &[ + ("findings", PyTuple::empty(py).into_any()), + ("evaluated_models", zero.clone()), + ("built_in_ms", zero.clone()), + ("custom_ms", zero), + ], + ) +} + +impl<'py> Driver<'_, 'py> { + /// Discover the project inputs and reject unit-test selectors. + fn discover(&self) -> PyResult<(Bound<'py, PyAny>, u64)> { + let py: Python<'py> = self.py; + let started: Instant = Instant::now(); + self.status.start("Discovering project...")?; + let false_value: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + let override_analysis: Bound<'py, PyAny> = if self + .options + .profile_flags + .getattr("skip_discovery_sql_analysis")? + .is_truthy()? + { + false_value.clone() + } else { + py.None().into_bound(py) + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, DISCOVER_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", self.options.project_dir.clone()), + ("extract_output_column_locations", false_value), + ("sql_analysis_enabled_override", override_analysis), + ], + )?; + call_with( + &imported(py, REJECT_SELECTORS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", self.select.clone()), + ("exclude", self.exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let discover_ms: u64 = elapsed_ms(started); + self.status + .complete(&format!("Discovered project. ({}s)", seconds(discover_ms)))?; + Ok((discovered, discover_ms)) + } + + /// The adapter the project and local configuration select. + fn adapter(&self, discovered: &Bound<'py, PyAny>) -> PyResult> { + let adapter_name: Bound<'py, PyAny> = call_with( + &imported( + self.py, + ADAPTER_NAME_MODULE, + "resolve_effective_adapter_name", + )?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + call_with( + &imported(self.py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", self.options.project_dir.clone()), + ], + ) + } + + /// Compile the project graph, starting the early SQL lint once inputs are ready. + fn graph( + &self, + (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + lint_dialect: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, u64)> { + let py: Python<'py> = self.py; + let options: &AnalysisOptions<'py> = self.options; + let started: Instant = Instant::now(); + self.status.start("Compiling project graph...")?; + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "project").into_any()), + ( + "operation_name", + PyString::new(py, "project_compile").into_any(), + ), + ], + )?; + let on_inputs_ready: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( + (self.early_lint.getattr("start")?,), + Some(&keywords(py, &[("dialect", lint_dialect.clone())])?), + )?; + let analysis_selection: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?.call( + (options.analysis_selection.clone(),), + Some(&keywords(py, &[("on_inputs_ready", on_inputs_ready)])?), + )?; + let lineage_mode: Bound<'py, PyAny> = + imported(py, LINEAGE_MODULE, "compile_analysis_lineage_mode")? + .call1((options.lineage_mode.clone(),))?; + let graph: Bound<'py, PyAny> = within(&lifecycle, |_| { + call_with( + &imported( + py, + SELECTED_GRAPH_MODULE, + "build_project_graph_with_analysis_selection", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.selected_target.clone()), + ("no_sql_validation", options.no_sql_validation.clone()), + ( + "skip_column_inference", + options.profile_flags.getattr("skip_column_inference")?, + ), + ("column_lineage_mode", lineage_mode), + ("cli_vars", options.cli_vars.clone()), + ("analysis_selection", analysis_selection), + ], + ) + })?; + let graph_ms: u64 = elapsed_ms(started); + self.status + .complete(&format!("Compiled project graph. ({}s)", seconds(graph_ms)))?; + Ok((graph, graph_ms)) + } + + /// Column lineage in the requested mode. + fn lineage( + &self, + graph: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyAny>, u64)> { + let started: Instant = Instant::now(); + self.status.start("Analyzing column lineage...")?; + let lineage: Bound<'py, PyAny> = call_with( + &imported(self.py, LINEAGE_MODULE, "build_compile_lineage")?, + &[ + ("graph", graph.clone()), + ("dialect", dialect.clone()), + ("mode", self.options.lineage_mode.clone()), + ], + )?; + let lineage_ms: u64 = elapsed_ms(started); + self.status.complete(&format!( + "Analyzed column lineage. ({}s)", + seconds(lineage_ms) + ))?; + Ok((lineage, lineage_ms)) + } + + /// Model contract validation, unless profiling skips it. + fn contracts( + &self, + project: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = self.py; + let thread_time: Bound<'py, PyAny> = imported(py, "time", "thread_time")?; + let started: Instant = Instant::now(); + let cpu_started: f64 = thread_time.call0()?.extract()?; + let skip: bool = self + .options + .profile_flags + .getattr("skip_contracts")? + .is_truthy()?; + let result: Bound<'py, PyAny> = if skip { + call_with( + &imported(py, CONTRACT_MODELS_MODULE, "ContractValidationResult")?, + &[("diagnostics", PyTuple::empty(py).into_any())], + )? + } else { + self.status.start("Validating model contracts...")?; + call_with( + &imported(py, CONTRACTS_MODULE, "evaluate_model_contracts")?, + &[("project", project.clone()), ("dialect", dialect.clone())], + )? + }; + let wall_ms: u64 = elapsed_ms(started); + let cpu_ended: f64 = thread_time.call0()?.extract()?; + let cpu_ms: i64 = ((cpu_ended - cpu_started) * 1000.0) as i64; + if !skip { + self.status.complete(&format!( + "Validated model contracts. ({}s)", + seconds(wall_ms) + ))?; + } + Ok(ContractPhase { + result, + wall_ms, + cpu_ms, + }) + } + + /// Stage artifacts when the compile is clean, then run the built-in and custom rules. + fn rules( + &self, + (graph, discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (core_diagnostics, selected_keys): (&Bound<'py, PyTuple>, &Bound<'py, PyAny>), + lint_dialect: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = self.py; + let options: &AnalysisOptions<'py> = self.options; + let project_dir: &Bound<'py, PyAny> = &options.project_dir; + let rules_config: Bound<'py, PyAny> = call_with( + &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { + let target_dir: Bound<'py, PyAny> = + project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; + let stage_artifacts: bool = rules_config.getattr("select")?.is_truthy()?; + call_with( + &options.prepared_artifacts.getattr("start")?, + &[ + ("project", graph.getattr("project")?), + ("adapter", adapter.clone()), + ("target_dir", target_dir), + ( + "stage_artifacts", + PyBool::new(py, stage_artifacts).to_owned().into_any(), + ), + ], + )?; + } + self.status + .start("Evaluating built-in and custom rules...")?; + let monotonic: Bound<'py, PyAny> = imported(py, "time", "monotonic")?; + let started: f64 = monotonic.call0()?.extract()?; + let selected: Bound<'py, PyAny> = if self.select.is_truthy()? || self.exclude.is_truthy()? { + selected_keys.clone() + } else { + py.None().into_bound(py) + }; + let result: Bound<'py, PyAny> = call_with( + &imported(py, RUN_RULES_MODULE, "run_rules")?, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", rules_config), + ("project_dir", project_dir.clone()), + ("dialect", lint_dialect.clone()), + ("selected_keys", selected), + ("prepared_sql", self.early_lint.getattr("preparation")?), + ( + "expansion_reuse", + self.early_lint.getattr("expansion_reuse")?, + ), + ("no_sql_analysis", options.no_sql_validation.clone()), + ], + )?; + let ended: f64 = monotonic.call0()?.extract()?; + let progress: String = call_with( + &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, + &[ + ( + "elapsed_seconds", + (ended - started).into_pyobject(py)?.into_any(), + ), + ("result", result.clone()), + ], + )? + .extract()?; + self.status.complete(&progress)?; + Ok(result) + } +} + +/// The core diagnostics followed by one rule-phase error per rule finding. +fn with_rule_diagnostics<'py>( + py: Python<'py>, + core: &Bound<'py, PyTuple>, + rules_result: &Bound<'py, PyAny>, +) -> PyResult> { + let diagnostic_class: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?; + let phase: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("RULE")?; + let severity: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; + let fix_note: Bound<'py, PyAny> = + imported(py, RULE_CONSTANTS_MODULE, "RULE_FIX_AVAILABLE_NOTE")?; + let mut diagnostics: Vec> = core.iter().collect(); + for fault in rules_result.getattr("findings")?.try_iter()? { + let fault: Bound<'py, PyAny> = fault?; + let notes: Bound<'py, PyTuple> = if fault.getattr("fixable")?.is_truthy()? { + PyTuple::new(py, [fix_note.clone()])? + } else { + PyTuple::empty(py) + }; + diagnostics.push(call_with( + &diagnostic_class, + &[ + ("phase", phase.clone()), + ("severity", severity.clone()), + ("code", fault.getattr("code")?), + ("message", fault.getattr("message")?), + ("path", fault.getattr("path")?), + ("line", fault.getattr("line")?), + ("column", fault.getattr("column")?), + ("help", fault.getattr("remediation")?), + ("notes", notes.into_any()), + ("affected_rules", fault.getattr("affected_rules")?), + ], + )?); + } + PyTuple::new(py, diagnostics) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(analyze_compile_project, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs new file mode 100644 index 0000000000..f93804f240 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -0,0 +1,492 @@ +//! The `sqb compile` command: analysis, output phases, reuse recording and printed output. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyStringMethods, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::analysis::analyze; +use crate::bindings::_helpers::compile_driver::phases::{ + elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, + target_dir, write_artifacts, write_dag_artifact, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + Status, call_with, imported, keywords, within, +}; + +const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; +const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; +const RECORDED_OUTPUT_MODULE: &str = "sqlbuild.cli.compile_reuse.classes.recorded_compile_output"; +const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; +const REUSE_TIMINGS_MODULE: &str = "sqlbuild.cli.compile_reuse.main._compile_reuse_timings"; +const WRITE_REUSABLE_MODULE: &str = "sqlbuild.cli.compile_reuse.main._write_reusable_compile"; +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; +const PREPARED_ARTIFACTS_MODULE: &str = "sqlbuild.cli.commands.classes.prepared_compile_artifacts"; +const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; +const SEMANTIC_NOTICE_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.semantic_notice"; +const OUTPUT_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.output"; +const SUPPORTS_COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; + +/// What one full compile reports for reuse recording. +struct CommandOutcome<'py> { + exit_code: i64, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Bound<'py, PyAny>, +} + +/// The per-invocation recorders a compile runs inside. +struct Recorders<'py> { + detailed_timings: Bound<'py, PyAny>, + input_reads: Bound<'py, PyAny>, + artifact_writes: Bound<'py, PyAny>, + prepared_artifacts: Bound<'py, PyAny>, +} + +/// Run one full compile, recording it for reuse; returns the exit code. +#[pyfunction] +#[pyo3(signature = (request, reuse_attempt=None))] +fn run_compile_command<'py>( + py: Python<'py>, + request: Bound<'py, PyAny>, + reuse_attempt: Option>, +) -> PyResult { + let total_start: f64 = match &reuse_attempt { + Some(attempt) => attempt.getattr("started")?.extract()?, + None => monotonic(py)?, + }; + let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; + let effective_request: Bound<'py, PyAny> = if request.getattr("project_dir")?.is_none() { + imported(py, "dataclasses", "replace")?.call( + (request.clone(),), + Some(&keywords( + py, + &[("project_dir", path_class.call_method0("cwd")?)], + )?), + )? + } else { + request.clone() + }; + let attempt: Bound<'py, PyAny> = match reuse_attempt { + Some(attempt) => attempt, + None => bypass_attempt(py, &effective_request, total_start)?, + }; + let output: Bound<'py, PyAny> = + imported(py, RECORDED_OUTPUT_MODULE, "RecordedCompileOutput")?.call0()?; + let status: Status<'py> = Status(compile_status(&request)?); + let gc: Bound<'py, PyModule> = py.import("gc")?; + let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; + gc.call_method0("disable")?; + let exit_code: PyResult = recorded_compile( + (&effective_request, &attempt), + (&output, &status), + total_start, + ); + if was_enabled { + gc.call_method0("enable")?; + } + exit_code +} + +/// `CompileReuseAttempt` of a compile that did not consult the reuse record. +fn bypass_attempt<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, + started: f64, +) -> PyResult> { + let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; + let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if project_dir.is_truthy()? { + project_dir + } else { + path_class.call_method0("cwd")? + }; + call_with( + &imported(py, REUSE_MODELS_MODULE, "CompileReuseAttempt")?, + &[ + ( + "outcome", + imported(py, REUSE_TYPES_MODULE, "CompileReuseOutcome")?.getattr("BYPASS")?, + ), + ("project_dir", path_class.call1((project_dir,))?), + ("started", started.into_pyobject(py)?.into_any()), + ], + ) +} + +/// The interactive status line, shown only on a terminal without `--json`. +fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let none: Bound<'py, PyAny> = py.None().into_bound(py); + if request.getattr("json_output")?.is_truthy()? { + return Ok(none); + } + let stdout: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; + if !stdout.hasattr("isatty")? || !stdout.call_method0("isatty")?.is_truthy()? { + return Ok(none); + } + let use_color: bool = !request.getattr("no_color")?.is_truthy()?; + call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[ + ("stream", stdout), + ( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + ), + ], + ) +} + +/// Close the status line when one is shown. +fn close_status(status: &Status<'_>) -> PyResult<()> { + if !status.0.is_none() { + status.0.call_method0("close")?; + } + Ok(()) +} + +/// Compile inside the recorders, then store the reusable compile and print its output. +fn recorded_compile<'py>( + (request, attempt): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (output, status): (&Bound<'py, PyAny>, &Status<'py>), + total_start: f64, +) -> PyResult { + let py: Python<'py> = request.py(); + let skip_write: bool = request + .getattr("profile_flags")? + .getattr("skip_write")? + .is_truthy()?; + let compiled: PyResult<(CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = within( + &imported(py, PROFILING_MODULE, "collect_compile_timings")?.call0()?, + |detailed_timings| { + within( + &imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")? + .call_method0("recording")?, + |input_reads| { + within( + &imported(py, REUSE_CONSTANTS_MODULE, "COMPILE_ARTIFACT_WRITES")? + .call_method0("recording")?, + |artifact_writes| { + let manager: Bound<'py, PyAny> = call_with( + &imported( + py, + PREPARED_ARTIFACTS_MODULE, + "PreparedCompileArtifacts", + )?, + &[( + "enabled", + PyBool::new(py, !skip_write).to_owned().into_any(), + )], + )?; + within(&manager, |prepared_artifacts| { + let recorders: Recorders<'py> = Recorders { + detailed_timings: detailed_timings.clone(), + input_reads: input_reads.clone(), + artifact_writes: artifact_writes.clone(), + prepared_artifacts: prepared_artifacts.clone(), + }; + let reuse_timings: Bound<'py, PyAny> = call_with( + &imported(py, REUSE_TIMINGS_MODULE, "compile_reuse_timings")?, + &[("attempt", attempt.clone())], + )?; + let outcome: CommandOutcome<'py> = compiled_with_status( + (request, &recorders), + (status, output), + (total_start, &reuse_timings), + )?; + Ok((outcome, recorders.input_reads, recorders.artifact_writes)) + }) + }, + ) + }, + ) + }, + ); + close_status(status)?; + let (outcome, input_reads, artifact_writes) = compiled?; + let stored: PyResult> = call_with( + &imported(py, WRITE_REUSABLE_MODULE, "write_reusable_compile")?, + &[ + ("attempt", attempt.clone()), + ("output", output.clone()), + ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), + ("input_reads", input_reads), + ("artifact_writes", artifact_writes), + ( + "compile_cache_enabled", + PyBool::new(py, outcome.compile_cache_enabled) + .to_owned() + .into_any(), + ), + ( + "artifacts_written", + PyBool::new(py, outcome.artifacts_written) + .to_owned() + .into_any(), + ), + ("dag_artifact_path", outcome.dag_artifact_path), + ("json_output", request.getattr("json_output")?), + ], + ); + let printed: PyResult> = output.call_method0("print_stdout"); + printed?; + stored?; + Ok(outcome.exit_code) +} + +/// `_run_compile_with_status`: analyze, run the output phases and record the command output. +fn compiled_with_status<'py>( + (request, recorders): (&Bound<'py, PyAny>, &Recorders<'py>), + (status, output): (&Status<'py>, &Bound<'py, PyAny>), + (total_start, reuse_timings): (f64, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if project_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + project_dir + }; + let prepared_artifacts: &Bound<'py, PyAny> = &recorders.prepared_artifacts; + call_with( + &prepared_artifacts.getattr("remove_abandoned_staging")?, + &[("target_dir", target_dir(&project_dir)?)], + )?; + let analysis: Bound<'py, PyAny> = analyze( + py, + &analysis_options(request, &project_dir, recorders, status)?, + )?; + let rules_failed: bool = rules_failed(&analysis)?; + let skip_write: bool = request + .getattr("profile_flags")? + .getattr("skip_write")? + .is_truthy()?; + let manifest_requested: bool = request.getattr("manifest")?.is_truthy()?; + let manifest: Bound<'py, PyAny> = + manifest_payload(manifest_requested && !rules_failed, &analysis, status)?; + let dag_path: Bound<'py, PyAny> = if rules_failed { + py.None().into_bound(py) + } else { + request.getattr("dag_path")? + }; + write_dag_artifact((&dag_path, &project_dir), &analysis, status)?; + let (written, write_ms): (Bound<'py, PyAny>, i64) = write_artifacts( + (skip_write || rules_failed, &project_dir), + (&analysis, &manifest), + prepared_artifacts, + status, + )?; + let timings: Bound<'py, PyDict> = timings_ms( + (&analysis, write_ms), + (&recorders.detailed_timings, reuse_timings), + total_start, + )?; + let withheld: Bound<'py, PyAny> = if rules_failed && !skip_write { + sql_test_planning_diagnostics(&project_dir, &analysis, prepared_artifacts)? + } else { + PyTuple::empty(py).into_any() + }; + let diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + analysis + .getattr("diagnostics")? + .try_iter()? + .chain(written.getattr("diagnostics")?.try_iter()?) + .chain(withheld.try_iter()?) + .collect::>>()?, + )?; + let mut exit_code: i64 = 0; + for diagnostic in diagnostics.as_any().try_iter()? { + if diagnostic?.getattr("is_error")?.is_truthy()? { + exit_code = 1; + break; + } + } + close_status(status)?; + let skipped_note: Bound<'py, PyAny> = call_with( + &imported(py, SKIPPED_RULES_MODULE, "format_skipped_type_proof_rules")?, + &[("codes", analysis.getattr("skipped_type_proof_rules")?)], + )?; + if !skipped_note.is_none() { + output.call_method1( + "print_stderr_line", + (format!("note: {}", skipped_note.str()?.to_cow()?),), + )?; + } + let dag_artifact_path: Bound<'py, PyAny> = + if rules_failed || request.getattr("dag_path")?.is_none() { + py.None().into_bound(py) + } else { + resolved_dag_path(&project_dir, &request.getattr("dag_path")?)? + }; + let outcome: CommandOutcome<'py> = CommandOutcome { + exit_code, + compile_cache_enabled: !analysis + .getattr("graph")? + .getattr("project")? + .getattr("compile_cache_dir")? + .is_none(), + artifacts_written: !(rules_failed || skip_write), + dag_artifact_path, + }; + record_output( + (request, &analysis), + (&written, &diagnostics, &timings), + output, + )?; + Ok(outcome) +} + +/// The keyword options of the analysis phase. +fn analysis_options<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + recorders: &Recorders<'py>, + status: &Status<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let selection: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("no_cache", request.getattr("no_cache")?), + ], + )?; + keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("selected_target", request.getattr("selected_target")?), + ("lineage_mode", request.getattr("lineage_mode")?), + ("cli_vars", request.getattr("cli_vars")?), + ("profile_flags", request.getattr("profile_flags")?), + ("analysis_selection", selection), + ("status", status.0.clone()), + ("prepared_artifacts", recorders.prepared_artifacts.clone()), + ], + ) +} + +/// Whether a rule finding failed the compile. +fn rules_failed(analysis: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = analysis.py(); + let rule_phase: Bound<'_, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("RULE")?; + for diagnostic in analysis.getattr("diagnostics")?.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? + && diagnostic.getattr("phase")?.is(&rule_phase) + { + return Ok(true); + } + } + Ok(false) +} + +/// The compile's phase timings, in the order `--json` reports them. +fn timings_ms<'py>( + (analysis, write_ms): (&Bound<'py, PyAny>, i64), + (detailed_timings, reuse_timings): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + total_start: f64, +) -> PyResult> { + let py: Python<'py> = analysis.py(); + let timings: Bound<'py, PyDict> = PyDict::new(py); + for (key, attribute) in [ + ("discover_ms", "discover_ms"), + ("graph_ms", "graph_ms"), + ("lineage_ms", "lineage_ms"), + ("contracts_ms", "contract_ms"), + ("contracts_cpu_ms", "contract_cpu_ms"), + ("built_in_rules_ms", "built_in_rules_ms"), + ("custom_rules_ms", "custom_rules_ms"), + ("custom_rules_cpu_ms", "custom_rules_cpu_ms"), + ("early_lint_wait_ms", "early_lint_wait_ms"), + ("rule_cache_hits", "rule_cache_hits"), + ("rule_cache_misses", "rule_cache_misses"), + ] { + timings.set_item(key, analysis.getattr(attribute)?)?; + } + timings.set_item("write_ms", write_ms)?; + timings.update( + detailed_timings + .call_method0("as_milliseconds")? + .downcast::()?, + )?; + timings.update(reuse_timings.downcast::()?)?; + timings.set_item("total_ms", elapsed_ms(py, total_start)?)?; + Ok(timings) +} + +/// Record the command's text or JSON output, with the semantic coverage notice for JSON. +fn record_output<'py>( + (request, analysis): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (written, diagnostics, timings): ( + &Bound<'py, PyAny>, + &Bound<'py, PyTuple>, + &Bound<'py, PyDict>, + ), + output: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = request.py(); + let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; + let selected_keys: Bound<'py, PyAny> = analysis.getattr("selected_keys")?; + let validation_enabled: Bound<'py, PyAny> = + PyBool::new(py, !request.getattr("no_sql_validation")?.is_truthy()?) + .to_owned() + .into_any(); + let shared: [(&str, Bound<'py, PyAny>); 7] = [ + ("graph", graph.clone()), + ("written", written.clone()), + ("manifest", request.getattr("manifest")?), + ("lineage", analysis.getattr("lineage")?), + ("lineage_mode", request.getattr("lineage_mode")?), + ("diagnostics", diagnostics.clone().into_any()), + ("selected_keys", selected_keys.clone()), + ]; + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); + let rendered: Bound<'py, PyAny> = if request.getattr("json_output")?.is_truthy()? { + let notice: Bound<'py, PyAny> = call_with( + &imported(py, SEMANTIC_NOTICE_MODULE, "semantic_coverage_notice")?, + &[ + ("project", graph.getattr("project")?), + ("selected_keys", selected_keys), + ("sql_validation_enabled", validation_enabled.clone()), + ], + )?; + if notice.is_truthy()? { + output.call_method1("print_stderr_line", (notice,))?; + } + arguments.push(("timings_ms", timings.clone().into_any())); + arguments.push(("sql_validation_enabled", validation_enabled)); + call_with( + &imported(py, OUTPUT_MODULE, "format_compile_json")?, + &arguments, + )? + } else { + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && imported(py, SUPPORTS_COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + arguments.push(( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + )); + arguments.push(("sql_validation_enabled", validation_enabled)); + call_with( + &imported(py, OUTPUT_MODULE, "format_compile_text")?, + &arguments, + )? + }; + output.call_method1("record_stdout", (rendered,))?; + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_compile_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs new file mode 100644 index 0000000000..2aa258368d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod analysis; +pub(crate) mod command; +pub(crate) mod phases; +pub(crate) mod project; +pub(crate) mod python_calls; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs new file mode 100644 index 0000000000..ee36509c2c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -0,0 +1,280 @@ +//! The compile output phases: manifest, DAG artifact, compiled artifacts and withheld test errors. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; + +const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const DAG_MODULE: &str = "sqlbuild.compiler.dag.main.build"; +const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const ARTIFACT_BATCH_MODULE: &str = "sqlbuild.cli.commands.classes.native_artifact_batch"; +const TARGET_WRITER_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.target_writer"; +const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const CLI_CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; +const TARGET_DIRECTORY: &str = "target"; +const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; + +/// Seconds since a `time.monotonic()` reading, formatted `{:.2f}`. +fn seconds_since(py: Python<'_>, started: f64) -> PyResult { + let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; + Ok(format!("{:.2}", now - started)) +} + +/// A `time.monotonic()` reading. +pub(crate) fn monotonic(py: Python<'_>) -> PyResult { + imported(py, "time", "monotonic")?.call0()?.extract() +} + +/// Milliseconds since a `time.monotonic()` reading, truncated as `int(... * 1000)`. +pub(crate) fn elapsed_ms(py: Python<'_>, started: f64) -> PyResult { + Ok(((monotonic(py)? - started) * 1000.0) as i64) +} + +/// `project_dir / "target"`. +pub(crate) fn target_dir<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,)) +} + +/// The manifest payload, when requested. +pub(crate) fn manifest_payload<'py>( + requested: bool, + analysis: &Bound<'py, PyAny>, + status: &Status<'py>, +) -> PyResult> { + let py: Python<'py> = analysis.py(); + if !requested { + return Ok(py.None().into_bound(py)); + } + let started: f64 = monotonic(py)?; + status.start("Building manifest...")?; + let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; + let adapter_type: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let payload: Bound<'py, PyAny> = call_with( + &imported(py, MANIFEST_MODULE, "build_graph_manifest")?, + &[ + ("project", graph.getattr("project")?), + ("plan_output", py.None().into_bound(py)), + ("project_name", project_config.getattr("name")?), + ("adapter_type", adapter_type), + ("graph", graph.getattr("native")?), + ], + )?; + status.complete(&format!( + "Built manifest. ({}s)", + seconds_since(py, started)? + ))?; + Ok(payload) +} + +/// The DAG artifact path a `--dag` value names, resolved against the project. +pub(crate) fn resolved_dag_path<'py>( + project_dir: &Bound<'py, PyAny>, + dag_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let empty: Bound<'py, PyAny> = imported(py, CLI_CONSTANTS_MODULE, "EMPTY_DAG_PATH")?; + if dag_path.eq(&empty)? { + return target_dir(project_dir)?.call_method1("__truediv__", (DEFAULT_DAG_FILE,)); + } + let path: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?.call1((dag_path,))?; + if path.call_method0("is_absolute")?.is_truthy()? { + return Ok(path); + } + project_dir.call_method1("__truediv__", (path,)) +} + +/// Write the DAG JSON artifact when a DAG path is requested. +pub(crate) fn write_dag_artifact<'py>( + (dag_path, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + analysis: &Bound<'py, PyAny>, + status: &Status<'py>, +) -> PyResult<()> { + if dag_path.is_none() { + return Ok(()); + } + let py: Python<'py> = analysis.py(); + let started: f64 = monotonic(py)?; + status.start("Writing DAG artifact...")?; + let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; + let python_graph: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_NODES_MODULE, + "build_discovered_python_node_graph", + )?, + &[("discovered_inputs", discovered.clone())], + )?; + let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; + let dag_json: Bound<'py, PyAny> = call_with( + &imported(py, DAG_MODULE, "build_dag_json")?, + &[ + ("graph", analysis.getattr("graph")?), + ( + "project_name", + discovered.getattr("project_config")?.getattr("name")?, + ), + ("python_graph", python_graph), + ], + )?; + let batch: Bound<'py, PyAny> = call_with( + &imported(py, ARTIFACT_BATCH_MODULE, "NativeArtifactBatch")?, + &[( + "check_existing", + PyBool::new(py, false).to_owned().into_any(), + )], + )?; + call_with( + &batch.getattr("queue")?, + &[ + ("path", path), + ("contents", dag_json.call_method1("encode", ("utf-8",))?), + ], + )?; + batch.call_method0("flush")?; + status.complete(&format!( + "Wrote DAG artifact. ({}s)", + seconds_since(py, started)? + )) +} + +/// SQL test planning errors of a compile whose artifacts are withheld, reusing staged planning. +pub(crate) fn sql_test_planning_diagnostics<'py>( + project_dir: &Bound<'py, PyAny>, + analysis: &Bound<'py, PyAny>, + prepared_artifacts: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = analysis.py(); + let planned: PyResult> = (|| { + if !prepared_artifacts.is_none() { + let staged: Bound<'py, PyAny> = + prepared_artifacts.call_method0("planning_diagnostics")?; + if !staged.is_none() { + return Ok(staged); + } + } + call_with( + &imported( + py, + TARGET_WRITER_MODULE, + "static_sql_test_planning_diagnostics", + )?, + &[ + ("target_dir", target_dir(project_dir)?), + ("adapter", analysis.getattr("adapter")?), + ("project", analysis.getattr("graph")?.getattr("project")?), + ], + ) + })(); + let error = match planned { + Ok(diagnostics) => return Ok(diagnostics), + Err(error) => error, + }; + let compile_input_error: Bound<'py, PyAny> = + imported(py, COMPILE_EXCEPTIONS_MODULE, "CompileInputError")?; + let planner_input_error: Bound<'py, PyAny> = + imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")?; + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if !(value.is_instance(&compile_input_error)? || value.is_instance(&planner_input_error)?) { + return Err(error); + } + let reference_error: Bound<'py, PyAny> = + imported(py, COMPILE_EXCEPTIONS_MODULE, "SqlTestReferenceError")?; + let location: Bound<'py, PyAny> = if value.is_instance(&reference_error)? { + value.getattr("location")? + } else { + py.None().into_bound(py) + }; + let diagnostic: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + &[ + ( + "phase", + imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("TEST")?, + ), + ( + "severity", + imported(py, COMPILE_TYPES_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?, + ), + ("code", value.getattr("code")?), + ("message", value.str()?.into_any()), + ("help", value.getattr("help")?), + ("location", location), + ], + )?; + Ok(PyTuple::new(py, [diagnostic])?.into_any()) +} + +/// Write compiled artifacts to `target/` and return `(written target, write milliseconds)`. +pub(crate) fn write_artifacts<'py>( + (skip_write, project_dir): (bool, &Bound<'py, PyAny>), + (analysis, manifest): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + prepared_artifacts: &Bound<'py, PyAny>, + status: &Status<'py>, +) -> PyResult<(Bound<'py, PyAny>, i64)> { + let py: Python<'py> = analysis.py(); + let started: f64 = monotonic(py)?; + let target: Bound<'py, PyAny> = target_dir(project_dir)?; + let written: Bound<'py, PyAny> = if skip_write { + let zero: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); + call_with( + &imported(py, OUTPUT_MODELS_MODULE, "WrittenTarget")?, + &[ + ("model_count", zero.clone()), + ("seed_count", zero.clone()), + ("function_count", zero.clone()), + ("audit_count", zero.clone()), + ("test_count", zero), + ("target_dir", target), + ], + )? + } else { + status.start("Writing compiled artifacts...")?; + let staged: Bound<'py, PyAny> = if prepared_artifacts.is_none() { + py.None().into_bound(py) + } else { + call_with( + &prepared_artifacts.getattr("publish")?, + &[ + ("target_dir", target.clone()), + ("manifest", manifest.clone()), + ], + )? + }; + if staged.is_none() { + call_with( + &imported(py, TARGET_WRITER_MODULE, "write_static_compile_target")?, + &[ + ("target_dir", target), + ("adapter", analysis.getattr("adapter")?), + ("project", analysis.getattr("graph")?.getattr("project")?), + ("manifest", manifest.clone()), + ], + )? + } else { + staged + } + }; + let write_ms: i64 = elapsed_ms(py, started)?; + if !skip_write { + status.complete(&format!( + "Wrote compiled artifacts. ({:.2}s)", + write_ms as f64 / 1000.0 + ))?; + } + Ok((written, write_ms)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs new file mode 100644 index 0000000000..499b76e4b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -0,0 +1,457 @@ +//! Compiled project and project graph construction: frontier stages, target defaults and checks. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; + +const FRONTIER_MODULE: &str = "sqlbuild.compiler.frontier.main._compile_frontier"; +const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; +const BUILD_INPUTS_MODULE: &str = "sqlbuild.compiler.compile.main._build_compile_inputs"; +const ASSEMBLE_MODULE: &str = "sqlbuild.compiler.compile.main._assemble_project"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_conflicts"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; +const TARGET_DEFAULTS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.target_defaults"; +const TARGET_VALIDATION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.target_validation"; +const GRAPH_HELPERS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.graph"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const COLLECTION_RENDERING_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; +/// Optional keywords of the project graph builders. +const GRAPH_OPTIONAL: [&str; 7] = [ + "selected_target", + "no_sql_validation", + "skip_column_inference", + "column_lineage_mode", + "cli_vars", + "external_sql_reference_resolver", + "on_progress", +]; + +/// The keyword arguments of one compiled-project build. +struct ProjectRequest { + discovered_inputs: Py, + adapter: Py, + selected_target: Py, + no_sql_validation: bool, + skip_column_inference: bool, + column_lineage_mode: Py, + cli_vars: Py, + external_sql_reference_resolver: Py, + resolved_connection: Py, + analysis_selection: Py, +} + +/// `column_lineage_mode`, defaulting to `ColumnLineageMode.FAST` when not passed. +fn lineage_mode<'py>(arguments: &KeywordArguments<'py>) -> PyResult> { + let mode: Bound<'py, PyAny> = arguments.value("column_lineage_mode")?; + if arguments.passed("column_lineage_mode")? { + return Ok(mode); + } + imported(mode.py(), LINEAGE_TYPES_MODULE, "ColumnLineageMode")?.getattr("FAST") +} + +/// `value` as a Python `bool`. +fn py_bool(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// Build one compiled project with adapter defaults and target validation applied. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_compiled_project<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_compiled_project", + options, + ( + &["discovered_inputs", "adapter"], + &[ + "selected_target", + "no_sql_validation", + "skip_column_inference", + "column_lineage_mode", + "cli_vars", + "external_sql_reference_resolver", + "resolved_connection", + "analysis_selection", + ], + ), + )?; + compiled_project(py, ProjectRequest::read(&arguments)?) +} + +impl ProjectRequest { + /// The request named by keyword `arguments`. + fn read(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(Self { + discovered_inputs: arguments.value("discovered_inputs")?.unbind(), + adapter: arguments.value("adapter")?.unbind(), + selected_target: arguments.value("selected_target")?.unbind(), + no_sql_validation: arguments.flag("no_sql_validation")?, + skip_column_inference: arguments.flag("skip_column_inference")?, + column_lineage_mode: lineage_mode(arguments)?.unbind(), + cli_vars: arguments.value("cli_vars")?.unbind(), + external_sql_reference_resolver: arguments + .value("external_sql_reference_resolver")? + .unbind(), + resolved_connection: arguments.value("resolved_connection")?.unbind(), + analysis_selection: arguments.value("analysis_selection")?.unbind(), + }) + } +} + +/// The compiled project produced through the `COMPILED_PROJECT` frontier. +fn compiled_project(py: Python<'_>, request: ProjectRequest) -> PyResult> { + let stage: Bound<'_, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + python_compiled_project(arguments.py(), &request).map(Bound::unbind) + }, + )?; + call_with( + &imported(py, FRONTIER_MODULE, "compile_frontier")?, + &[ + ( + "until", + imported(py, FRONTIER_TYPES_MODULE, "CompilerStage")? + .getattr("COMPILED_PROJECT")?, + ), + ("python_stage", stage.into_any()), + ], + ) +} + +/// `_build_python_compiled_project`. +fn python_compiled_project<'py>( + py: Python<'py>, + request: &ProjectRequest, +) -> PyResult> { + let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); + let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); + call_with( + &imported( + py, + TARGET_VALIDATION_MODULE, + "validate_named_target_schema_strategy", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", request.selected_target.bind(py).clone()), + ], + )?; + call_with( + &imported( + py, + TARGET_VALIDATION_MODULE, + "validate_managed_loader_target_isolation", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ( + "resolved_connection", + request.resolved_connection.bind(py).clone(), + ), + ], + )?; + let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); + let no_cache: bool = !selection.is_none() && selection.getattr("no_cache")?.is_truthy()?; + let timing: Bound<'py, PyAny> = + imported(py, PROFILING_MODULE, "record_compile_timing")?.call1(("attachment_ms",))?; + let compile_inputs: Bound<'py, PyAny> = + within(&timing, |_| frontier_compile_inputs(py, request, no_cache))?; + if !selection.is_none() { + let on_inputs_ready: Bound<'py, PyAny> = selection.getattr("on_inputs_ready")?; + if !on_inputs_ready.is_none() { + call_with(&on_inputs_ready, &[("inputs", compile_inputs.clone())])?; + } + } + let inference_profile: Bound<'py, PyAny> = + adapter.call_method0("expression_inference_profile")?; + let analysis_model_names: Bound<'py, PyAny> = call_with( + &imported( + py, + ANALYSIS_SELECTION_MODULE, + "resolve_analysis_model_names", + )?, + &[ + ("compile_inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ("selection", selection.clone()), + ], + )?; + let project: Bound<'py, PyAny> = call_with( + &imported(py, ASSEMBLE_MODULE, "assemble_project")?, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile), + ( + "skip_column_inference", + py_bool(py, request.skip_column_inference), + ), + ( + "column_lineage_mode", + request.column_lineage_mode.bind(py).clone(), + ), + ( + "analysis_cache_dir", + compile_inputs.getattr("compile_cache_dir")?, + ), + ("analysis_model_names", analysis_model_names), + ], + )?; + targeted_project(py, request, &project) +} + +/// The assembled project with target defaults, target checks and promotion conflicts applied. +fn targeted_project<'py>( + py: Python<'py>, + request: &ProjectRequest, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); + let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); + call_with( + &imported( + py, + TARGET_VALIDATION_MODULE, + "validate_managed_write_schemas", + )?, + &[("adapter", adapter.clone()), ("project", project.clone())], + )?; + let effective_schema: Bound<'py, PyAny> = project.getattr("effective_target_schema")?; + let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { + effective_schema + } else { + adapter.call_method0("default_schema")? + }; + let project: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_DEFAULTS_MODULE, "apply_target_defaults")?, + &[ + ("project", project.clone()), + ("default_schema", default_schema), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "python_functions_inherit_default_namespace", + adapter.call_method0("python_functions_inherit_default_namespace")?, + ), + ], + )?; + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", project_config), + ("local_config", local_config.clone()), + ], + )?; + call_with( + &imported(py, TARGET_VALIDATION_MODULE, "validate_project_targets")?, + &[("adapter_name", adapter_name), ("project", project.clone())], + )?; + let promotion_key: Bound<'py, PyAny> = imported( + py, + DISCOVERY_CONSTANTS_MODULE, + "TABLE_PROMOTION_MODE_SETTING_KEY", + )?; + let settings_file: Bound<'py, PyAny> = imported( + py, + DISCOVERY_CONSTANTS_MODULE, + if local_config + .getattr("setting_overrides")? + .contains(promotion_key)? + { + "LOCAL_CONFIG_FILENAME" + } else { + "PROJECT_CONFIG_FILENAME" + }, + )?; + let conflicts: Bound<'py, PyAny> = call_with( + &imported(py, PROMOTION_MODULE, "promotion_conflict_diagnostics")?, + &[ + ("project", project.clone()), + ( + "adapter_default", + adapter.call_method0("default_table_promotion_mode")?, + ), + ("settings_file", settings_file), + ], + )?; + let diagnostics: Bound<'py, PyTuple> = PyTuple::new( + py, + project + .getattr("diagnostics")? + .try_iter()? + .chain(conflicts.try_iter()?) + .collect::>>()?, + )?; + call_with( + &imported(py, PROJECT_WITH_MODULE, "project_with")?, + &[ + ("project", project), + ("diagnostics", diagnostics.into_any()), + ], + ) +} + +/// The compile inputs produced through the `COMPILE_PROJECT_INPUTS` frontier. +fn frontier_compile_inputs<'py>( + py: Python<'py>, + request: &ProjectRequest, + no_cache: bool, +) -> PyResult> { + let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); + let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); + let collection_rendering: Bound<'py, PyAny> = call_with( + &imported( + py, + COLLECTION_RENDERING_MODULE, + "resolve_effective_collection_rendering", + )?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("declaration_override", py.None().into_bound(py)), + ], + )?; + let adapter_context: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileAdapterContext")?, + &[ + ("value_renderer", adapter.clone()), + ("collection_rendering", collection_rendering), + ( + "python_functions_inherit_default_namespace", + adapter.call_method0("python_functions_inherit_default_namespace")?, + ), + ("sql_lexical_syntax", adapter.getattr("sql_lexical_syntax")?), + ], + )?; + let stage: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( + (imported(py, BUILD_INPUTS_MODULE, "build_compile_inputs")?,), + Some(&keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter_context", adapter_context), + ("selected_target", request.selected_target.bind(py).clone()), + ("no_sql_validation", py_bool(py, request.no_sql_validation)), + ("defer_model_sql_validation", py_bool(py, true)), + ("cli_vars", request.cli_vars.bind(py).clone()), + ( + "resolved_connection", + request.resolved_connection.bind(py).clone(), + ), + ( + "external_sql_reference_resolver", + request.external_sql_reference_resolver.bind(py).clone(), + ), + ("no_cache", py_bool(py, no_cache)), + ], + )?), + )?; + call_with( + &imported(py, FRONTIER_MODULE, "compile_frontier")?, + &[ + ( + "until", + imported(py, FRONTIER_TYPES_MODULE, "CompilerStage")? + .getattr("COMPILE_PROJECT_INPUTS")?, + ), + ("python_stage", stage), + ], + ) +} + +/// Build a static graph with an explicit deep-analysis selection. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_project_graph_with_analysis_selection<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_graph_with_analysis_selection", + options, + ( + &["discovered_inputs", "adapter", "analysis_selection"], + &GRAPH_OPTIONAL, + ), + )?; + selected_graph(&arguments, arguments.value("analysis_selection")?) +} + +/// Build the static dependency graph for a compiled project. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_project_graph<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let mut optional: Vec<&str> = GRAPH_OPTIONAL.to_vec(); + optional.push("no_cache"); + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_project_graph", + options, + (&["discovered_inputs", "adapter"], &optional), + )?; + let selection: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")?, + &[("no_cache", py_bool(py, arguments.flag("no_cache")?))], + )?; + selected_graph(&arguments, selection) +} + +/// `build_project_graph_with_analysis_selection` with its `analysis_selection`. +fn selected_graph<'py>( + arguments: &KeywordArguments<'py>, + analysis_selection: Bound<'py, PyAny>, +) -> PyResult> { + let on_progress: Bound<'py, PyAny> = arguments.value("on_progress")?; + let py: Python<'py> = on_progress.py(); + if !on_progress.is_none() { + on_progress.call1(("Compiling project...",))?; + } + let started: f64 = monotonic(py)?; + let mut request: ProjectRequest = ProjectRequest::read(arguments)?; + request.analysis_selection = analysis_selection.unbind(); + let project: Bound<'py, PyAny> = compiled_project(py, request)?; + if !on_progress.is_none() { + let elapsed: f64 = monotonic(py)? - started; + on_progress.call1((format!("Compiled project. ({elapsed:.2}s)"),))?; + } + imported(py, GRAPH_HELPERS_MODULE, "build_project_graph_impl")?.call1((project,)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_compiled_project, module)?)?; + module.add_function(wrap_pyfunction!( + build_project_graph_with_analysis_selection, + module + )?)?; + module.add_function(wrap_pyfunction!(build_project_graph, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs new file mode 100644 index 0000000000..84ca7fe197 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -0,0 +1,130 @@ +//! Calling the Python stages the native driver still runs: imports, keyword calls and `with`. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +/// `module.name`, imported. +pub(crate) fn imported<'py>( + py: Python<'py>, + module: &str, + name: &str, +) -> PyResult> { + py.import(module)?.getattr(name) +} + +/// Keyword arguments built from `(name, value)` pairs, in order. +pub(crate) fn keywords<'py>( + py: Python<'py>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in pairs { + arguments.set_item(*name, value)?; + } + Ok(arguments) +} + +/// `callable(**pairs)`. +pub(crate) fn call_with<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + callable.call((), Some(&keywords(callable.py(), pairs)?)) +} + +/// Run `body` inside `with manager as entered:`; managers here never suppress exceptions. +pub(crate) fn within<'py, T>( + manager: &Bound<'py, PyAny>, + body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, +) -> PyResult { + let py: Python<'py> = manager.py(); + let entered: Bound<'py, PyAny> = manager.call_method0("__enter__")?; + match body(&entered) { + Ok(value) => { + manager.call_method1("__exit__", (py.None(), py.None(), py.None()))?; + Ok(value) + } + Err(error) => { + manager.call_method1( + "__exit__", + (error.get_type(py), error.value(py), error.traceback(py)), + )?; + Err(error) + } + } +} + +/// The transient status line, when one is shown. +pub(crate) struct Status<'py>(pub(crate) Bound<'py, PyAny>); + +impl Status<'_> { + pub(crate) fn start(&self, message: &str) -> PyResult<()> { + if !self.0.is_none() { + self.0.call_method1("start", (message,))?; + } + Ok(()) + } + + pub(crate) fn complete(&self, message: &str) -> PyResult<()> { + if !self.0.is_none() { + call_with( + &self.0.getattr("complete")?, + &[("message", PyString::new(self.0.py(), message).into_any())], + )?; + } + Ok(()) + } +} + +/// Keyword-only arguments of a driver entry point, checked as Python checks a signature. +pub(crate) struct KeywordArguments<'py> { + values: Bound<'py, PyDict>, +} + +impl<'py> KeywordArguments<'py> { + /// Accept `options` for `function`, rejecting missing `required` and unknown names. + pub(crate) fn read( + py: Python<'py>, + function: &str, + options: Option>, + (required, optional): (&[&str], &[&str]), + ) -> PyResult { + let values: Bound<'py, PyDict> = options.unwrap_or_else(|| PyDict::new(py)); + for name in values.keys() { + let name: String = name.extract()?; + if !required.contains(&name.as_str()) && !optional.contains(&name.as_str()) { + return Err(PyTypeError::new_err(format!( + "{function}() got an unexpected keyword argument '{name}'" + ))); + } + } + for name in required { + if !values.contains(*name)? { + return Err(PyTypeError::new_err(format!( + "{function}() missing 1 required keyword-only argument: '{name}'" + ))); + } + } + Ok(Self { values }) + } + + /// The argument `name`, or `None` when it was not passed. + pub(crate) fn value(&self, name: &str) -> PyResult> { + let py: Python<'py> = self.values.py(); + Ok(self + .values + .get_item(name)? + .unwrap_or_else(|| py.None().into_bound(py))) + } + + /// The argument `name` as a truth value, `False` when it was not passed. + pub(crate) fn flag(&self, name: &str) -> PyResult { + self.value(name)?.is_truthy() + } + + /// Whether `name` was passed. + pub(crate) fn passed(&self, name: &str) -> PyResult { + self.values.contains(name) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs new file mode 100644 index 0000000000..fed37dac7a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -0,0 +1,11 @@ +//! Register the native compile command driver. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::compile_driver::{analysis, command, project}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + analysis::register(module)?; + command::register(module)?; + project::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 29b805a100..70c9e1eb58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod contracts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs new file mode 100644 index 0000000000..1644ce7838 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -0,0 +1,147 @@ +//! What each model, seed, source and table function declares about its columns before analysis. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const UNKNOWN_TYPE: &str = "UNKNOWN"; +const NOT_NULL_AUDIT_NAME: &str = "not_null"; +const ENFORCED_CONTRACT: &str = "enforced"; +const CONTRACT_KEY: &str = "contract"; +const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; +const NULLABILITY_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const NULLABILITY_CLASS: &str = "InferredNullability"; +const NON_NULL_MEMBER: &str = "NON_NULL"; +const UNKNOWN_MEMBER: &str = "UNKNOWN"; + +/// The three per-relation declaration maps the analysis starts from. +type DeclaredRelations<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); + +/// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. +fn column_types<'py>(columns: &Bound<'py, PyAny>, exact: bool) -> PyResult> { + let types: Bound<'py, PyDict> = PyDict::new(columns.py()); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let declared: Bound<'py, PyAny> = column.getattr("type")?; + if exact || declared.is_truthy()? { + types.set_item(column.getattr("name")?, declared)?; + } else { + types.set_item(column.getattr("name")?, UNKNOWN_TYPE)?; + } + } + Ok(types) +} + +/// Each column's declared nullability: non-null when `nullable` is false or a `not_null` audit. +fn column_nullability<'py>( + columns: &Bound<'py, PyAny>, + nullability: &Bound<'py, PyAny>, +) -> PyResult> { + let facts: Bound<'py, PyDict> = PyDict::new(columns.py()); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let mut non_null: bool = column + .getattr("nullable")? + .is(PyBool::new(columns.py(), false).as_any()); + for audit in column.getattr("audits")?.try_iter()? { + non_null = non_null || audit?.getattr("definition_name")?.eq(NOT_NULL_AUDIT_NAME)?; + } + let state: &str = if non_null { + NON_NULL_MEMBER + } else { + UNKNOWN_MEMBER + }; + facts.set_item(column.getattr("name")?, nullability.getattr(state)?)?; + } + Ok(facts) +} + +fn table_function_name(function: &Bound<'_, PyAny>) -> PyResult { + let name: String = function.getattr("name")?.extract()?; + Ok(format!("{TABLE_FUNCTION_PREFIX}{name}")) +} + +/// Declared column types, declared nullability and the authoritative binding schemas. +#[pyfunction] +fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = inputs.py(); + let nullability: Bound<'py, PyAny> = + py.import(NULLABILITY_MODULE)?.getattr(NULLABILITY_CLASS)?; + let types: Bound<'py, PyDict> = PyDict::new(py); + let nulls: Bound<'py, PyDict> = PyDict::new(py); + let complete: Bound<'py, PyDict> = PyDict::new(py); + for model in inputs.getattr("model_inputs")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + continue; + } + let name: Bound<'py, PyAny> = entry.getattr("name")?; + let columns: Bound<'py, PyAny> = entry.getattr("columns")?; + types.set_item(&name, column_types(&columns, false)?)?; + nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + let contract: Option> = model + .getattr("config")? + .getattr("values")? + .downcast_into::()? + .get_item(CONTRACT_KEY)?; + let enforced: bool = match contract { + Some(contract) => contract.eq(ENFORCED_CONTRACT)?, + None => false, + }; + if enforced && columns.is_truthy()? && !entry.getattr("dynamic_columns")?.is_truthy()? { + let stem: Bound<'py, PyAny> = model + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")?; + complete.set_item(stem, column_types(&columns, false)?)?; + } + } + let mut seeds: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for seed in inputs.getattr("seed_inputs")?.try_iter()? { + let entry: Bound<'py, PyAny> = seed?.getattr("schema_entry")?; + let name: Bound<'py, PyAny> = entry.getattr("name")?; + let columns: Bound<'py, PyAny> = entry.getattr("columns")?; + types.set_item(&name, column_types(&columns, false)?)?; + nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + seeds.push((name, columns)); + } + for source in inputs.getattr("source_inputs")?.try_iter()? { + let entry: Bound<'py, PyAny> = source?.getattr("source_entry")?; + let name: Bound<'py, PyAny> = entry.getattr("name")?; + let columns: Bound<'py, PyAny> = entry.getattr("columns")?; + types.set_item(&name, column_types(&columns, false)?)?; + nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + if entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)? && columns.is_truthy()? { + complete.set_item(&name, column_types(&columns, false)?)?; + } + } + for (name, columns) in seeds { + if columns.is_truthy()? { + complete.set_item(name, column_types(&columns, false)?)?; + } + } + for function in inputs.getattr("sql_function_inputs")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let columns: Bound<'py, PyAny> = function.getattr("return_columns")?; + if !columns.is_truthy()? { + continue; + } + let name: String = table_function_name(&function)?; + types.set_item(&name, column_types(&columns, true)?)?; + let unknown: Bound<'py, PyDict> = PyDict::new(py); + for column in columns.try_iter()? { + unknown.set_item( + column?.getattr("name")?, + nullability.getattr(UNKNOWN_MEMBER)?, + )?; + } + nulls.set_item(&name, unknown)?; + complete.set_item(&name, column_types(&columns, true)?)?; + } + Ok((types, nulls, complete)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declared_relations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index 79eb3404a1..c837f3400f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1,2 +1,4 @@ +pub(crate) mod declared_relations; pub(crate) mod registration; pub(crate) mod resources; +pub(crate) mod syntax_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 67e7f084fc..2f9924ab18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -2,8 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::project_assembly::resources; +use crate::bindings::_helpers::project_assembly::{ + declared_relations, resources, syntax_validation, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + declared_relations::register(module)?; + syntax_validation::register(module)?; resources::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs new file mode 100644 index 0000000000..2ef4678d63 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -0,0 +1,255 @@ +//! Reject model and hook SQL the parser cannot read, naming the model, hook and file. + +use std::collections::HashMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; + +use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; +const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; +const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; +const OPT_OUT_HELP: &str = "sql_analysis_opt_out_help"; +const GENERIC_DIALECT: &str = "generic"; + +/// `(placeholder defaults, dialect)` a statement parses with. +type ParseOptions = (Option>, Option); + +/// What every statement of one model is parsed with. +struct SyntaxContext<'a, 'py> { + py: Python<'py>, + model_name: &'a str, + placeholders: &'a Option>, + dialect: &'a str, + sql_hook_entry: Bound<'py, PyAny>, + python_hook_entry: Bound<'py, PyAny>, +} + +/// One hook value to validate: the hook key, its file and the label errors name it by. +struct HookSite<'a, 'py> { + hook_name: &'a str, + file_path: Bound<'py, PyAny>, + label: Option, +} + +/// `CompileInputError(message, help=sql_analysis_opt_out_help(model_owned=True))`. +fn syntax_error(py: Python<'_>, message: String) -> PyErr { + let raised = || -> PyResult { + let help_arguments: Bound<'_, PyDict> = PyDict::new(py); + help_arguments.set_item("model_owned", true)?; + let help: Bound<'_, PyAny> = py + .import(VALIDATION_MODULE)? + .getattr(OPT_OUT_HELP)? + .call((), Some(&help_arguments))?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + let error: Bound<'_, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call((message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +/// The parser's message for `sql`, or `None` where it parses. +fn parse_error(context: &SyntaxContext<'_, '_>, sql: &str) -> PyResult> { + let check: SyntaxCheck = SyntaxCheck { + sql: sql.to_owned(), + placeholders: context + .placeholders + .clone() + .unwrap_or_default() + .into_iter() + .collect(), + }; + syntax_error_message(context.py, &check, context.dialect, SyntaxMode::Validate) +} + +fn sql_entry_label(prefix: &str, entry: &Bound<'_, PyAny>) -> PyResult { + let name: Bound<'_, PyAny> = entry.getattr("name")?; + Ok(if name.is_none() { + format!("{prefix} inline_sql(\"...\")") + } else { + format!("{prefix} sql(\"{}\")", name.str()?) + }) +} + +/// `validate_hook_sql_syntax`: each SQL string inside a hook value, depth first. +fn check_hook_value( + context: &SyntaxContext<'_, '_>, + value: &Bound<'_, PyAny>, + site: HookSite<'_, '_>, +) -> PyResult<()> { + if value.is_instance_of::() { + let label: String = site.label.unwrap_or_else(|| site.hook_name.to_owned()); + if let Some(message) = parse_error(context, &value.extract::()?)? { + return Err(syntax_error( + context.py, + format!( + "Polyglot could not parse model '{}' {label} ({}): {message}", + context.model_name, + site.file_path.str()? + ), + )); + } + return Ok(()); + } + if value.is_instance(&context.sql_hook_entry)? { + let label: String = match site.label { + Some(label) => label, + None => sql_entry_label(site.hook_name, value)?, + }; + let relative_path: Bound<'_, PyAny> = value.getattr("relative_path")?; + let file_path = if relative_path.is_truthy()? { + relative_path + } else { + site.file_path + }; + let statement: Bound<'_, PyAny> = value.getattr("statement")?; + return check_hook_value( + context, + &statement, + HookSite { + hook_name: site.hook_name, + file_path, + label: Some(label), + }, + ); + } + if value.is_instance(&context.python_hook_entry)? { + return Ok(()); + } + if value.is_instance_of::() || value.is_instance_of::() { + for (index, item) in value.try_iter()?.enumerate() { + let item: Bound<'_, PyAny> = item?; + let prefix: String = format!("{}[{index}]", site.hook_name); + let label: Option = if item.is_instance(&context.sql_hook_entry)? { + Some(sql_entry_label(&prefix, &item)?) + } else if item.is_instance_of::() { + Some(format!("{prefix} inline_sql(\"...\")")) + } else { + None + }; + check_hook_value( + context, + &item, + HookSite { + hook_name: site.hook_name, + file_path: site.file_path.clone(), + label, + }, + )?; + } + } + Ok(()) +} + +/// Validate the SQL of one model's hook value, raising the first parse failure. +#[pyfunction] +#[pyo3(signature = (value, hook_name, model, parsing=(None, None)))] +fn validate_hook_sql_syntax( + value: Bound<'_, PyAny>, + hook_name: &str, + model: (String, Bound<'_, PyAny>), + parsing: ParseOptions, +) -> Result<(), PyErr> { + let (model_name, file_path) = model; + let (placeholders, dialect) = parsing; + let py: Python<'_> = value.py(); + let models: Bound<'_, PyModule> = py.import(DISCOVERY_MODELS_MODULE)?; + let dialect: String = dialect + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + let context: SyntaxContext<'_, '_> = SyntaxContext { + py, + model_name: &model_name, + placeholders: &placeholders, + dialect: &dialect, + sql_hook_entry: models.getattr(SQL_HOOK_ENTRY)?, + python_hook_entry: models.getattr(PYTHON_HOOK_ENTRY)?, + }; + check_hook_value( + &context, + &value, + HookSite { + hook_name, + file_path, + label: None, + }, + ) +} + +/// Validate that a model's query parses, raising its syntax error. +#[pyfunction] +#[pyo3(signature = (query_sql, model, parsing=(None, None)))] +fn validate_sql_syntax( + py: Python<'_>, + query_sql: &str, + model: (String, Bound<'_, PyAny>), + parsing: ParseOptions, +) -> Result<(), PyErr> { + let (model_name, file_path) = model; + let (placeholders, dialect) = parsing; + let check: SyntaxCheck = SyntaxCheck { + sql: query_sql.to_owned(), + placeholders: placeholders.unwrap_or_default().into_iter().collect(), + }; + let dialect: String = dialect + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); + match syntax_error_message(py, &check, &dialect, SyntaxMode::Validate)? { + Some(message) => Err(syntax_error( + py, + format!( + "SQL syntax error in model '{model_name}' ({}): {message}", + file_path.str()? + ), + )), + None => Ok(()), + } +} + +/// `hook_sql_statements`: the hook SQL strings `validate_hook_sql_syntax` checks, in its order. +pub(crate) fn hook_statements( + value: &Bound<'_, PyAny>, + sql_hook_entry: &Bound<'_, PyAny>, +) -> PyResult> { + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); + } + if value.is_instance(sql_hook_entry)? { + return hook_statements(&value.getattr("statement")?, sql_hook_entry); + } + let mut statements: Vec = Vec::new(); + if value.is_instance_of::() || value.is_instance_of::() { + for item in value.try_iter()? { + statements.extend(hook_statements(&item?, sql_hook_entry)?); + } + } + Ok(statements) +} + +/// The hook SQL strings a hook value holds, in validation order. +#[pyfunction] +fn hook_sql_statements(value: Bound<'_, PyAny>) -> PyResult> { + let sql_hook_entry: Bound<'_, PyAny> = value + .py() + .import(DISCOVERY_MODELS_MODULE)? + .getattr(SQL_HOOK_ENTRY)?; + hook_statements(&value, &sql_hook_entry) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_hook_sql_syntax, module)?)?; + module.add_function(wrap_pyfunction!(validate_sql_syntax, module)?)?; + module.add_function(wrap_pyfunction!(hook_sql_statements, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs new file mode 100644 index 0000000000..8b561d1a36 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -0,0 +1,114 @@ +//! Static inventory of project macro modules for the Python macro loader, with the GIL released. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::macro_modules::main::inventory_macro_modules::inventory_macro_modules as inventory; +use sqlbuild_pysource::macro_modules::models::{ + MacroExport, MacroFault, MacroModuleInventory, MacroModuleLoad, MacroModulePath, + MacroModuleSource, +}; +use sqlbuild_pysource::macro_modules::types::MacroScopeKind; + +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; +use crate::bindings::types::CompilerDetach; + +/// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. +type SourceRow = ( + String, + String, + Vec, + Option>, + Option>, + String, + String, + Option, +); + +/// `(name, file index, parameters, line, source digest, dependency macro names)`. +type ExportRow = (String, usize, Vec, usize, String, Vec); + +/// `(file index, message)`. +type FaultRow = (usize, String); + +/// `(file index, module name, imported project module names)`. +type ModuleRow = (usize, String, Vec); + +/// Export rows, fault rows and the load plan rows when the strict analysis passed. +type InventoryRows = (Vec, Vec, Option>); + +/// The tier a `ScopeKind` value names; any other value sees nothing, like `PRIVATE`. +fn scope_kind_of(value: &str) -> MacroScopeKind { + match value { + "global" => MacroScopeKind::Global, + "inherited" => MacroScopeKind::Inherited, + "local" => MacroScopeKind::Local, + _ => MacroScopeKind::Private, + } +} + +fn source(row: SourceRow) -> MacroModuleSource { + let (display, posix, parts, declaration_root, owning_path, scope_kind, contents, syntax_error) = + row; + MacroModuleSource { + relative_path: MacroModulePath { + display, + posix, + parts, + }, + declaration_root, + owning_path, + scope_kind: scope_kind_of(&scope_kind), + contents, + syntax_error, + } +} + +fn export_row(export: MacroExport) -> ExportRow { + ( + export.name, + export.file_index, + export.parameters, + export.line, + export.source_digest, + export.dependencies, + ) +} + +fn fault_row(fault: MacroFault) -> FaultRow { + (fault.file_index, fault.message) +} + +fn module_row(module: MacroModuleLoad) -> ModuleRow { + ( + module.file_index, + module.module_name, + module.module_dependencies, + ) +} + +/// Exports, faults and, when the strict analysis passes, the load plan of the macro modules. +#[pyfunction] +pub(crate) fn inventory_macro_modules( + py: Python<'_>, + sources: Vec, +) -> PyResult { + let sources: Vec = sources.into_iter().map(source).collect(); + let MacroModuleInventory { + exports, + faults, + modules, + } = py + .compiler_detach(|| Ok(inventory(&sources))) + .map_err(compiler_error)? + .map_err(|error| NativeCompilerError::new_err(error.to_string()))?; + Ok(( + exports.into_iter().map(export_row).collect(), + faults.into_iter().map(fault_row).collect(), + modules.map(|modules| modules.into_iter().map(module_row).collect()), + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index 1383b1f743..0a81e0ef59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -9,4 +9,5 @@ pub(crate) mod macro_evaluation; pub(crate) mod macro_expansion; pub(crate) mod macro_expansion_facts; pub(crate) mod macro_generated_references; +pub(crate) mod macro_modules; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index e627aa1daf..913b13d099 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::render::{ declaration_expansion, declaration_indexes, macro_arguments, macro_calls, macro_expansion, + macro_modules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { macro_arguments::register(module)?; macro_calls::register(module)?; macro_expansion::register(module)?; + macro_modules::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 2516a544d7..7773c54f72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -21,7 +21,7 @@ type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); /// Consumer, declaration, usage kind, what it is used through, and the enum member. -type UsageRow = ( +pub(crate) type UsageRow = ( QualifiedRow, QualifiedRow, String, @@ -37,9 +37,9 @@ type DiagnosticRow = (Vec, String, String, String); /// Usages as `(consumer, declaration)` record positions, S006s, else a scope path error. type SchemaUsageRows = (Vec<(usize, usize)>, Vec, Option); /// `(own, position)`: an own dependency's position, else a distinct usage's. -type SourceRow = (bool, usize); +pub(crate) type SourceRow = (bool, usize); /// Distinct collected positions, merged sources per declaration, and final usage positions. -type MergeRows = (Vec, Vec>>, Vec); +pub(crate) type MergeRows = (Vec, Vec>>, Vec); fn invalid(what: &str) -> pyo3::PyErr { PyValueError::new_err(format!("invalid compile usage {what}")) @@ -131,13 +131,12 @@ fn source_rows(sources: Option>) -> Option> } /// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. -#[pyfunction] -fn scope_merged_usages( - declarations: Vec<(QualifiedRow, Option>)>, - prior: Vec, - collected: Vec, +pub(crate) fn merged_usage_rows( + declarations: &[(QualifiedRow, Option>)], + prior: &[UsageRow], + collected: &[UsageRow], ) -> MergeRows { - let distinct: Vec = distinct_positions(&collected); + let distinct: Vec = distinct_positions(collected); let mut consumed: Vec<(Option, QualifiedRow)> = Vec::with_capacity(distinct.len()); for position in &distinct { @@ -147,7 +146,7 @@ fn scope_merged_usages( .then(|| consumer.clone()); consumed.push((by_declaration, declaration.clone())); } - let merged: Vec>> = merged_dependencies(&declarations, &consumed) + let merged: Vec>> = merged_dependencies(declarations, &consumed) .into_iter() .map(source_rows) .collect(); @@ -159,6 +158,5 @@ fn scope_merged_usages( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scope_schema_usages, module)?)?; - module.add_function(wrap_pyfunction!(scope_merged_usages, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index a42c417031..88f94269f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -8,5 +8,9 @@ pub(crate) mod relationship_scan; pub(crate) mod scope_cache; pub(crate) mod scope_index; pub(crate) mod scope_lookup; -pub(crate) mod scope_report_rows; +pub(crate) mod scope_objects; +pub(crate) mod scope_report_objects; pub(crate) mod scope_reports; +pub(crate) mod scope_tolerant; +pub(crate) mod scope_validation; +pub(crate) mod scope_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs index 54c646f950..2644315669 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs @@ -1,28 +1,17 @@ -//! Unused-declaration and exact-placement rules over a usage-complete scope index's rows. +//! Scope identity, root and record rows as the native scope records. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::placement::main::declaration_directory::declaration_directory; -use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; -use sqlbuild_scopes::placement::main::relocate_for_resource_move::relocate_for_resource_move; -use sqlbuild_scopes::placement::models::{ - PlacementDiagnostic, PlacementIndex, PlacementUsage, Relocation, -}; +use pyo3::prelude::PyResult; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, - QualifiedIdentity, ResourceIdentity, ResourceKind, ResourceRecord, ScopeKind, + QualifiedIdentity, ResourceIdentity, ResourceKind, ScopeKind, }; -use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::types::CompilerDetach; - pub(crate) type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. pub(crate) type QualifiedRow = (String, String, Option); /// An ownership root's path, kind and resource kind. pub(crate) type RootRow = (String, String, Option); -pub(crate) type ResourceRow = (IdentityRow, String, RootRow); /// Identity, path, line, column, scope, ownership root and owning path. pub(crate) type DeclarationRow = ( QualifiedRow, @@ -33,12 +22,6 @@ pub(crate) type DeclarationRow = ( RootRow, Option, ); -/// Consumer, declaration and what the consumer uses it through. -type UsageRow = (QualifiedRow, QualifiedRow, Option); -type IndexRows = (Vec, Vec, Vec); -/// Declaration position, path, scope, ownership root and owning path. -type RelocationRow = (usize, String, &'static str, RootOut, Option); -type RootOut = (String, &'static str, Option<&'static str>); fn invalid(what: &str) -> pyo3::PyErr { PyValueError::new_err(format!("invalid scope placement {what}")) @@ -84,37 +67,10 @@ pub(crate) fn ownership_root((path, kind, resource_kind): RootRow) -> PyResult RootOut { - ( - root.path.clone(), - root.kind.as_str(), - root.resource_kind.map(ResourceKind::as_str), - ) -} - -fn relocation_row(item: Relocation) -> RelocationRow { - ( - item.declaration, - item.path, - item.scope.as_str(), - root_out(&item.ownership_root), - item.owning_path, - ) -} - pub(crate) fn scope_kind(value: &str) -> PyResult { ScopeKind::parse(value).ok_or_else(|| invalid("scope")) } -/// A resource row as its record. -pub(crate) fn resource_record((identity, path, root): ResourceRow) -> PyResult { - Ok(ResourceRecord { - identity: resource_identity(identity)?, - path, - ownership_root: ownership_root(root)?, - }) -} - /// A declaration row as its record, without macro dependencies. pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult { let (identity, path, line, column, scope, root, owning_path) = row; @@ -129,68 +85,3 @@ pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult PyResult { - let mut index: PlacementIndex = PlacementIndex::default(); - for row in resources { - index.resources.push(resource_record(row)?); - } - for row in declarations { - index.declarations.push(declaration_record(row)?); - } - for (consumer, declaration, through) in usages { - index.usages.push(PlacementUsage { - consumer: qualified_identity(consumer)?, - declaration: declaration_identity(declaration)?, - through: through.map(qualified_identity).transpose()?, - }); - } - Ok(index) -} - -/// Each `(declaration position, code, message)` placement diagnostic, in declaration order. -#[pyfunction] -fn scope_placement_diagnostics( - py: Python<'_>, - rows: IndexRows, -) -> PyResult> { - let index: PlacementIndex = placement_index(rows)?; - let diagnostics: Vec = py - .compiler_detach(|| Ok(placement_diagnostics(&index))) - .map_err(compiler_error)?; - Ok(diagnostics - .into_iter() - .map(|item| (item.declaration, item.code.as_str(), item.message)) - .collect()) -} - -/// Each declaration that moves when `resource` moves to `destination`, or `None` if unsettled. -#[pyfunction] -fn scope_relocations( - py: Python<'_>, - rows: IndexRows, - resource: IdentityRow, - destination: &str, -) -> PyResult>> { - let index: PlacementIndex = placement_index(rows)?; - let resource: ResourceIdentity = resource_identity(resource)?; - let relocations: Option> = py - .compiler_detach(|| Ok(relocate_for_resource_move(&index, &resource, destination))) - .map_err(compiler_error)?; - Ok(relocations.map(|items| items.into_iter().map(relocation_row).collect())) -} - -/// The project-relative folder where a declaration of `kind` with this placement lives. -#[pyfunction] -fn scope_declaration_directory(kind: &str, scope: &str, path: Option<&str>) -> PyResult { - let kind: DeclarationKind = - DeclarationKind::parse(kind).ok_or_else(|| invalid("declaration kind"))?; - Ok(declaration_directory(kind, scope_kind(scope)?, path)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_placement_diagnostics, module)?)?; - module.add_function(wrap_pyfunction!(scope_relocations, module)?)?; - module.add_function(wrap_pyfunction!(scope_declaration_directory, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs index 0c4cb93e52..faf5c5da31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -3,18 +3,20 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::scopes::{ - compile_usages, declaration_contexts, placement, relationship_names, scope_cache, scope_index, - scope_lookup, scope_reports, + compile_usages, declaration_contexts, relationship_names, scope_cache, scope_index, + scope_lookup, scope_reports, scope_tolerant, scope_validation, scope_visibility, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { scope_index::register(module)?; relationship_names::register(module)?; declaration_contexts::register(module)?; - placement::register(module)?; compile_usages::register(module)?; scope_lookup::register(module)?; scope_cache::register(module)?; scope_reports::register(module)?; + scope_tolerant::register(module)?; + scope_validation::register(module)?; + scope_visibility::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs index 0efe8fe7cc..a098555f82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs @@ -3,350 +3,23 @@ use std::path::PathBuf; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; -use sqlbuild_scopes::scope_cache::models::{ - CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, - CachedVisibility, ConstantFacts, EnumFacts, Fingerprint, FingerprintInputs, MacroFacts, -}; -use sqlbuild_scopes::scope_index::models::{ - DeclarationIdentity, DeclarationRecord, GrantKind, OwnershipRoot, QualifiedIdentity, - ReportGrant, ResourceIdentity, ResourceKind, ResourceRecord, -}; +use sqlbuild_scopes::scope_cache::models::{CachedIndex, Fingerprint, FingerprintInputs}; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::scopes::placement::{ - DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, - declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, index_object, read_completeness, read_index, }; use crate::bindings::types::CompilerDetach; -/// Parameters, dependencies and source digest of a macro. -type MacroRow = (Vec, Vec, String); -/// Member names and scalar type of an enum. -type EnumRow = (Vec, String); -/// Logical type, collection kind, item count, nullability and rendering of a constant. -type ConstantRow = (String, Option, Option, bool, Option); -type FullDeclarationRow = ( - DeclarationRow, - Option, - Option, - Option, -); -type UsageRow = ( - QualifiedRow, - QualifiedRow, - String, - Option, - Option, -); -type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); -type VisibilityRow = (IdentityRow, QualifiedRow, String, Option); -type InaccessibleRow = (IdentityRow, QualifiedRow, String); -type DiagnosticRow = ( - String, - String, - String, - Option, - Option, - Option, - Option, - Option, -); -type Completeness = (bool, bool, bool, bool, bool, bool); -/// Every field of a full scope index as rows. -type IndexRows = ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Completeness, -); /// Config files, candidate roots, declaration folders, named declaration folders, version. type FingerprintRow = (Vec, Vec, Vec, Vec, String); -fn invalid(what: &str) -> pyo3::PyErr { - PyValueError::new_err(format!("invalid scope cache {what}")) -} - -fn grant_kind(value: &str) -> PyResult { - [GrantKind::ExpectedModel, GrantKind::TestedMacro] - .into_iter() - .find(|kind| kind.as_str() == value) - .ok_or_else(|| invalid("grant kind")) -} - -fn optional_identity(row: Option) -> PyResult> { - row.map(qualified_identity).transpose() -} - -fn full_declaration(row: FullDeclarationRow) -> PyResult { - let (record, macro_row, enum_row, constant_row) = row; - let macro_facts: Option = match macro_row { - None => None, - Some((parameters, dependencies, source_digest)) => Some(MacroFacts { - parameters, - dependencies: dependencies - .into_iter() - .map(declaration_identity) - .collect::>()?, - source_digest, - }), - }; - Ok(CachedDeclaration { - record: declaration_record(record)?, - macro_facts, - enum_facts: enum_row.map(|(members, scalar_type)| EnumFacts { - members, - scalar_type, - }), - constant_facts: constant_row.map( - |(logical_type, collection_kind, item_count, nullable, render_as)| ConstantFacts { - logical_type, - collection_kind, - item_count, - nullable, - render_as, - }, - ), - }) -} - -fn cached_index(rows: IndexRows) -> PyResult { - let ( - roots, - resources, - declarations, - usages, - grants, - visibility, - inaccessible, - diagnostics, - done, - ) = rows; - let mut index: CachedIndex = CachedIndex { - completeness: [done.0, done.1, done.2, done.3, done.4, done.5], - ..CachedIndex::default() - }; - for row in roots { - index.roots.push(ownership_root(row)?); - } - for row in resources { - index.resources.push(resource_record(row)?); - } - for row in declarations { - index.declarations.push(full_declaration(row)?); - } - for (consumer, declaration, kind, through, enum_member) in usages { - index.usages.push(CachedUsage { - consumer: qualified_identity(consumer)?, - declaration: declaration_identity(declaration)?, - kind, - through: optional_identity(through)?, - enum_member, - }); - } - for (resource, declaration, through, kind) in grants { - index.grants.push(ReportGrant { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - through: qualified_identity(through)?, - kind: grant_kind(&kind)?, - }); - } - for (resource, declaration, reason, through) in visibility { - index.visibility.push(CachedVisibility { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - reason, - through: optional_identity(through)?, - }); - } - for (resource, declaration, reason) in inaccessible { - index.inaccessible.push(CachedInaccessible { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - reason, - }); - } - for (code, message, severity, path, line, column, declaration, resource) in diagnostics { - index.diagnostics.push(CachedDiagnostic { - code, - message, - severity, - path, - line, - column, - declaration: declaration.map(declaration_identity).transpose()?, - resource: resource.map(resource_identity).transpose()?, - }); - } - Ok(index) -} - -fn identity_row(identity: &ResourceIdentity) -> IdentityRow { - (identity.kind.as_str().to_owned(), identity.name.clone()) -} - -fn declaration_row(identity: &DeclarationIdentity) -> QualifiedRow { - ( - identity.kind.as_str().to_owned(), - identity.name.clone(), - identity.owner.as_ref().map(identity_row), - ) -} - -fn qualified_row(identity: &QualifiedIdentity) -> QualifiedRow { - match identity { - QualifiedIdentity::Resource(identity) => ( - identity.kind.as_str().to_owned(), - identity.name.clone(), - None, - ), - QualifiedIdentity::Declaration(identity) => declaration_row(identity), - } -} - -fn root_row(root: &OwnershipRoot) -> RootRow { - ( - root.path.clone(), - root.kind.as_str().to_owned(), - root.resource_kind - .map(|kind| ResourceKind::as_str(kind).to_owned()), - ) -} - -fn resource_row(record: &ResourceRecord) -> ResourceRow { - ( - identity_row(&record.identity), - record.path.clone(), - root_row(&record.ownership_root), - ) -} - -fn record_row(record: &DeclarationRecord) -> DeclarationRow { - ( - declaration_row(&record.identity), - record.path.clone(), - record.line, - record.column, - record.scope.as_str().to_owned(), - root_row(&record.ownership_root), - record.owning_path.clone(), - ) -} - -fn full_declaration_row(item: &CachedDeclaration) -> FullDeclarationRow { - ( - record_row(&item.record), - item.macro_facts.as_ref().map(|facts| { - ( - facts.parameters.clone(), - facts.dependencies.iter().map(declaration_row).collect(), - facts.source_digest.clone(), - ) - }), - item.enum_facts - .as_ref() - .map(|facts| (facts.members.clone(), facts.scalar_type.clone())), - item.constant_facts.as_ref().map(|facts| { - ( - facts.logical_type.clone(), - facts.collection_kind.clone(), - facts.item_count, - facts.nullable, - facts.render_as.clone(), - ) - }), - ) -} - -fn index_rows(index: &CachedIndex) -> IndexRows { - let done: [bool; 6] = index.completeness; - ( - index.roots.iter().map(root_row).collect(), - index.resources.iter().map(resource_row).collect(), - index - .declarations - .iter() - .map(full_declaration_row) - .collect(), - index - .usages - .iter() - .map(|item| { - ( - qualified_row(&item.consumer), - declaration_row(&item.declaration), - item.kind.clone(), - item.through.as_ref().map(qualified_row), - item.enum_member.clone(), - ) - }) - .collect(), - index - .grants - .iter() - .map(|item| { - ( - identity_row(&item.resource), - declaration_row(&item.declaration), - qualified_row(&item.through), - item.kind.as_str().to_owned(), - ) - }) - .collect(), - index - .visibility - .iter() - .map(|item| { - ( - identity_row(&item.resource), - declaration_row(&item.declaration), - item.reason.clone(), - item.through.as_ref().map(qualified_row), - ) - }) - .collect(), - index - .inaccessible - .iter() - .map(|item| { - ( - identity_row(&item.resource), - declaration_row(&item.declaration), - item.reason.clone(), - ) - }) - .collect(), - index - .diagnostics - .iter() - .map(|item| { - ( - item.code.clone(), - item.message.clone(), - item.severity.clone(), - item.path.clone(), - item.line, - item.column, - item.declaration.as_ref().map(declaration_row), - item.resource.as_ref().map(identity_row), - ) - }) - .collect(), - (done[0], done[1], done[2], done[3], done[4], done[5]), - ) -} - /// The scope index fingerprint of `project_dir`, `None` when it cannot be read. #[pyfunction] fn scope_index_fingerprint( @@ -375,31 +48,38 @@ fn scope_index_fingerprint( } } -/// The verified cached index rows at `path` for `fingerprint`, else `None`. +/// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. #[pyfunction] fn read_scope_index_cache( py: Python<'_>, path: PathBuf, fingerprint: String, -) -> PyResult> { +) -> PyResult>> { let cached = py .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) .map_err(compiler_error)?; - Ok(match cached { - Ok(index) => Some(index_rows(&index)), - Err(_) => None, - }) + let Ok(index) = cached else { + return Ok(None); + }; + match index_object(&ScopeClasses::new(py)?, &index) { + Ok(object) => Ok(Some(object.unbind())), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } } -/// Atomically write the canonical index `rows` for `fingerprint`; whether the write landed. +/// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. #[pyfunction] fn write_scope_index_cache( py: Python<'_>, location: (PathBuf, String), fingerprint: String, - rows: IndexRows, + index: &Bound<'_, PyAny>, ) -> PyResult { - let index: CachedIndex = cached_index(rows)?; + if !read_completeness(index)?.iter().all(|done| *done) { + return Ok(false); + } + let index: CachedIndex = read_index(index)?; let (directory, file_name) = location; py.compiler_detach(|| { Ok(match encode_scope_cache(&index, &fingerprint) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 271d118fb4..0aa58ddfae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -1,267 +1,929 @@ -//! Native declaration scopes as the plain rows the Python scope facade materialises. +//! The static scope index built natively from discovered inputs, as the Python scope models. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{IntoPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use std::collections::HashMap; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_scopes::relationship_names::models::RelationshipScan; use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; +use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; use sqlbuild_scopes::scope_index::models::{ - DeclarationIdentity, DeclarationInput, DeclarationRecord, GrantRecord, GrantThrough, - RelationshipFact, ResourceIdentity, ResourceInput, ScopeDeferral, ScopeDiagnostic, ScopeIndex, - ScopeInputs, -}; -use sqlbuild_scopes::scope_index::models::{ - DeclarationKind, ResourceKind, ResourceRoot, ScopeKind, + DeclarationIdentity, DeclarationInput, DeclarationKind, GrantRecord, GrantThrough, + RejectedPath, RelationshipFact, ResourceIdentity, ResourceInput, ResourceKind, ResourceRoot, + ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, }; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::scopes::relationship_scan::{ RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, }; +use crate::bindings::_helpers::scopes::scope_lookup::{path_message, scope_path_error}; +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, field, optional_item, root_object, value_of, +}; +use crate::bindings::_helpers::scopes::scope_validation::replace; use crate::bindings::types::CompilerDetach; -use sqlbuild_scopes::relationship_names::models::RelationshipScan; -type IdentityRow = (String, String); -type DeclarationIdentityRow = (String, String, Option); -type ResourceRow = (String, String, String, String); -type DeclarationRow = ( - String, - String, - Option, - String, - i64, - String, - Option, - String, - Option, - Vec, -); -type DeclarationRecordRow = ( - String, - String, - &'static str, - Option<&'static str>, - Option, -); -type DiagnosticRow = ( - &'static str, - String, - Option, - Option, - Option, - Option, - Option, -); -type IndexRows = ( - Vec, - Vec, - Vec, - Vec, - Vec<(usize, usize)>, - Vec, - bool, -); -type GrantRow = (&'static str, String, usize, ThroughRow, &'static str); -/// The grants, or `None` when the index cannot place them, and each `(path, message)` fault. -type GrantedRelationships = (Option>, Vec<(String, String)>); - -/// What a grant came through: an expected model's name or a tested macro's declaration index. -#[derive(IntoPyObject)] -enum ThroughRow { - Model(String), - Declaration(usize), +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const INSPECT_MODULE: &str = "inspect"; +const CODE_TYPE_MODULE: &str = "types"; +const PATHLIB_MODULE: &str = "pathlib"; +const PRIVATE_ROOT: &str = "models"; +const MACRO_FALLBACK: &str = "macros"; +const ENUM_FALLBACK: &str = "enums"; +const CONSTANT_FALLBACK: &str = "constants"; +const COLLECTION_KINDS: [&str; 3] = ["list", "set", "object"]; +const NULL_KIND: &str = "null"; +/// Each resource kind's own ownership root, in the order Python sorts them by path. +const KIND_ROOTS: [(ResourceKind, &str); 5] = [ + (ResourceKind::Function, "functions/sql"), + (ResourceKind::Model, "models"), + (ResourceKind::Source, "sources"), + (ResourceKind::Scenario, "tests/scenarios"), + (ResourceKind::Test, "tests/unit"), +]; + +/// The Python parts of one declaration record the native index does not hold. +struct DeclarationValues<'py> { + identity: Bound<'py, PyAny>, + line: i64, + scope: Bound<'py, PyAny>, + metadata: [Option>; 3], + dependencies: Vec>, } -/// A built scope index held natively between the facade's index, grant and lookup steps. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct NativeScopeIndex { - index: ScopeIndex, - grants: Vec, +/// Where one declaration lives: path, ownership root, root fallback and owning path. +struct Placement { + path: String, + ownership_root: Option, + root_fallback: String, + owning_path: Option, } -#[pymethods] -impl NativeScopeIndex { - /// The index's records, builder orders, usages, diagnostics and relationship flag. - fn records(&self) -> IndexRows { - let index: &ScopeIndex = &self.index; - ( - index - .resources - .iter() - .map(|record| record.path.clone()) - .collect(), - index.declarations.iter().map(declaration_row).collect(), - index.resource_order.clone(), - index.declaration_order.clone(), - index - .usages - .iter() - .map(|usage| (usage.consumer, usage.dependency)) - .collect(), - index.diagnostics.iter().map(diagnostic_row).collect(), - index.has_scoped_relationship_declarations, +/// The builder's walk of discovered inputs: native inputs and the Python values beside them. +struct Walk<'py> { + classes: ScopeClasses<'py>, + inputs: ScopeInputs, + resource_identities: Vec>, + /// Each resource's kind and root, which pick its ownership root. + resource_roots: Vec<(ResourceKind, ResourceRoot)>, + declarations: Vec>, + /// Whether every walked kind, scope and root parsed natively. + indexable: bool, +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn optional_text(item: &Bound<'_, PyAny>, name: &str) -> PyResult> { + optional_item(item, name)? + .map(|value| text(&value)) + .transpose() +} + +fn stem(path: &Bound<'_, PyAny>) -> PyResult { + path.getattr("stem")?.extract() +} + +/// `value or fallback` with Python truthiness. +fn or_else<'py>( + value: Bound<'py, PyAny>, + fallback: impl FnOnce() -> PyResult>, +) -> PyResult> { + if value.is_truthy()? { + Ok(value) + } else { + fallback() + } +} + +/// `value if isinstance(value, str) and value else path.stem`. +fn name_or_stem(value: &Bound<'_, PyAny>, path: &Bound<'_, PyAny>) -> PyResult { + if let Ok(name) = value.extract::() + && !name.is_empty() + { + return Ok(name); + } + stem(path) +} + +fn header_name<'py>(file: &Bound<'py, PyAny>) -> PyResult> { + file.getattr("header_values")? + .call_method1("get", ("name",)) +} + +fn items<'py>(item: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + item.getattr(name)?.try_iter()?.collect() +} + +impl<'py> Walk<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + classes: ScopeClasses::new(py)?, + inputs: ScopeInputs::default(), + resource_identities: Vec::new(), + resource_roots: Vec::new(), + declarations: Vec::new(), + indexable: true, + }) + } + + fn add_resource( + &mut self, + kind: ResourceKind, + name: String, + location: (&Bound<'py, PyAny>, ResourceRoot), + ) -> PyResult<()> { + let (path, root) = location; + self.resource_identities + .push(self.classes.model("ResourceIdentity")?.call1(( + self.classes.member("ResourceKind", kind.as_str())?, + name.as_str(), + ))?); + self.resource_roots.push((kind, root)); + self.inputs.resources.push(ResourceInput { + identity: ResourceIdentity { kind, name }, + path: text(path)?, + root, + }); + Ok(()) + } + + fn native_identity(&mut self, identity: &Bound<'py, PyAny>) -> PyResult { + let kind: String = value_of(identity, "kind")?; + let owner: Option = match optional_item(identity, "owner")? { + None => None, + Some(owner) => { + let owner_kind: String = value_of(&owner, "kind")?; + match ResourceKind::parse(&owner_kind) { + Some(kind) => Some(ResourceIdentity { + kind, + name: field(&owner, "name")?, + }), + None => { + self.indexable = false; + None + } + } + } + }; + let kind: DeclarationKind = DeclarationKind::parse(&kind).unwrap_or_else(|| { + self.indexable = false; + DeclarationKind::Macro + }); + Ok(DeclarationIdentity { + kind, + name: field(identity, "name")?, + owner, + }) + } + + fn add_declaration( + &mut self, + values: DeclarationValues<'py>, + placement: Placement, + ) -> PyResult<()> { + let identity: DeclarationIdentity = self.native_identity(&values.identity)?; + let scope_text: String = values.scope.getattr("value")?.extract()?; + let scope: ScopeKind = ScopeKind::parse(&scope_text).unwrap_or_else(|| { + self.indexable = false; + ScopeKind::Global + }); + let dependencies: Vec = values + .dependencies + .iter() + .map(|item| self.native_identity(item)) + .collect::>()?; + self.inputs.declarations.push(DeclarationInput { + identity, + path: placement.path, + line: values.line, + scope, + ownership_root: placement.ownership_root, + root_fallback: placement.root_fallback, + owning_path: placement.owning_path, + dependencies, + }); + self.declarations.push(values); + Ok(()) + } + + fn declaration_identity( + &self, + kind: DeclarationKind, + name: &Bound<'py, PyAny>, + owner: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + self.classes.model("DeclarationIdentity")?.call1(( + self.classes.member("DeclarationKind", kind.as_str())?, + name, + owner, + )) + } + + fn values( + &self, + identity: Bound<'py, PyAny>, + scope: Bound<'py, PyAny>, + metadata: [Option>; 3], + ) -> DeclarationValues<'py> { + DeclarationValues { + identity, + line: 1, + scope, + metadata, + dependencies: Vec::new(), + } + } + + fn enum_metadata(&self, declaration: &Bound<'py, PyAny>) -> PyResult> { + let member_class: Bound<'py, PyAny> = self.classes.model("EnumMemberMetadata")?; + let members: Vec> = items(declaration, "members")? + .iter() + .map(|member| member_class.call1((member.getattr("name")?,))) + .collect::>()?; + self.classes.model("EnumMetadata")?.call1(( + PyTuple::new(declaration.py(), members)?, + declaration.getattr("scalar_type")?, + )) + } + + fn constant_metadata(&self, declaration: &Bound<'py, PyAny>) -> PyResult> { + let value: Bound<'py, PyAny> = declaration.getattr("value")?; + let kind: Bound<'py, PyAny> = value.getattr("kind")?; + let kind_text: String = kind.getattr("value")?.extract()?; + let is_collection: bool = COLLECTION_KINDS.contains(&kind_text.as_str()); + let payload: Bound<'py, PyAny> = value.getattr("value")?; + let item_count: Option = if is_collection && payload.is_instance_of::() { + Some(payload.len()?) + } else { + None + }; + let render_as: Option> = optional_item(declaration, "render_as")? + .map(|rendering| rendering.getattr("value")) + .transpose()?; + self.classes.model("ConstantMetadata")?.call1(( + declaration + .getattr("logical_type")? + .getattr("display_name")?, + if is_collection { + Some(kind.getattr("value")?) + } else { + None + }, + item_count, + kind_text == NULL_KIND, + render_as, + )) + } + + fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + for model_file in items(inputs, "model_files")? { + let path: Bound<'py, PyAny> = model_file.getattr("relative_path")?; + let model_name: String = name_or_stem(&header_name(&model_file)?, &path)?; + self.add_resource( + ResourceKind::Model, + model_name.clone(), + (&path, ResourceRoot::Kind), + )?; + let owner: Bound<'py, PyAny> = self.classes.model("ResourceIdentity")?.call1(( + self.classes + .member("ResourceKind", ResourceKind::Model.as_str())?, + model_name, + ))?; + let private: Bound<'py, PyAny> = self + .classes + .member("ScopeKind", ScopeKind::Private.as_str())?; + for declaration in items(&model_file, "enum_declarations")? { + let identity: Bound<'py, PyAny> = self.declaration_identity( + DeclarationKind::Enum, + &declaration.getattr("name")?, + Some(&owner), + )?; + let metadata = [None, Some(self.enum_metadata(&declaration)?), None]; + let values = self.values(identity, private.clone(), metadata); + self.add_private(values, &path)?; + } + for declaration in items(&model_file, "constant_declarations")? { + let identity: Bound<'py, PyAny> = self.declaration_identity( + DeclarationKind::Constant, + &declaration.getattr("name")?, + Some(&owner), + )?; + let metadata = [None, None, Some(self.constant_metadata(&declaration)?)]; + let values = self.values(identity, private.clone(), metadata); + self.add_private(values, &path)?; + } + } + Ok(()) + } + + fn add_private( + &mut self, + values: DeclarationValues<'py>, + path: &Bound<'py, PyAny>, + ) -> PyResult<()> { + self.add_declaration( + values, + Placement { + path: text(path)?, + ownership_root: Some(PRIVATE_ROOT.to_owned()), + root_fallback: PRIVATE_ROOT.to_owned(), + owning_path: Some(text(&path.getattr("parent")?)?), + }, ) } - /// Read every test's and scenario's relationship facts, then resolve and keep their grants. - fn grant_relationships( + fn add_resources(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + for test_file in items(inputs, "test_files")? { + let path: Bound<'py, PyAny> = test_file.getattr("relative_path")?; + for block in items(&test_file, "blocks")? { + let name: Bound<'py, PyAny> = + or_else(block.getattr("name")?, || path.getattr("stem"))?; + self.add_resource( + ResourceKind::Test, + name.extract()?, + (&path, ResourceRoot::Kind), + )?; + } + } + for scenario in items(inputs, "scenario_files")? { + self.add_resource( + ResourceKind::Scenario, + field(&scenario, "name")?, + (&scenario.getattr("relative_path")?, ResourceRoot::Kind), + )?; + } + for function in items(inputs, "sql_function_files")? { + let path: Bound<'py, PyAny> = function.getattr("relative_path")?; + let name: String = name_or_stem(&header_name(&function)?, &path)?; + self.add_resource(ResourceKind::Function, name, (&path, ResourceRoot::Kind))?; + } + for function in items(inputs, "python_function_files")? { + self.add_resource( + ResourceKind::Function, + stem(&function.getattr("file_path")?)?, + ( + &function.getattr("relative_path")?, + ResourceRoot::PythonFunction, + ), + )?; + } + for seed in items(inputs, "seed_files")? { + self.add_resource( + ResourceKind::Seed, + stem(&seed.getattr("file_path")?)?, + (&seed.getattr("relative_path")?, ResourceRoot::Seed), + )?; + } + for source_file in items(inputs, "source_files")? { + let path: Bound<'py, PyAny> = source_file.getattr("relative_path")?; + for source in items(&source_file, "source_entries")? { + self.add_resource( + ResourceKind::Source, + field(&source, "name")?, + (&path, ResourceRoot::Kind), + )?; + } + } + Ok(()) + } + + fn file_placement(file: &Bound<'py, PyAny>, root_fallback: String) -> PyResult { + Ok(Placement { + path: text(&file.getattr("relative_path")?)?, + ownership_root: optional_text(file, "ownership_root")?, + root_fallback, + owning_path: optional_text(file, "owning_path")?, + }) + } + + fn add_declaration_files(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + for enum_file in items(inputs, "enum_files")? { + for declaration in items(&enum_file, "declarations")? { + let identity: Bound<'py, PyAny> = self.declaration_identity( + DeclarationKind::Enum, + &declaration.getattr("name")?, + None, + )?; + let metadata = [None, Some(self.enum_metadata(&declaration)?), None]; + let values = self.values(identity, enum_file.getattr("scope_kind")?, metadata); + let placement = Self::file_placement(&enum_file, ENUM_FALLBACK.to_owned())?; + self.add_declaration(values, placement)?; + } + } + for constant_file in items(inputs, "constant_files")? { + for declaration in items(&constant_file, "declarations")? { + let identity: Bound<'py, PyAny> = self.declaration_identity( + DeclarationKind::Constant, + &declaration.getattr("name")?, + None, + )?; + let metadata = [None, None, Some(self.constant_metadata(&declaration)?)]; + let values = self.values(identity, constant_file.getattr("scope_kind")?, metadata); + let placement = Self::file_placement(&constant_file, CONSTANT_FALLBACK.to_owned())?; + self.add_declaration(values, placement)?; + } + } + Ok(()) + } + + fn add_named( &mut self, - py: Python<'_>, - files: Vec, - scenarios: Vec, - settings: RelationshipScanSettings, - ) -> PyResult { - let scan: RelationshipScan = scanned_relationships(py, &files, &scenarios, settings)?; - let index: &ScopeIndex = &self.index; - let facts: &[RelationshipFact] = &scan.facts; - let grants: Result, ScopeDeferral> = py - .compiler_detach(|| Ok(relationship_grants(index, facts))) - .map_err(compiler_error)?; - let Ok(grants) = grants else { - return Ok((None, scan.faults)); + item: &Bound<'py, PyAny>, + kind: DeclarationKind, + name: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let identity: Bound<'py, PyAny> = self.declaration_identity(kind, name, None)?; + let values = self.values(identity, item.getattr("scope_kind")?, [None, None, None]); + let fallback: Bound<'py, PyAny> = or_else(item.getattr("declaration_root")?, || { + item.getattr("relative_path")?.getattr("parent") + })?; + let placement = Self::file_placement(item, text(&fallback)?)?; + self.add_declaration(values, placement) + } + + fn add_named_declarations(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + let audit_kind: String = DeclarationKind::Audit.as_str().to_owned(); + for audit_file in items(inputs, "audit_files")? { + let stem_name: Bound<'py, PyAny> = + audit_file.getattr("relative_path")?.getattr("stem")?; + if value_of(&audit_file, "declaration_kind")? == audit_kind { + self.add_named(&audit_file, DeclarationKind::Audit, &stem_name)?; + continue; + } + for block in items(&audit_file, "blocks")? { + let name: Bound<'py, PyAny> = + or_else(block.getattr("name")?, || Ok(stem_name.clone()))?; + self.add_named(&audit_file, DeclarationKind::SingularAudit, &name)?; + } + } + for schema_file in items(inputs, "model_schema_files")? { + for declaration in items(&schema_file, "declarations")? { + self.add_named( + &schema_file, + DeclarationKind::Schema, + &declaration.getattr("name")?, + )?; + } + } + for hook_file in items(inputs, "sql_hook_files")? { + self.add_named( + &hook_file, + DeclarationKind::SqlHook, + &hook_file.getattr("name")?, + )?; + } + for hook in items(inputs, "hook_functions")? { + self.add_named(&hook, DeclarationKind::PythonHook, &hook.getattr("name")?)?; + } + Ok(()) + } + + fn macro_values( + &self, + macro_file: Option<&Bound<'py, PyAny>>, + source: (&Bound<'py, PyAny>, i64, Bound<'py, PyAny>), + ) -> PyResult<(DeclarationValues<'py>, Placement)> { + let (export, line, metadata) = source; + let identity: Bound<'py, PyAny> = + self.declaration_identity(DeclarationKind::Macro, &export.getattr("name")?, None)?; + let scope: Bound<'py, PyAny> = match macro_file { + Some(file) => file.getattr("scope_kind")?, + None => self + .classes + .member("ScopeKind", ScopeKind::Global.as_str())?, }; - let rows: Vec = grants.iter().map(grant_row).collect(); - self.grants = grants; - Ok((Some(rows), scan.faults)) + let dependencies: Vec> = items(export, "dependencies")?; + let placement: Placement = Placement { + path: text(&export.getattr("relative_path")?)?, + ownership_root: match macro_file { + Some(file) => optional_text(file, "ownership_root")?, + None => None, + }, + root_fallback: MACRO_FALLBACK.to_owned(), + owning_path: match macro_file { + Some(file) => optional_text(file, "owning_path")?, + None => None, + }, + }; + Ok(( + DeclarationValues { + identity, + line, + scope, + metadata: [Some(metadata), None, None], + dependencies, + }, + placement, + )) } -} -/// Build the static scope index natively, or return `None` when Python must build it. -#[pyfunction] -fn build_native_scope_index( - py: Python<'_>, - resources: Vec, - declarations: Vec, -) -> PyResult> { - let Some(inputs) = scope_inputs(resources, declarations) else { - return Ok(None); - }; - let index: Result = py - .compiler_detach(|| Ok(build_scope_index(inputs))) - .map_err(compiler_error)?; - match index { - Ok(index) => Ok(Some(NativeScopeIndex { - index, - grants: Vec::new(), - })), - Err(_deferral) => Ok(None), + fn loaded_metadata(&self, loaded: &Bound<'py, PyAny>) -> PyResult<(i64, Bound<'py, PyAny>)> { + let py: Python<'py> = loaded.py(); + let function: Bound<'py, PyAny> = loaded.getattr("function")?; + let parameters: Vec> = py + .import(INSPECT_MODULE)? + .getattr("signature")? + .call1((&function,))? + .getattr("parameters")? + .try_iter()? + .collect::>()?; + let raw_source: String = field(loaded, "raw_source")?; + let metadata: Bound<'py, PyAny> = self.classes.model("MacroMetadata")?.call1(( + PyTuple::new(py, parameters)?, + loaded.getattr("dependencies")?, + macro_source_digest(&raw_source), + ))?; + let code_type: Bound<'py, PyAny> = py.import(CODE_TYPE_MODULE)?.getattr("CodeType")?; + let line: i64 = match function.getattr_opt("__code__")? { + Some(code) if code.is_instance(&code_type)? => { + code.getattr("co_firstlineno")?.extract()? + } + _ => 1, + }; + Ok((line, metadata)) + } + + fn add_macros( + &mut self, + inputs: &Bound<'py, PyAny>, + macros: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + ) -> PyResult<()> { + let (loaded_macros, static_macros) = macros; + let py: Python<'py> = inputs.py(); + let mut macro_files: HashMap> = HashMap::new(); + for macro_file in items(inputs, "macro_files")? { + let key: String = normalized(py, ¯o_file.getattr("relative_path")?)?; + macro_files.insert(key, macro_file); + } + for loaded in loaded_macros.call_method0("values")?.try_iter()? { + let loaded: Bound<'py, PyAny> = loaded?; + let key: String = normalized(py, &loaded.getattr("relative_path")?)?; + let (line, metadata) = self.loaded_metadata(&loaded)?; + let (values, placement) = + self.macro_values(macro_files.get(&key), (&loaded, line, metadata))?; + self.add_declaration(values, placement)?; + } + for export in static_macros.try_iter()? { + let export: Bound<'py, PyAny> = export?; + let key: String = normalized(py, &export.getattr("relative_path")?)?; + let metadata: Bound<'py, PyAny> = self.classes.model("MacroMetadata")?.call1(( + export.getattr("parameters")?, + export.getattr("dependencies")?, + export.getattr("source_digest")?, + ))?; + let (values, placement) = self.macro_values( + macro_files.get(&key), + (&export, field(&export, "line")?, metadata), + )?; + self.add_declaration(values, placement)?; + } + Ok(()) } } -fn scope_inputs( - resources: Vec, - declarations: Vec, -) -> Option { - Some(ScopeInputs { - resources: resources - .into_iter() - .map(|(kind, name, path, root)| { - Some(ResourceInput { - identity: resource_identity((kind, name))?, - path, - root: match root.as_str() { - "kind" => ResourceRoot::Kind, - "seed" => ResourceRoot::Seed, - "python_function" => ResourceRoot::PythonFunction, - _ => return None, - }, - }) - }) - .collect::>()?, - declarations: declarations - .into_iter() - .map(declaration_input) - .collect::>()?, +/// Python's scope `normalize_path` of a path; a rejected path raises `InvalidScopePathError`. +fn normalized(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { + let path: String = text(path)?; + normalize_scope_path(&path).map_err(|failure| { + scope_path_error(py, path_message(&RejectedPath { failure, path })) + .unwrap_or_else(|error| error) }) } -fn declaration_input(row: DeclarationRow) -> Option { - let ( - kind, - name, - owner, - path, - line, - scope, - ownership_root, - root_fallback, - owning_path, - dependencies, - ) = row; - Some(DeclarationInput { - identity: declaration_identity((kind, name, owner))?, - path, - line, - scope: ScopeKind::parse(&scope)?, - ownership_root, - root_fallback, - owning_path, - dependencies: dependencies - .into_iter() - .map(declaration_identity) - .collect::>()?, - }) +fn unindexable(py: Python<'_>) -> PyResult { + Ok(PyErr::from_value( + py.import(EXCEPTIONS_MODULE)? + .getattr("UnindexableScopeError")? + .call0()?, + )) } -fn resource_identity((kind, name): IdentityRow) -> Option { - Some(ResourceIdentity { - kind: ResourceKind::parse(&kind)?, - name, - }) +/// A built scope index and the Python records materialised from it. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct NativeScopeIndex { + native: ScopeIndex, + /// Each declaration record in input order. + declarations: Vec>, + index: Py, + grants: Py, } -fn declaration_identity( - (kind, name, owner): DeclarationIdentityRow, -) -> Option { - Some(DeclarationIdentity { - kind: DeclarationKind::parse(&kind)?, - name, - owner: match owner { - Some(owner) => Some(resource_identity(owner)?), - None => None, +fn resource_root<'py>( + classes: &ScopeClasses<'py>, + kind: ResourceKind, + root: ResourceRoot, +) -> PyResult> { + let (path, root_kind, resource_kind): (&str, &str, ResourceKind) = match root { + ResourceRoot::Seed => ("seeds", "global", ResourceKind::Seed), + ResourceRoot::PythonFunction => ("functions/python", "global", ResourceKind::Function), + ResourceRoot::Kind => match KIND_ROOTS.iter().find(|(candidate, _)| *candidate == kind) { + Some((_, path)) => (path, "resource", kind), + None => { + return Err(PyKeyError::new_err( + classes.member("ResourceKind", kind.as_str())?.unbind(), + )); + } }, + }; + classes.model("OwnershipRoot")?.call1(( + path, + classes.member("OwnershipRootKind", root_kind)?, + classes.member("ResourceKind", resource_kind.as_str())?, + )) +} + +fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult { + let classes: &ScopeClasses<'py> = &walk.classes; + let py: Python<'py> = classes.py(); + let mut resources: Vec> = Vec::with_capacity(native.resources.len()); + for ((record, (kind, root)), identity) in native + .resources + .iter() + .zip(&walk.resource_roots) + .zip(&walk.resource_identities) + { + resources.push(classes.model("ResourceRecord")?.call1(( + identity, + record.path.as_str(), + resource_root(classes, *kind, *root)?, + ))?); + } + let mut declarations: Vec> = Vec::with_capacity(native.declarations.len()); + for (record, values) in native.declarations.iter().zip(&walk.declarations) { + let [macro_metadata, enum_metadata, constant_metadata] = &values.metadata; + let arguments: Vec> = vec![ + values.identity.clone(), + record.path.as_str().into_bound_py_any(py)?, + values.line.into_bound_py_any(py)?, + 1_i64.into_bound_py_any(py)?, + values.scope.clone(), + root_object(classes, &record.ownership_root)?, + record.owning_path.as_deref().into_bound_py_any(py)?, + macro_metadata.into_bound_py_any(py)?, + enum_metadata.into_bound_py_any(py)?, + constant_metadata.into_bound_py_any(py)?, + ]; + declarations.push( + classes + .model("DeclarationRecord")? + .call1(PyTuple::new(py, arguments)?)?, + ); + } + let usage_class: Bound<'py, PyAny> = classes.model("UsageRecord")?; + let dependency_kind: Bound<'py, PyAny> = + classes.member("UsageKind", "declaration_dependency")?; + let mut usages: Vec> = Vec::with_capacity(native.usages.len()); + for usage in &native.usages { + usages.push(usage_class.call1(( + declarations[usage.consumer].getattr("identity")?, + &walk.declarations[usage.consumer].dependencies[usage.dependency], + &dependency_kind, + ))?); + } + let diagnostic_class: Bound<'py, PyAny> = classes.model("ScopeDiagnostic")?; + let mut diagnostics: Vec> = Vec::with_capacity(native.diagnostics.len()); + for diagnostic in &native.diagnostics { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("path", diagnostic.path.as_deref())?; + keywords.set_item("line", diagnostic.line)?; + keywords.set_item("column", diagnostic.column)?; + keywords.set_item( + "declaration", + diagnostic + .declaration + .map(|position| declarations[position].getattr("identity")) + .transpose()?, + )?; + keywords.set_item( + "resource", + diagnostic + .resource + .map(|position| resources[position].getattr("identity")) + .transpose()?, + )?; + diagnostics.push(diagnostic_class.call( + ( + classes.member("ScopeDiagnosticCode", diagnostic.code.as_str())?, + diagnostic.message.as_str(), + ), + Some(&keywords), + )?); + } + let roots: Vec> = KIND_ROOTS + .iter() + .map(|(kind, path)| { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("path", *path)?; + keywords.set_item( + "resource_kind", + classes.member("ResourceKind", kind.as_str())?, + )?; + classes.model("OwnershipRoot")?.call((), Some(&keywords)) + }) + .collect::>()?; + let completeness: Bound<'py, PyAny> = { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for section in [ + "runtime_usage", + "relationships", + "placement", + "promotion_impact", + ] { + keywords.set_item(section, false)?; + } + classes + .model("ScopeCompleteness")? + .call((), Some(&keywords))? + }; + let ordered = + |records: &[Bound<'py, PyAny>], order: &[usize]| -> PyResult> { + PyTuple::new(py, order.iter().map(|position| records[*position].clone())) + }; + let index: Bound<'py, PyAny> = classes.model("ScopeIndex")?.call1(( + PyTuple::new(py, roots)?, + ordered(&resources, &native.resource_order)?, + ordered(&declarations, &native.declaration_order)?, + PyTuple::new(py, usages)?, + PyTuple::empty(py), + PyTuple::empty(py), + PyTuple::empty(py), + PyTuple::new(py, diagnostics)?, + completeness, + ))?; + Ok(NativeScopeIndex { + native, + declarations: declarations.into_iter().map(Bound::unbind).collect(), + index: index.unbind(), + grants: PyTuple::empty(py).unbind(), }) } -fn declaration_row(record: &DeclarationRecord) -> DeclarationRecordRow { - ( - record.path.clone(), - record.ownership_root.path.clone(), - record.ownership_root.kind.as_str(), - record - .ownership_root - .resource_kind - .map(ResourceKind::as_str), - record.owning_path.clone(), - ) +fn test_file_row(test_file: &Bound<'_, PyAny>) -> PyResult { + let path: Bound<'_, PyAny> = test_file.getattr("relative_path")?; + let mut blocks: Vec<(String, String, String)> = Vec::new(); + for block in items(test_file, "blocks")? { + let name: Bound<'_, PyAny> = or_else(block.getattr("name")?, || path.getattr("stem"))?; + blocks.push(( + name.extract()?, + field(&block, "sql_body")?, + value_of(&block, "mode")?, + )); + } + Ok((text(&path)?, blocks)) } -fn diagnostic_row(diagnostic: &ScopeDiagnostic) -> DiagnosticRow { - ( - diagnostic.code.as_str(), - diagnostic.message.clone(), - diagnostic.path.clone(), - diagnostic.line, - diagnostic.column, - diagnostic.declaration, - diagnostic.resource, - ) +fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult { + Ok(( + field(scenario, "name")?, + text(&scenario.getattr("relative_path")?)?, + field(scenario, "sql_body")?, + )) } -fn grant_row(grant: &GrantRecord) -> GrantRow { - ( - grant.resource.kind.as_str(), - grant.resource.name.clone(), - grant.declaration, - match &grant.through { - GrantThrough::Model(name) => ThroughRow::Model(name.clone()), - GrantThrough::Declaration(declaration) => ThroughRow::Declaration(*declaration), - }, - grant.kind.as_str(), - ) +#[pymethods] +impl NativeScopeIndex { + /// The static index, without grants. + #[getter] + fn index(&self, py: Python<'_>) -> Py { + self.index.clone_ref(py) + } + + /// Whether any macro, enum or constant is scoped, which makes relationship grants matter. + #[getter] + fn has_scoped_relationship_declarations(&self) -> bool { + self.native.has_scoped_relationship_declarations + } + + /// Grant every test's and scenario's relationships; return each extraction fault. + #[pyo3(signature = (*, discovered_inputs, syntax, scan_store))] + fn grant_relationships( + &mut self, + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + syntax: &Bound<'_, PyAny>, + scan_store: (Bound<'_, PyAny>, String), + ) -> PyResult> { + let files: Vec = items(discovered_inputs, "test_files")? + .iter() + .map(test_file_row) + .collect::>()?; + let scenarios: Vec = items(discovered_inputs, "scenario_files")? + .iter() + .map(scenario_row) + .collect::>()?; + let (store, algorithm) = scan_store; + let settings: Bound<'_, PyDict> = PyDict::new(py); + settings.set_item("syntax", syntax.getattr("native_mapping")?)?; + settings.set_item("store", store)?; + settings.set_item("scan_key", (algorithm, syntax.getattr("cache_key")?))?; + let settings: RelationshipScanSettings = settings.extract()?; + let scan: RelationshipScan = scanned_relationships(py, &files, &scenarios, settings)?; + let index: &ScopeIndex = &self.native; + let facts: &[RelationshipFact] = &scan.facts; + let grants: Result, ScopeDeferral> = py + .compiler_detach(|| Ok(relationship_grants(index, facts))) + .map_err(compiler_error)?; + let Ok(grants) = grants else { + return Err(unindexable(py)?); + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let mut records: Vec> = Vec::with_capacity(grants.len()); + for grant in &grants { + let through: Bound<'_, PyAny> = match &grant.through { + GrantThrough::Model(name) => classes.model("ResourceIdentity")?.call1(( + classes.member("ResourceKind", ResourceKind::Model.as_str())?, + name.as_str(), + ))?, + GrantThrough::Declaration(position) => { + self.declarations[*position].bind(py).getattr("identity")? + } + }; + records.push( + classes.model("GrantRecord")?.call1(( + classes.model("ResourceIdentity")?.call1(( + classes.member("ResourceKind", grant.resource.kind.as_str())?, + grant.resource.name.as_str(), + ))?, + self.declarations[grant.declaration] + .bind(py) + .getattr("identity")?, + through, + classes.member("GrantKind", grant.kind.as_str())?, + ))?, + ); + } + self.grants = PyTuple::new(py, records)?.unbind(); + let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; + let faults: Vec> = scan + .faults + .into_iter() + .map(|(path, message)| { + Ok(PyTuple::new( + py, + [path_class.call1((path,))?, message.into_bound_py_any(py)?], + )? + .into_any()) + }) + .collect::>()?; + Ok(PyTuple::new(py, faults)?.unbind()) + } + + /// The index with its grants attached and relationships complete. + fn index_with_relationships(&self, py: Python<'_>) -> PyResult> { + let index: &Bound<'_, PyAny> = self.index.bind(py); + let completeness: Bound<'_, PyAny> = replace( + &index.getattr("completeness")?, + &[("relationships", true.into_bound_py_any(py)?)], + )?; + Ok(replace( + index, + &[ + ("grants", self.grants.bind(py).clone().into_any()), + ("completeness", completeness), + ], + )? + .unbind()) + } +} + +/// Python's `open_native_scope_index`: the natively built index of discovered inputs. +#[pyfunction] +fn open_native_scope_index( + py: Python<'_>, + discovered_inputs: &Bound<'_, PyAny>, + loaded_macros: &Bound<'_, PyAny>, + static_macros: &Bound<'_, PyAny>, +) -> PyResult { + let mut walk: Walk<'_> = Walk::new(py)?; + walk.add_models(discovered_inputs)?; + walk.add_resources(discovered_inputs)?; + walk.add_declaration_files(discovered_inputs)?; + walk.add_named_declarations(discovered_inputs)?; + walk.add_macros(discovered_inputs, (loaded_macros, static_macros))?; + if !walk.indexable { + return Err(unindexable(py)?); + } + let inputs: ScopeInputs = std::mem::take(&mut walk.inputs); + let built: Result = py + .compiler_detach(|| Ok(build_scope_index(inputs))) + .map_err(compiler_error)?; + let Ok(native) = built else { + return Err(unindexable(py)?); + }; + materialise(walk, native) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(build_native_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 67a4ca1d50..f6dd0d5a72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -1,76 +1,35 @@ //! The scope lookup of a full scope index: canonical orders, mappings, targets and visibility. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{pyclass, pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; use sqlbuild_scopes::scope_index::main::report_target::report_target; use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; use sqlbuild_scopes::scope_index::models::{ - GrantKind, IdentityFailure, KeyedGroups, RejectedPath, ReportDiagnosticKey, ReportGrant, - ReportLookup, ReportRecords, ReportTarget, ReportUsage, ResolvedVisibility, ResolvedVisible, - ScopePathFailure, TargetFailure, TargetMatch, + GrantKind, IdentityFailure, KeyedGroups, RejectedPath, ReportLookup, ReportRecords, + ReportTarget, ResolvedVisibility, ResolvedVisible, ResourceRecord, ScopePathFailure, + TargetFailure, TargetMatch, VisibilityGroups, }; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::scopes::placement::{ - DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, - declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, +use crate::bindings::_helpers::scopes::placement::{QualifiedRow, qualified_identity}; +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, read_index, report_records, resource_identity_object, tuple_attribute, }; use crate::bindings::types::CompilerDetach; -/// Consumer, declaration, usage kind and what the consumer uses it through. -type UsageRow = (QualifiedRow, QualifiedRow, String, Option); -/// Resource, declaration, what it came through, and the grant kind. -type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); -/// Path, line, column and code of a diagnostic. -type DiagnosticKeyRow = (Option, Option, Option, String); -/// Every record of a full scope index as rows. -type RecordRows = ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, -); -type GroupRows = Vec>; -/// Roots, resources, declarations, usages, grants and diagnostics in canonical order. -type OrderRows = ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, -); -/// Each mapping's groups, then every input resource's normalized path key. -type MappingRows = ( - GroupRows, - GroupRows, - GroupRows, - GroupRows, - GroupRows, - GroupRows, - Vec, -); -/// Global, private, local and inherited canonical declaration positions. -type VisibilityRows = ( - Vec, - Vec<(IdentityRow, Vec)>, - Vec<(String, Vec)>, - Vec<(String, Vec)>, -); -/// The canonical orders, mappings and visibility groups, else the rejected path's message. -type LookupRows = ( - Option<(OrderRows, MappingRows, VisibilityRows)>, - Option, -); +const TYPES_MODULE: &str = "types"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; + /// `(declarations, canonical positions)` matched, else identity and path error messages. -type TargetRows = (bool, Vec, Option, Option); +pub(crate) type TargetRows = (bool, Vec, Option, Option); /// Visible `(position, reason, grant)`, inaccessible `(position, reason)`, else the path error. -type ResolvedRows = ( +pub(crate) type ResolvedRows = ( Vec<(usize, &'static str, Option)>, Vec<(usize, &'static str)>, Option, @@ -110,45 +69,6 @@ pub(crate) fn grant_kind(value: &str) -> PyResult { } } -fn report_records(rows: RecordRows) -> PyResult { - let (roots, resources, declarations, usages, grants, diagnostics) = rows; - let mut records: ReportRecords = ReportRecords::default(); - for root in roots { - records.roots.push(ownership_root(root)?); - } - for row in resources { - records.resources.push(resource_record(row)?); - } - for row in declarations { - records.declarations.push(declaration_record(row)?); - } - for (consumer, declaration, kind, through) in usages { - records.usages.push(ReportUsage { - consumer: qualified_identity(consumer)?, - declaration: declaration_identity(declaration)?, - kind, - through: through.map(qualified_identity).transpose()?, - }); - } - for (resource, declaration, through, kind) in grants { - records.grants.push(ReportGrant { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - through: qualified_identity(through)?, - kind: grant_kind(&kind)?, - }); - } - for (path, line, column, code) in diagnostics { - records.diagnostics.push(ReportDiagnosticKey { - path, - line, - column, - code, - }); - } - Ok(records) -} - /// Python's `InvalidScopePathError` message for a rejected path. pub(crate) fn path_message(rejected: &RejectedPath) -> String { match rejected.failure { @@ -177,44 +97,6 @@ pub(crate) fn identity_message(failure: &IdentityFailure, value: &str) -> String } } -fn group_rows(groups: KeyedGroups) -> GroupRows { - groups.groups -} - -fn lookup_rows(lookup: &ReportLookup) -> (OrderRows, MappingRows, VisibilityRows) { - let groups = lookup.groups.clone(); - let visibility = lookup.visibility.clone(); - ( - ( - lookup.roots.clone(), - lookup.resources.clone(), - lookup.declarations.clone(), - lookup.usages.clone(), - lookup.grants.clone(), - lookup.diagnostics.clone(), - ), - ( - group_rows(groups.resources), - group_rows(groups.resources_by_path), - group_rows(groups.declarations), - group_rows(groups.usages_by_consumer), - group_rows(groups.usages_by_declaration), - group_rows(groups.grants_by_resource), - lookup.resource_paths.clone(), - ), - ( - visibility.global, - visibility - .private - .into_iter() - .map(|(owner, positions)| ((owner.kind.as_str().to_owned(), owner.name), positions)) - .collect(), - visibility.local, - visibility.inherited, - ), - ) -} - /// Input positions as canonical positions. fn canonical(order: &[usize], positions: Vec) -> Vec { positions @@ -223,10 +105,13 @@ fn canonical(order: &[usize], positions: Vec) -> Vec { .collect() } -#[pymethods] impl NativeScopeLookup { /// The records `target` names: an identity row, else qualified identity or path text. - fn target(&self, identity: Option, text: &str) -> PyResult { + pub(crate) fn target( + &self, + identity: Option, + text: &str, + ) -> PyResult { let target: ReportTarget = match identity { Some(row) => ReportTarget::Identity(qualified_identity(row)?), None => ReportTarget::Text(text.to_owned()), @@ -257,11 +142,14 @@ impl NativeScopeLookup { } /// Python's `_classify_resource` of one resource, with inaccessible reasons. - fn resolve(&self, py: Python<'_>, resource: ResourceRow) -> PyResult { - let resource = resource_record(resource)?; + pub(crate) fn resolve( + &self, + py: Python<'_>, + resource: &ResourceRecord, + ) -> PyResult { let (records, lookup) = (&self.records, &self.lookup); let resolved = py - .compiler_detach(|| Ok(resolve_report_visibility(records, lookup, &resource))) + .compiler_detach(|| Ok(resolve_report_visibility(records, lookup, resource))) .map_err(compiler_error)?; Ok(match resolved { Ok(resolved) => resolved_rows(resolved, &self.canonical.grants), @@ -292,33 +180,171 @@ fn resolved_rows(resolved: ResolvedVisibility, grants: &[usize]) -> ResolvedRows ) } -/// The lookup of a full scope index's records, else the message of the path Python rejects. +fn picked<'py>(values: &Bound<'py, PyTuple>, order: &[usize]) -> PyResult> { + let items: Vec> = order + .iter() + .map(|position| values.get_item(*position)) + .collect::>()?; + PyTuple::new(values.py(), items) +} + +fn mapping_proxy<'py>(entries: &Bound<'py, PyDict>) -> PyResult> { + entries + .py() + .import(TYPES_MODULE)? + .getattr("MappingProxyType")? + .call1((entries,)) +} + +/// Python's `_mapping`: each group keyed by its first record's `key` attribute. +fn grouped<'py>( + groups: &KeyedGroups, + records: &Bound<'py, PyTuple>, + key: &str, +) -> PyResult> { + let entries: Bound<'py, PyDict> = PyDict::new(records.py()); + for group in &groups.groups { + let members: Bound<'py, PyTuple> = picked(records, group)?; + entries.set_item(members.get_item(0)?.getattr(key)?, members)?; + } + mapping_proxy(&entries) +} + +fn path_grouped<'py>( + groups: &KeyedGroups, + records: &Bound<'py, PyTuple>, + paths: &[String], +) -> PyResult> { + let entries: Bound<'py, PyDict> = PyDict::new(records.py()); + for group in &groups.groups { + entries.set_item(paths[group[0]].as_str(), picked(records, group)?)?; + } + mapping_proxy(&entries) +} + +fn positions_by<'py>( + py: Python<'py>, + entries: &[(String, Vec)], +) -> PyResult> { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, positions) in entries { + mapping.set_item(key.as_str(), PyTuple::new(py, positions)?)?; + } + mapping_proxy(&mapping) +} + +fn visibility_index<'py>( + classes: &ScopeClasses<'py>, + canonical_declarations: &Bound<'py, PyTuple>, + visibility: &VisibilityGroups, +) -> PyResult> { + let py: Python<'py> = canonical_declarations.py(); + let identities: Vec> = canonical_declarations + .iter() + .map(|record| record.getattr("identity")) + .collect::>()?; + let private: Bound<'py, PyDict> = PyDict::new(py); + for (owner, positions) in &visibility.private { + private.set_item( + resource_identity_object(classes, owner)?, + PyTuple::new(py, positions)?, + )?; + } + classes.model("DeclarationVisibilityIndex")?.call1(( + PyTuple::new(py, identities)?, + PyTuple::new(py, &visibility.global)?, + mapping_proxy(&private)?, + positions_by(py, &visibility.local)?, + positions_by(py, &visibility.inherited)?, + )) +} + +pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import(EXCEPTIONS_MODULE)? + .getattr("InvalidScopePathError")? + .call1((message,))?, + )) +} + +/// Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. #[pyfunction] -fn build_native_scope_lookup( - py: Python<'_>, - rows: RecordRows, -) -> PyResult<(Option, LookupRows)> { - let records: ReportRecords = report_records(rows)?; +fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { + let records: ReportRecords = report_records(&read_index(index)?); let built = py .compiler_detach(|| Ok(report_lookup(&records))) .map_err(compiler_error)?; - Ok(match built { - Ok(lookup) => { - let rows: (OrderRows, MappingRows, VisibilityRows) = lookup_rows(&lookup); - let canonical: CanonicalPositions = CanonicalPositions { - resources: inverse(&lookup.resources), - declarations: inverse(&lookup.declarations), - grants: inverse(&lookup.grants), - }; - let native: NativeScopeLookup = NativeScopeLookup { - records, - lookup, - canonical, - }; - (Some(native), (Some(rows), None)) - } - Err(rejected) => (None, (None, Some(path_message(&rejected)))), - }) + let lookup: ReportLookup = match built { + Ok(lookup) => lookup, + Err(rejected) => return Err(scope_path_error(py, path_message(&rejected))?), + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; + let declarations: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let usages: Bound<'_, PyTuple> = tuple_attribute(index, "usages")?; + let grants: Bound<'_, PyTuple> = tuple_attribute(index, "grants")?; + let canonical_declarations: Bound<'_, PyTuple> = picked(&declarations, &lookup.declarations)?; + let canonical_index: Bound<'_, PyAny> = classes.model("ScopeIndex")?.call1(( + picked(&tuple_attribute(index, "ownership_roots")?, &lookup.roots)?, + picked(&resources, &lookup.resources)?, + &canonical_declarations, + picked(&usages, &lookup.usages)?, + picked(&grants, &lookup.grants)?, + index.getattr("visibility")?, + index.getattr("inaccessible")?, + picked(&tuple_attribute(index, "diagnostics")?, &lookup.diagnostics)?, + index.getattr("completeness")?, + ))?; + let groups = &lookup.groups; + let mappings: [Bound<'_, PyAny>; 6] = [ + grouped(&groups.resources, &resources, "identity")?, + path_grouped( + &groups.resources_by_path, + &resources, + &lookup.resource_paths, + )?, + grouped(&groups.declarations, &declarations, "identity")?, + grouped(&groups.usages_by_consumer, &usages, "consumer")?, + grouped(&groups.usages_by_declaration, &usages, "declaration")?, + grouped(&groups.grants_by_resource, &grants, "resource")?, + ]; + let visibility: Bound<'_, PyAny> = + visibility_index(&classes, &canonical_declarations, &lookup.visibility)?; + let canonical: CanonicalPositions = CanonicalPositions { + resources: inverse(&lookup.resources), + declarations: inverse(&lookup.declarations), + grants: inverse(&lookup.grants), + }; + let native: Py = Py::new( + py, + NativeScopeLookup { + records, + lookup, + canonical, + }, + )?; + let [ + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + ] = mappings; + Ok(classes + .model("ScopeLookup")? + .call1(( + canonical_index, + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + visibility, + native, + ))? + .unbind()) } /// Python's scope `normalize_path`, else the `InvalidScopePathError` message. @@ -338,7 +364,7 @@ fn scope_normalize_path(path: &str) -> (Option, Option) { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(build_native_scope_lookup, module)?)?; + module.add_function(wrap_pyfunction!(build_scope_lookup, module)?)?; module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs new file mode 100644 index 0000000000..7ac27f2348 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -0,0 +1,733 @@ +//! Python `ScopeIndex` models read into the native full index, and built back from it. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_scopes::placement::models::{PlacementIndex, PlacementUsage}; +use sqlbuild_scopes::scope_cache::models::{ + CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, + CachedVisibility, ConstantFacts, EnumFacts, MacroFacts, +}; +use sqlbuild_scopes::scope_index::main::format_identity::format_identity; +use sqlbuild_scopes::scope_index::models::{ + DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportDiagnosticKey, ReportGrant, + ReportRecords, ReportUsage, ResourceIdentity, ResourceKind, ResourceRecord, +}; +use sqlbuild_scopes::scope_report::models::{ + Completeness, ConstantValues, DeclarationValues, DiagnosticFacts, EnumValues, MacroValues, + ProjectionGrant, ProjectionIndex, ProjectionStanding, ProjectionUsage, ReportValues, +}; + +use crate::bindings::_helpers::scopes::placement::{ + IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, ownership_root, + qualified_identity, resource_identity, +}; +use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; + +const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const ENUM_MODULE: &str = "enum"; + +/// The scope model and enum modules every built object comes from. +pub(crate) struct ScopeClasses<'py> { + models: Bound<'py, PyModule>, + types: Bound<'py, PyModule>, +} + +impl<'py> ScopeClasses<'py> { + /// Import the scope models and enums. + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + models: py.import(MODELS_MODULE)?, + types: py.import(TYPES_MODULE)?, + }) + } + + /// One scope model class. + pub(crate) fn model(&self, name: &str) -> PyResult> { + self.models.getattr(name) + } + + /// The member of scope enum `name` with `value`; an unknown value raises `ValueError`. + pub(crate) fn member(&self, name: &str, value: &str) -> PyResult> { + self.types.getattr(name)?.call1((value,)) + } + + /// The interpreter the classes belong to. + pub(crate) fn py(&self) -> Python<'py> { + self.models.py() + } +} + +/// `str(value.value)` of an enum, else `str(value)`. +pub(crate) fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { + let enum_type = value.py().import(ENUM_MODULE)?.getattr("Enum")?; + if value.is_instance(&enum_type)? { + return value.getattr("value")?.str()?.extract(); + } + value.str()?.extract() +} + +pub(crate) fn value_of(item: &Bound<'_, PyAny>, name: &str) -> PyResult { + item.getattr(name)?.getattr("value")?.extract() +} + +pub(crate) fn field<'py, T: pyo3::FromPyObject<'py>>( + item: &Bound<'py, PyAny>, + name: &str, +) -> PyResult { + item.getattr(name)?.extract() +} + +pub(crate) fn optional_item<'py>( + item: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + let value: Bound<'py, PyAny> = item.getattr(name)?; + Ok(if value.is_none() { None } else { Some(value) }) +} + +fn items<'py>(item: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + item.getattr(name)?.try_iter()?.collect() +} + +/// A resource identity's `(kind, name)` row. +pub(crate) fn identity_row(identity: &Bound<'_, PyAny>) -> PyResult { + Ok((value_of(identity, "kind")?, field(identity, "name")?)) +} + +/// Python's `qualified_row` of a resource or declaration identity. +pub(crate) fn qualified_row(identity: &Bound<'_, PyAny>) -> PyResult { + let owner: Option = match identity.getattr_opt("owner")? { + Some(owner) if !owner.is_none() => Some(identity_row(&owner)?), + _ => None, + }; + Ok((value_of(identity, "kind")?, field(identity, "name")?, owner)) +} + +/// A resource or declaration identity as the native qualified identity. +pub(crate) fn read_qualified(identity: &Bound<'_, PyAny>) -> PyResult { + qualified_identity(qualified_row(identity)?) +} + +fn read_resource_identity(identity: &Bound<'_, PyAny>) -> PyResult { + resource_identity(identity_row(identity)?) +} + +pub(crate) fn read_declaration_identity( + identity: &Bound<'_, PyAny>, +) -> PyResult { + declaration_identity(qualified_row(identity)?) +} + +fn read_optional_qualified( + item: &Bound<'_, PyAny>, + name: &str, +) -> PyResult> { + optional_item(item, name)? + .map(|value| read_qualified(&value)) + .transpose() +} + +fn root_row(root: &Bound<'_, PyAny>) -> PyResult { + let resource_kind: Option = optional_item(root, "resource_kind")? + .map(|kind| kind.getattr("value")?.extract()) + .transpose()?; + Ok((field(root, "path")?, value_of(root, "kind")?, resource_kind)) +} + +fn read_root(root: &Bound<'_, PyAny>) -> PyResult { + ownership_root(root_row(root)?) +} + +/// One `ResourceRecord` as its native record. +pub(crate) fn read_resource(record: &Bound<'_, PyAny>) -> PyResult { + Ok(ResourceRecord { + identity: read_resource_identity(&record.getattr("identity")?)?, + path: field(record, "path")?, + ownership_root: read_root(&record.getattr("ownership_root")?)?, + }) +} + +fn read_macro(facts: &Bound<'_, PyAny>) -> PyResult { + Ok(MacroFacts { + parameters: field(facts, "parameters")?, + dependencies: items(facts, "dependencies")? + .iter() + .map(read_declaration_identity) + .collect::>()?, + source_digest: field(facts, "source_digest")?, + }) +} + +fn read_enum(facts: &Bound<'_, PyAny>) -> PyResult { + Ok(EnumFacts { + members: items(facts, "members")? + .iter() + .map(|member| field(member, "name")) + .collect::>()?, + scalar_type: field(facts, "scalar_type")?, + }) +} + +fn read_constant(facts: &Bound<'_, PyAny>) -> PyResult { + Ok(ConstantFacts { + logical_type: field(facts, "logical_type")?, + collection_kind: field(facts, "collection_kind")?, + item_count: field(facts, "item_count")?, + nullable: field(facts, "nullable")?, + render_as: field(facts, "render_as")?, + }) +} + +/// One `DeclarationRecord` with its value metadata. +pub(crate) fn read_declaration(record: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedDeclaration { + record: declaration_record(( + qualified_row(&record.getattr("identity")?)?, + field(record, "path")?, + field(record, "line")?, + field(record, "column")?, + value_of(record, "scope")?, + root_row(&record.getattr("ownership_root")?)?, + field(record, "owning_path")?, + ))?, + macro_facts: optional_item(record, "macro")? + .map(|facts| read_macro(&facts)) + .transpose()?, + enum_facts: optional_item(record, "enum")? + .map(|facts| read_enum(&facts)) + .transpose()?, + constant_facts: optional_item(record, "constant")? + .map(|facts| read_constant(&facts)) + .transpose()?, + }) +} + +fn read_usage(usage: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedUsage { + consumer: read_qualified(&usage.getattr("consumer")?)?, + declaration: read_declaration_identity(&usage.getattr("declaration")?)?, + kind: value_of(usage, "kind")?, + through: read_optional_qualified(usage, "through")?, + enum_member: field(usage, "enum_member")?, + }) +} + +fn read_grant(grant: &Bound<'_, PyAny>) -> PyResult { + Ok(ReportGrant { + resource: read_resource_identity(&grant.getattr("resource")?)?, + declaration: read_declaration_identity(&grant.getattr("declaration")?)?, + through: read_qualified(&grant.getattr("through")?)?, + kind: grant_kind(&value_of(grant, "kind")?)?, + }) +} + +fn read_visibility(fact: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedVisibility { + resource: read_resource_identity(&fact.getattr("resource")?)?, + declaration: read_declaration_identity(&fact.getattr("declaration")?)?, + reason: value_of(fact, "reason")?, + through: read_optional_qualified(fact, "through")?, + }) +} + +fn read_inaccessible(fact: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedInaccessible { + resource: read_resource_identity(&fact.getattr("resource")?)?, + declaration: read_declaration_identity(&fact.getattr("declaration")?)?, + reason: value_of(fact, "reason")?, + }) +} + +/// One `ScopeDiagnostic`, with its code and severity as their text. +pub(crate) fn read_diagnostic(diagnostic: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedDiagnostic { + code: enum_text(&diagnostic.getattr("code")?)?, + message: field(diagnostic, "message")?, + severity: enum_text(&diagnostic.getattr("severity")?)?, + path: field(diagnostic, "path")?, + line: field(diagnostic, "line")?, + column: field(diagnostic, "column")?, + declaration: optional_item(diagnostic, "declaration")? + .map(|identity| read_declaration_identity(&identity)) + .transpose()?, + resource: optional_item(diagnostic, "resource")? + .map(|identity| read_resource_identity(&identity)) + .transpose()?, + }) +} + +/// The six `ScopeCompleteness` sections in field order. +pub(crate) fn read_completeness(index: &Bound<'_, PyAny>) -> PyResult<[bool; 6]> { + let done: Bound<'_, PyAny> = index.getattr("completeness")?; + Ok([ + field(&done, "discovery")?, + field(&done, "static_visibility")?, + field(&done, "runtime_usage")?, + field(&done, "relationships")?, + field(&done, "placement")?, + field(&done, "promotion_impact")?, + ]) +} + +fn read_all( + index: &Bound<'_, PyAny>, + name: &str, + read: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult> { + items(index, name)?.iter().map(read).collect() +} + +/// Every record of a Python `ScopeIndex`, in its tuple orders. +pub(crate) fn read_index(index: &Bound<'_, PyAny>) -> PyResult { + Ok(CachedIndex { + roots: read_all(index, "ownership_roots", read_root)?, + resources: read_all(index, "resources", read_resource)?, + declarations: read_all(index, "declarations", read_declaration)?, + usages: read_all(index, "usages", read_usage)?, + grants: read_all(index, "grants", read_grant)?, + visibility: read_all(index, "visibility", read_visibility)?, + inaccessible: read_all(index, "inaccessible", read_inaccessible)?, + diagnostics: read_all(index, "diagnostics", read_diagnostic)?, + completeness: read_completeness(index)?, + }) +} + +/// A Python `ScopeIndex`'s resources, declarations and usages for the placement rules. +pub(crate) fn read_placement_index(index: &Bound<'_, PyAny>) -> PyResult { + Ok(PlacementIndex { + resources: read_all(index, "resources", read_resource)?, + declarations: read_all(index, "declarations", |record| { + Ok(read_declaration(record)?.record) + })?, + usages: read_all(index, "usages", |usage| { + Ok(PlacementUsage { + consumer: read_qualified(&usage.getattr("consumer")?)?, + declaration: read_declaration_identity(&usage.getattr("declaration")?)?, + through: read_optional_qualified(usage, "through")?, + }) + })?, + }) +} + +/// The lookup's records of a full index. +pub(crate) fn report_records(index: &CachedIndex) -> ReportRecords { + ReportRecords { + roots: index.roots.clone(), + resources: index.resources.clone(), + declarations: index + .declarations + .iter() + .map(|item| item.record.clone()) + .collect(), + usages: index + .usages + .iter() + .map(|item| ReportUsage { + consumer: item.consumer.clone(), + declaration: item.declaration.clone(), + kind: item.kind.clone(), + through: item.through.clone(), + }) + .collect(), + grants: index.grants.clone(), + diagnostics: index + .diagnostics + .iter() + .map(|item| ReportDiagnosticKey { + path: item.path.clone(), + line: item.line, + column: item.column, + code: item.code.clone(), + }) + .collect(), + } +} + +/// One declaration's report values. +pub(crate) fn declaration_values(item: &CachedDeclaration) -> DeclarationValues { + DeclarationValues { + macro_values: item.macro_facts.as_ref().map(|facts| MacroValues { + parameters: facts.parameters.clone(), + dependencies: facts.dependencies.clone(), + source_digest: facts.source_digest.clone(), + }), + enum_values: item.enum_facts.as_ref().map(|facts| EnumValues { + members: facts.members.clone(), + scalar_type: facts.scalar_type.clone(), + }), + constant_values: item.constant_facts.as_ref().map(|facts| ConstantValues { + logical_type: facts.logical_type.clone(), + collection_kind: facts.collection_kind.clone(), + item_count: facts.item_count, + nullable: facts.nullable, + render_as: facts.render_as.clone(), + }), + } +} + +/// The six completeness sections as the report engine reads them. +pub(crate) fn completeness(done: [bool; 6]) -> Completeness { + Completeness { + discovery: done[0], + static_visibility: done[1], + runtime_usage: done[2], + relationships: done[3], + placement: done[4], + promotion_impact: done[5], + } +} + +/// A diagnostic's facts, with its identities as qualified text. +pub(crate) fn diagnostic_facts(item: CachedDiagnostic) -> DiagnosticFacts { + DiagnosticFacts { + code: item.code, + message: item.message, + severity: item.severity, + path: item.path, + line: item.line, + column: item.column, + declaration: item + .declaration + .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity))), + resource: item + .resource + .map(|identity| format_identity(&QualifiedIdentity::Resource(identity))), + } +} + +/// The metadata projection's view of a full index. +pub(crate) fn projection_index(index: CachedIndex) -> ProjectionIndex { + ProjectionIndex { + roots: index.roots, + resources: index.resources, + values: ReportValues { + declarations: index.declarations.iter().map(declaration_values).collect(), + completeness: completeness(index.completeness), + }, + declarations: index + .declarations + .into_iter() + .map(|item| item.record) + .collect(), + usages: index + .usages + .into_iter() + .map(|item| ProjectionUsage { + consumer: item.consumer, + declaration: item.declaration, + kind: item.kind, + through: item.through, + enum_member: item.enum_member, + }) + .collect(), + grants: index + .grants + .into_iter() + .map(|item| ProjectionGrant { + resource: item.resource, + declaration: item.declaration, + through: item.through, + kind: item.kind, + }) + .collect(), + visibility: index + .visibility + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: item.through, + }) + .collect(), + inaccessible: index + .inaccessible + .into_iter() + .map(|item| ProjectionStanding { + resource: item.resource, + declaration: item.declaration, + reason: item.reason, + through: None, + }) + .collect(), + diagnostics: index + .diagnostics + .into_iter() + .map(diagnostic_facts) + .collect(), + } +} + +/// A `ResourceIdentity`. +pub(crate) fn resource_identity_object<'py>( + classes: &ScopeClasses<'py>, + identity: &ResourceIdentity, +) -> PyResult> { + classes.model("ResourceIdentity")?.call1(( + classes.member("ResourceKind", identity.kind.as_str())?, + identity.name.as_str(), + )) +} + +/// A `DeclarationIdentity`. +pub(crate) fn declaration_identity_object<'py>( + classes: &ScopeClasses<'py>, + identity: &DeclarationIdentity, +) -> PyResult> { + let owner: Option> = identity + .owner + .as_ref() + .map(|owner| resource_identity_object(classes, owner)) + .transpose()?; + classes.model("DeclarationIdentity")?.call1(( + classes.member("DeclarationKind", identity.kind.as_str())?, + identity.name.as_str(), + owner, + )) +} + +/// A `ResourceIdentity` or `DeclarationIdentity`. +pub(crate) fn qualified_object<'py>( + classes: &ScopeClasses<'py>, + identity: &QualifiedIdentity, +) -> PyResult> { + match identity { + QualifiedIdentity::Resource(identity) => resource_identity_object(classes, identity), + QualifiedIdentity::Declaration(identity) => declaration_identity_object(classes, identity), + } +} + +fn optional_qualified_object<'py>( + classes: &ScopeClasses<'py>, + identity: Option<&QualifiedIdentity>, +) -> PyResult>> { + identity + .map(|identity| qualified_object(classes, identity)) + .transpose() +} + +/// An `OwnershipRoot`. +pub(crate) fn root_object<'py>( + classes: &ScopeClasses<'py>, + root: &OwnershipRoot, +) -> PyResult> { + let resource_kind: Option> = root + .resource_kind + .map(|kind| classes.member("ResourceKind", ResourceKind::as_str(kind))) + .transpose()?; + classes.model("OwnershipRoot")?.call1(( + root.path.as_str(), + classes.member("OwnershipRootKind", root.kind.as_str())?, + resource_kind, + )) +} + +fn resource_object<'py>( + classes: &ScopeClasses<'py>, + record: &ResourceRecord, +) -> PyResult> { + classes.model("ResourceRecord")?.call1(( + resource_identity_object(classes, &record.identity)?, + record.path.as_str(), + root_object(classes, &record.ownership_root)?, + )) +} + +fn tuple_of<'py, T>( + py: Python<'py>, + values: &[T], + build: impl Fn(&T) -> PyResult>, +) -> PyResult> { + let built: Vec> = values.iter().map(build).collect::>()?; + PyTuple::new(py, built) +} + +fn macro_object<'py>( + classes: &ScopeClasses<'py>, + facts: &MacroFacts, +) -> PyResult> { + let py: Python<'py> = classes.py(); + classes.model("MacroMetadata")?.call1(( + PyTuple::new(py, &facts.parameters)?, + tuple_of(py, &facts.dependencies, |identity| { + declaration_identity_object(classes, identity) + })?, + facts.source_digest.as_str(), + )) +} + +fn enum_object<'py>(classes: &ScopeClasses<'py>, facts: &EnumFacts) -> PyResult> { + let member_class: Bound<'py, PyAny> = classes.model("EnumMemberMetadata")?; + classes.model("EnumMetadata")?.call1(( + tuple_of(classes.py(), &facts.members, |name| { + member_class.call1((name.as_str(),)) + })?, + facts.scalar_type.as_str(), + )) +} + +fn constant_object<'py>( + classes: &ScopeClasses<'py>, + facts: &ConstantFacts, +) -> PyResult> { + classes.model("ConstantMetadata")?.call1(( + facts.logical_type.as_str(), + facts.collection_kind.as_deref(), + facts.item_count, + facts.nullable, + facts.render_as.as_deref(), + )) +} + +fn declaration_object<'py>( + classes: &ScopeClasses<'py>, + item: &CachedDeclaration, +) -> PyResult> { + let record = &item.record; + let values: [Option>; 3] = [ + item.macro_facts + .as_ref() + .map(|facts| macro_object(classes, facts)) + .transpose()?, + item.enum_facts + .as_ref() + .map(|facts| enum_object(classes, facts)) + .transpose()?, + item.constant_facts + .as_ref() + .map(|facts| constant_object(classes, facts)) + .transpose()?, + ]; + let [macro_metadata, enum_metadata, constant_metadata] = values; + classes.model("DeclarationRecord")?.call1(( + declaration_identity_object(classes, &record.identity)?, + record.path.as_str(), + record.line, + record.column, + classes.member("ScopeKind", record.scope.as_str())?, + root_object(classes, &record.ownership_root)?, + record.owning_path.as_deref(), + macro_metadata, + enum_metadata, + constant_metadata, + )) +} + +fn usage_object<'py>( + classes: &ScopeClasses<'py>, + item: &CachedUsage, +) -> PyResult> { + classes.model("UsageRecord")?.call1(( + qualified_object(classes, &item.consumer)?, + declaration_identity_object(classes, &item.declaration)?, + classes.member("UsageKind", &item.kind)?, + optional_qualified_object(classes, item.through.as_ref())?, + item.enum_member.as_deref(), + )) +} + +fn grant_object<'py>( + classes: &ScopeClasses<'py>, + item: &ReportGrant, +) -> PyResult> { + classes.model("GrantRecord")?.call1(( + resource_identity_object(classes, &item.resource)?, + declaration_identity_object(classes, &item.declaration)?, + qualified_object(classes, &item.through)?, + classes.member("GrantKind", item.kind.as_str())?, + )) +} + +fn visibility_object<'py>( + classes: &ScopeClasses<'py>, + item: &CachedVisibility, +) -> PyResult> { + classes.model("VisibilityRecord")?.call1(( + resource_identity_object(classes, &item.resource)?, + declaration_identity_object(classes, &item.declaration)?, + classes.member("VisibilityReason", &item.reason)?, + optional_qualified_object(classes, item.through.as_ref())?, + )) +} + +fn inaccessible_object<'py>( + classes: &ScopeClasses<'py>, + item: &CachedInaccessible, +) -> PyResult> { + classes.model("InaccessibleRecord")?.call1(( + resource_identity_object(classes, &item.resource)?, + declaration_identity_object(classes, &item.declaration)?, + classes.member("InaccessibleReason", &item.reason)?, + )) +} + +/// A `ScopeDiagnostic`. +pub(crate) fn diagnostic_object<'py>( + classes: &ScopeClasses<'py>, + item: &CachedDiagnostic, +) -> PyResult> { + classes.model("ScopeDiagnostic")?.call1(( + classes.member("ScopeDiagnosticCode", &item.code)?, + item.message.as_str(), + classes.member("DiagnosticSeverity", &item.severity)?, + item.path.as_deref(), + item.line, + item.column, + item.declaration + .as_ref() + .map(|identity| declaration_identity_object(classes, identity)) + .transpose()?, + item.resource + .as_ref() + .map(|identity| resource_identity_object(classes, identity)) + .transpose()?, + )) +} + +/// A `ScopeCompleteness`. +pub(crate) fn completeness_object<'py>( + classes: &ScopeClasses<'py>, + done: [bool; 6], +) -> PyResult> { + classes + .model("ScopeCompleteness")? + .call1((done[0], done[1], done[2], done[3], done[4], done[5])) +} + +/// A Python `ScopeIndex` of a full native index. +pub(crate) fn index_object<'py>( + classes: &ScopeClasses<'py>, + index: &CachedIndex, +) -> PyResult> { + let py: Python<'py> = classes.py(); + classes.model("ScopeIndex")?.call1(( + tuple_of(py, &index.roots, |root| root_object(classes, root))?, + tuple_of(py, &index.resources, |record| { + resource_object(classes, record) + })?, + tuple_of(py, &index.declarations, |item| { + declaration_object(classes, item) + })?, + tuple_of(py, &index.usages, |item| usage_object(classes, item))?, + tuple_of(py, &index.grants, |item| grant_object(classes, item))?, + tuple_of(py, &index.visibility, |item| { + visibility_object(classes, item) + })?, + tuple_of(py, &index.inaccessible, |item| { + inaccessible_object(classes, item) + })?, + tuple_of(py, &index.diagnostics, |item| { + diagnostic_object(classes, item) + })?, + completeness_object(classes, index.completeness)?, + )) +} + +/// A tuple attribute of a scope model. +pub(crate) fn tuple_attribute<'py>( + item: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + Ok(item.getattr(name)?.downcast_into::()?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs new file mode 100644 index 0000000000..4198c9a939 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -0,0 +1,612 @@ +//! Scope report requests read from, and results built as, the Python report models. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, + PyTypeMethods, +}; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_scopes::scope_report::models::{ + BrowseResult, DeclarationReport, DeclarationTarget, Explanation, ListResult, MovePreview, + Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, ScopeChainEntry, + ScopeFolder, ScopeReport, ScopeResult, ScopeSection, SourceLocation, +}; + +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, diagnostic_facts, field, optional_item, read_declaration_identity, + read_diagnostic, read_qualified, +}; + +type PyObject = Py; + +const PATHLIB_MODULE: &str = "pathlib"; + +fn texts(item: &Bound<'_, PyAny>, name: &str) -> PyResult> { + item.getattr(name)? + .try_iter()? + .map(|value| value?.str()?.extract()) + .collect() +} + +/// A query target: an identity model, else its text, read as a path when it is a `PurePath`. +pub(crate) fn read_target( + classes: &ScopeClasses<'_>, + target: &Bound<'_, PyAny>, +) -> PyResult { + if target.is_instance(&classes.model("ResourceIdentity")?)? + || target.is_instance(&classes.model("DeclarationIdentity")?)? + { + return Ok(QueryTarget::Identity(read_qualified(target)?)); + } + let text: String = target.str()?.extract()?; + let pure_path: Bound<'_, PyAny> = target.py().import(PATHLIB_MODULE)?.getattr("PurePath")?; + Ok(if target.is_instance(&pure_path)? { + QueryTarget::Path(text) + } else { + QueryTarget::Text(text) + }) +} + +/// An optional query target. +pub(crate) fn read_optional_target( + classes: &ScopeClasses<'_>, + target: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + target + .filter(|target| !target.is_none()) + .map(|target| read_target(classes, target)) + .transpose() +} + +/// A declaration to explain: a `DeclarationIdentity`, else its text. +pub(crate) fn read_declaration_target( + classes: &ScopeClasses<'_>, + declaration: &Bound<'_, PyAny>, +) -> PyResult { + if declaration.is_instance(&classes.model("DeclarationIdentity")?)? { + return Ok(DeclarationTarget::Identity(read_declaration_identity( + declaration, + )?)); + } + Ok(DeclarationTarget::Text(declaration.str()?.extract()?)) +} + +/// `ScopeReportFilters`, with each kind as its text. +pub(crate) fn read_filters(filters: &Bound<'_, PyAny>) -> PyResult { + Ok(ReportFilters { + sections: field(filters, "sections")?, + include_nearby: field(filters, "include_nearby")?, + defined_under: field(filters, "defined_under")?, + kinds: texts(filters, "kinds")?, + glob: field(filters, "glob")?, + used_only: field(filters, "used_only")?, + dependency_depth: field(filters, "dependency_depth")?, + sort: field(filters, "sort")?, + cursor: field(filters, "cursor")?, + page_size: field(filters, "page_size")?, + nearby_depth: field(filters, "nearby_depth")?, + globals: field(filters, "globals")?, + }) +} + +/// A JSON value as Python builds it; arrays become tuples or lists. +pub(crate) fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { + match value { + JsonValue::Null => Ok(py.None()), + JsonValue::Bool(flag) => flag.into_py_any(py), + JsonValue::Integer(integer) => match integer.to_i128() { + Some(number) => number.into_py_any(py), + None => Ok(py + .import("builtins")? + .getattr("int")? + .call1((integer.as_str(),))? + .unbind()), + }, + JsonValue::Float(number) => number.into_py_any(py), + JsonValue::String(text) => text.into_py_any(py), + JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), + JsonValue::Array(items) => { + let values: Vec = items + .into_iter() + .map(|item| json_to_python(py, item, tuples)) + .collect::>()?; + if tuples { + PyTuple::new(py, values)?.into_py_any(py) + } else { + PyList::new(py, values)?.into_py_any(py) + } + } + JsonValue::Object(entries) => { + let dictionary: Bound<'_, PyDict> = PyDict::new(py); + for (key, item) in entries { + dictionary.set_item(key, json_to_python(py, item, tuples)?)?; + } + dictionary.into_py_any(py) + } + } +} + +/// `_value` of one metadata value; an unsupported type is the error Python's serializer raises. +pub(crate) fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + let py: Python<'_> = value.py(); + if py + .import("enum")? + .getattr("Enum") + .and_then(|enum_type| value.is_instance(&enum_type))? + { + return python_to_json(&value.getattr("value")?); + } + if value.is_instance_of::() { + let decimal: String = value.str()?.extract()?; + return JsonInteger::parse(&decimal) + .map(JsonValue::Integer) + .ok_or_else(|| PyValueError::new_err("Unsupported scope report value: int")); + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract::()?)); + } + if value.is_instance_of::() { + return Ok(JsonValue::String(value.extract()?)); + } + let pure_path = py.import("pathlib")?.getattr("PurePath")?; + if value.is_instance(&pure_path)? { + return Ok(JsonValue::String( + value.call_method0("as_posix")?.extract()?, + )); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(python_to_json(&item?)?); + } + return Ok(JsonValue::Array(items)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + entries.push((key.str()?.extract()?, python_to_json(&item)?)); + } + return Ok(JsonValue::Object(entries)); + } + let name: String = value.get_type().name()?.extract()?; + Err(PyValueError::new_err(format!( + "Unsupported scope report value: {name}" + ))) +} + +/// `{str(key): value}` with later duplicate keys replacing earlier values in place. +fn unique_entries(entries: Vec<(String, JsonValue)>) -> Vec<(String, JsonValue)> { + let mut unique: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); + for (key, value) in entries { + match unique.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => unique.push((key, value)), + } + } + unique +} + +fn tuple_of<'py, T>( + py: Python<'py>, + values: Vec, + build: impl Fn(T) -> PyResult>, +) -> PyResult> { + let built: Vec> = values.into_iter().map(build).collect::>()?; + Ok(PyTuple::new(py, built)?.into_any()) +} + +fn declaration_object<'py>( + classes: &ScopeClasses<'py>, + report: DeclarationReport, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let metadata: Bound<'py, PyAny> = tuple_of(py, report.metadata, |(key, value)| { + Ok(PyTuple::new(py, [key.into_py_any(py)?, json_to_python(py, value, true)?])?.into_any()) + })?; + let visibility: Option> = report + .visibility + .map(|provenance| { + classes + .model("VisibilityProvenance")? + .call1((provenance.reason, provenance.through)) + }) + .transpose()?; + let head: [Bound<'py, PyAny>; 7] = [ + report.identity.into_bound_py_any(py)?, + report.kind.into_bound_py_any(py)?, + report.name.into_bound_py_any(py)?, + report.owner.into_bound_py_any(py)?, + classes.model("SourceLocation")?.call1(( + report.definition.path, + report.definition.line, + report.definition.column, + ))?, + report.scope.into_bound_py_any(py)?, + report.owning_path.into_bound_py_any(py)?, + ]; + let standing: [Bound<'py, PyAny>; 3] = [ + visibility.into_bound_py_any(py)?, + report.inaccessible_reason.into_bound_py_any(py)?, + metadata, + ]; + let links: [Bound<'py, PyAny>; 6] = [ + PyTuple::new(py, report.consumers)?.into_any(), + PyTuple::new(py, report.dependencies)?.into_any(), + PyTuple::new(py, report.grants)?.into_any(), + report.required_scope.into_bound_py_any(py)?, + report.required_path.into_bound_py_any(py)?, + PyTuple::new(py, report.promotion_impact)?.into_any(), + ]; + let arguments: Vec> = head.into_iter().chain(standing).chain(links).collect(); + classes + .model("DeclarationReport")? + .call1(PyTuple::new(py, arguments)?) +} + +fn declarations_object<'py>( + classes: &ScopeClasses<'py>, + reports: Vec, +) -> PyResult> { + tuple_of(classes.py(), reports, |report| { + declaration_object(classes, report) + }) +} + +fn section_object<'py>( + classes: &ScopeClasses<'py>, + section: ScopeSection, +) -> PyResult> { + classes.model("ScopeSection")?.call1(( + section.name, + section.total, + section.returned, + section.collapsed, + section.collapsed_count, + section.truncated, + section.complete, + section.next_cursor, + section.cursor, + section.page_size, + )) +} + +/// The report's diagnostics: an index diagnostic of `index_diagnostics`, else a raised one. +pub(crate) fn diagnostics_object<'py>( + classes: &ScopeClasses<'py>, + index_diagnostics: &Bound<'py, PyAny>, + diagnostics: Vec, +) -> PyResult> { + let py: Python<'py> = classes.py(); + tuple_of(py, diagnostics, |diagnostic| match diagnostic { + ReportDiagnostic::Index(position) => index_diagnostics.get_item(position), + ReportDiagnostic::Raised(facts) => { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("path", facts.path)?; + classes.model("ScopeDiagnostic")?.call( + ( + classes.member("ScopeDiagnosticCode", &facts.code)?, + facts.message, + ), + Some(&keywords), + ) + } + }) +} + +/// A `DeclarationExplanation`. +pub(crate) fn explanation_object<'py>( + classes: &ScopeClasses<'py>, + explanation: Explanation, +) -> PyResult> { + let declaration: Option> = explanation + .declaration + .map(|report| declaration_object(classes, report)) + .transpose()?; + classes + .model("DeclarationExplanation")? + .call1((declaration, explanation.complete)) +} + +/// A `MovePreview`. +pub(crate) fn move_object<'py>( + classes: &ScopeClasses<'py>, + preview: MovePreview, +) -> PyResult> { + classes.model("MovePreview")?.call1(( + preview.resource, + preview.destination, + preview.new_ownership_root, + declarations_object(classes, preview.retained)?, + declarations_object(classes, preview.gained)?, + declarations_object(classes, preview.lost)?, + PyTuple::new(classes.py(), preview.invalidated_usages)?, + declarations_object(classes, preview.private_retained)?, + declarations_object(classes, preview.relationship_retained)?, + preview.complete, + )) +} + +/// A `ScopeReport` carrying the caller's `filters` object. +pub(crate) fn report_object<'py>( + classes: &ScopeClasses<'py>, + report: ScopeReport, + context: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let (filters, index_diagnostics) = context; + let py: Python<'py> = classes.py(); + let resource: ResourceReport = report.resource; + let chain_class: Bound<'py, PyAny> = classes.model("ScopeChainEntry")?; + let explanation: Option> = report + .explanation + .map(|explanation| explanation_object(classes, explanation)) + .transpose()?; + let move_preview: Option> = report + .move_preview + .map(|preview| move_object(classes, preview)) + .transpose()?; + classes.model("ScopeReport")?.call1(( + classes.model("ScopeResourceReport")?.call1(( + resource.target, + resource.identity, + resource.path, + resource.prospective, + resource.directory, + resource.duplicate_count, + ))?, + tuple_of(py, report.scope_chain, |entry| { + chain_class.call1((entry.kind, entry.path, entry.declaration_count)) + })?, + declarations_object(classes, report.available)?, + declarations_object(classes, report.used)?, + declarations_object(classes, report.relationship_scope)?, + declarations_object(classes, report.nearby_unavailable)?, + filters, + tuple_of(py, report.sections, |section| { + section_object(classes, section) + })?, + explanation, + move_preview, + diagnostics_object(classes, index_diagnostics, report.diagnostics)?, + report.complete, + )) +} + +/// A `ScopeBrowseResult`. +pub(crate) fn browse_object<'py>( + classes: &ScopeClasses<'py>, + result: BrowseResult, + index_diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let folder_class: Bound<'py, PyAny> = classes.model("ScopeFolder")?; + classes.model("ScopeBrowseResult")?.call1(( + result.folder, + tuple_of(py, result.folders, |folder| { + folder_class.call1(( + folder.path, + folder.name, + folder.descendant_count, + folder.child_count, + folder.used_count, + PyTuple::new(py, folder.kind_counts)?, + )) + })?, + diagnostics_object(classes, index_diagnostics, result.diagnostics)?, + result.complete, + )) +} + +/// A `ScopeListResult`. +pub(crate) fn list_object<'py>( + classes: &ScopeClasses<'py>, + result: ListResult, + index_diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + classes.model("ScopeListResult")?.call1(( + result.folder, + declarations_object(classes, result.declarations)?, + section_object(classes, result.section)?, + diagnostics_object(classes, index_diagnostics, result.diagnostics)?, + )) +} + +fn read_declaration_report(report: &Bound<'_, PyAny>) -> PyResult { + let definition: Bound<'_, PyAny> = report.getattr("definition")?; + let mut metadata: Vec<(String, JsonValue)> = Vec::new(); + for entry in report.getattr("metadata")?.try_iter()? { + let (key, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = entry?.extract()?; + metadata.push((key.str()?.extract()?, python_to_json(&value)?)); + } + Ok(DeclarationReport { + identity: field(report, "identity")?, + kind: field(report, "kind")?, + name: field(report, "name")?, + owner: field(report, "owner")?, + definition: SourceLocation { + path: field(&definition, "path")?, + line: field(&definition, "line")?, + column: field(&definition, "column")?, + }, + scope: field(report, "scope")?, + owning_path: field(report, "owning_path")?, + visibility: optional_item(report, "visibility")? + .map(|provenance| -> PyResult { + Ok(Provenance { + reason: field(&provenance, "reason")?, + through: field(&provenance, "through")?, + }) + }) + .transpose()?, + inaccessible_reason: field(report, "inaccessible_reason")?, + metadata: unique_entries(metadata), + consumers: field(report, "consumers")?, + dependencies: field(report, "dependencies")?, + grants: field(report, "grants")?, + required_scope: field(report, "required_scope")?, + required_path: field(report, "required_path")?, + promotion_impact: field(report, "promotion_impact")?, + }) +} + +fn read_declaration_reports( + item: &Bound<'_, PyAny>, + name: &str, +) -> PyResult> { + item.getattr(name)? + .try_iter()? + .map(|report| read_declaration_report(&report?)) + .collect() +} + +fn read_section(section: &Bound<'_, PyAny>) -> PyResult { + Ok(ScopeSection { + name: field(section, "name")?, + total: field(section, "total")?, + returned: field(section, "returned")?, + collapsed: field(section, "collapsed")?, + collapsed_count: field(section, "collapsed_count")?, + truncated: field(section, "truncated")?, + complete: field(section, "complete")?, + next_cursor: field(section, "next_cursor")?, + cursor: field(section, "cursor")?, + page_size: field(section, "page_size")?, + }) +} + +fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { + let mut kind_counts: Vec<(String, i64)> = Vec::new(); + for entry in folder.getattr("kind_counts")?.try_iter()? { + let (kind, count): (Bound<'_, PyAny>, i64) = entry?.extract()?; + let kind: String = kind.str()?.extract()?; + match kind_counts + .iter_mut() + .find(|(existing, _)| *existing == kind) + { + Some(entry) => entry.1 = count, + None => kind_counts.push((kind, count)), + } + } + Ok(ScopeFolder { + path: field(folder, "path")?, + name: field(folder, "name")?, + descendant_count: field(folder, "descendant_count")?, + child_count: field(folder, "child_count")?, + used_count: field(folder, "used_count")?, + kind_counts, + }) +} + +fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> { + item.getattr("diagnostics")? + .try_iter()? + .map(|diagnostic| { + Ok(ReportDiagnostic::Raised(diagnostic_facts(read_diagnostic( + &diagnostic?, + )?))) + }) + .collect() +} + +fn read_explanation(explanation: &Bound<'_, PyAny>) -> PyResult { + Ok(Explanation { + declaration: optional_item(explanation, "declaration")? + .map(|report| read_declaration_report(&report)) + .transpose()?, + complete: field(explanation, "complete")?, + }) +} + +fn read_move(preview: &Bound<'_, PyAny>) -> PyResult { + Ok(MovePreview { + resource: field(preview, "resource")?, + destination: field(preview, "destination")?, + new_ownership_root: field(preview, "new_ownership_root")?, + retained: read_declaration_reports(preview, "retained")?, + gained: read_declaration_reports(preview, "gained")?, + lost: read_declaration_reports(preview, "lost")?, + invalidated_usages: field(preview, "invalidated_usages")?, + private_retained: read_declaration_reports(preview, "private_retained")?, + relationship_retained: read_declaration_reports(preview, "relationship_retained")?, + complete: field(preview, "complete")?, + }) +} + +fn read_report(report: &Bound<'_, PyAny>) -> PyResult { + let resource: Bound<'_, PyAny> = report.getattr("resource")?; + Ok(ScopeReport { + resource: ResourceReport { + target: field(&resource, "target")?, + identity: field(&resource, "identity")?, + path: field(&resource, "path")?, + prospective: field(&resource, "prospective")?, + directory: field(&resource, "directory")?, + duplicate_count: field(&resource, "duplicate_count")?, + }, + scope_chain: report + .getattr("scope_chain")? + .try_iter()? + .map(|entry| { + let entry: Bound<'_, PyAny> = entry?; + Ok(ScopeChainEntry { + kind: field(&entry, "kind")?, + path: field(&entry, "path")?, + declaration_count: field(&entry, "declaration_count")?, + }) + }) + .collect::>()?, + available: read_declaration_reports(report, "available")?, + used: read_declaration_reports(report, "used")?, + relationship_scope: read_declaration_reports(report, "relationship_scope")?, + nearby_unavailable: read_declaration_reports(report, "nearby_unavailable")?, + filters: read_filters(&report.getattr("filters")?)?, + sections: report + .getattr("sections")? + .try_iter()? + .map(|section| read_section(§ion?)) + .collect::>()?, + explanation: optional_item(report, "explanation")? + .map(|explanation| read_explanation(&explanation)) + .transpose()?, + move_preview: optional_item(report, "move_preview")? + .map(|preview| read_move(&preview)) + .transpose()?, + diagnostics: read_diagnostics(report)?, + complete: field(report, "complete")?, + }) +} + +/// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. +pub(crate) fn read_result( + classes: &ScopeClasses<'_>, + result: &Bound<'_, PyAny>, +) -> PyResult { + if result.is_instance(&classes.model("ScopeReport")?)? { + return Ok(ScopeResult::Report(Box::new(read_report(result)?))); + } + if result.is_instance(&classes.model("ScopeBrowseResult")?)? { + return Ok(ScopeResult::Browse(BrowseResult { + folder: field(result, "folder")?, + folders: result + .getattr("folders")? + .try_iter()? + .map(|folder| read_folder(&folder?)) + .collect::>()?, + diagnostics: read_diagnostics(result)?, + complete: field(result, "complete")?, + })); + } + Ok(ScopeResult::List(ListResult { + folder: field(result, "folder")?, + declarations: read_declaration_reports(result, "declarations")?, + section: read_section(&result.getattr("section")?)?, + diagnostics: read_diagnostics(result)?, + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs deleted file mode 100644 index 584e61190a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs +++ /dev/null @@ -1,741 +0,0 @@ -//! Rows of scope report requests and results, in both directions across the boundary. - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{ - PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, - PyTypeMethods, -}; -use sqlbuild_core::json::models::{JsonInteger, JsonValue}; -use sqlbuild_scopes::scope_index::main::format_identity::format_identity; -use sqlbuild_scopes::scope_index::models::QualifiedIdentity; -use sqlbuild_scopes::scope_report::models::{ - BrowseResult, Completeness, ConstantValues, DeclarationReport, DeclarationTarget, - DeclarationValues, DiagnosticFacts, EnumValues, Explanation, ListResult, MacroValues, - MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, - ScopeChainEntry, ScopeFolder, ScopeReport, ScopeSection, SourceLocation, -}; - -use crate::bindings::_helpers::scopes::placement::{ - IdentityRow, QualifiedRow, declaration_identity, qualified_identity, resource_identity, -}; - -type PyObject = Py; - -/// A query target: an identity row, else text, and whether the text came from a `PurePath`. -pub(crate) type TargetRow = (Option, String, bool); -/// Every `ScopeReportFilters` field in order. -pub(crate) type FiltersRow = ( - Vec, - bool, - Option, - Vec, - Option, - bool, - i64, - String, - Option, - i64, - i64, - String, -); -/// Parameters, dependencies and source digest. -type MacroRow = (Vec, Vec, String); -/// Member names and scalar type. -type EnumRow = (Vec, String); -/// Logical type, collection kind, item count, nullable and render-as. -type ConstantRow = (String, Option, Option, bool, Option); -pub(crate) type ValuesRow = (Option, Option, Option); -pub(crate) type CompletenessRow = (bool, bool, bool, bool, bool, bool); -/// Code, message, severity, path, line, column, declaration and resource identities. -pub(crate) type DiagnosticInRow = ( - String, - String, - String, - Option, - Option, - Option, - Option, - Option, -); -/// An index diagnostic's canonical position, else a raised `(code, message, path)`. -type DiagnosticOutRow = (Option, Option<(String, String, Option)>); -/// Identity, kind, name, owner, definition, scope and owning path. -type HeadRow = ( - String, - String, - String, - Option, - (String, i64, i64), - String, - Option, -); -/// Visibility, inaccessible reason and metadata entries. -type StandingRow = ( - Option<(String, Option)>, - Option, - Vec<(String, M)>, -); -/// Consumers, dependencies, grants, required scope, required path and promotion impact. -type LinksRow = ( - Vec, - Vec, - Vec, - Option, - Option, - Vec, -); -pub(crate) type DeclarationRow = (HeadRow, StandingRow, LinksRow); -type SectionRow = ( - String, - i64, - i64, - bool, - i64, - bool, - bool, - Option, - Option, - i64, -); -type ResourceReportRow = (String, Option, Option, bool, bool, i64); -type ChainRow = (String, String, i64); -type ExplanationRow = (Option>, bool); -/// Every `MovePreview` field in order. -type MoveRow = ( - String, - String, - Option, - Vec>, - Vec>, - Vec>, - Vec, - Vec>, - Vec>, - bool, -); -type FolderRow = (String, String, i64, i64, i64, Vec<(String, i64)>); -/// Every `ScopeReport` field in order. -pub(crate) type ReportRow = ( - ResourceReportRow, - Vec, - Vec>, - Vec>, - Vec>, - Vec>, - FiltersRow, - Vec, - Option>, - Option>, - Vec, - bool, -); -pub(crate) type BrowseRow = (String, Vec, Vec, bool); -pub(crate) type ListRow = (String, Vec>, SectionRow, Vec); -pub(crate) type ExplanationOutRow = (ExplanationRow, Vec); -pub(crate) type MoveOutRow = (Option>, Vec); -pub(crate) type ReportOutRow = ReportRow; -pub(crate) type BrowseOutRow = BrowseRow; -pub(crate) type ListOutRow = ListRow; - -pub(crate) fn target(row: TargetRow) -> PyResult { - let (identity, text, path) = row; - Ok(match (identity, path) { - (Some(identity), _) => QueryTarget::Identity(qualified_identity(identity)?), - (None, true) => QueryTarget::Path(text), - (None, false) => QueryTarget::Text(text), - }) -} - -pub(crate) fn declaration_target( - (identity, text): (Option, String), -) -> PyResult { - Ok(match identity { - Some(identity) => DeclarationTarget::Identity(declaration_identity(identity)?), - None => DeclarationTarget::Text(text), - }) -} - -pub(crate) fn filters(row: FiltersRow) -> ReportFilters { - let ( - sections, - include_nearby, - defined_under, - kinds, - glob, - used_only, - dependency_depth, - sort, - cursor, - page_size, - nearby_depth, - globals, - ) = row; - ReportFilters { - sections, - include_nearby, - defined_under, - kinds, - glob, - used_only, - dependency_depth, - sort, - cursor, - page_size, - nearby_depth, - globals, - } -} - -fn filters_row(filters: &ReportFilters) -> FiltersRow { - ( - filters.sections.clone(), - filters.include_nearby, - filters.defined_under.clone(), - filters.kinds.clone(), - filters.glob.clone(), - filters.used_only, - filters.dependency_depth, - filters.sort.clone(), - filters.cursor.clone(), - filters.page_size, - filters.nearby_depth, - filters.globals.clone(), - ) -} - -pub(crate) fn declaration_values(row: ValuesRow) -> PyResult { - let (macro_row, enum_row, constant_row) = row; - Ok(DeclarationValues { - macro_values: match macro_row { - Some((parameters, dependencies, source_digest)) => Some(MacroValues { - parameters, - dependencies: dependencies - .into_iter() - .map(declaration_identity) - .collect::>()?, - source_digest, - }), - None => None, - }, - enum_values: enum_row.map(|(members, scalar_type)| EnumValues { - members, - scalar_type, - }), - constant_values: constant_row.map( - |(logical_type, collection_kind, item_count, nullable, render_as)| ConstantValues { - logical_type, - collection_kind, - item_count, - nullable, - render_as, - }, - ), - }) -} - -pub(crate) fn completeness(row: CompletenessRow) -> Completeness { - let (discovery, static_visibility, runtime_usage, relationships, placement, promotion_impact) = - row; - Completeness { - discovery, - static_visibility, - runtime_usage, - relationships, - placement, - promotion_impact, - } -} - -/// A diagnostic row's facts, with its identities as qualified text. -pub(crate) fn diagnostic_facts(row: DiagnosticInRow) -> PyResult { - let (code, message, severity, path, line, column, declaration, resource) = row; - Ok(DiagnosticFacts { - code, - message, - severity, - path, - line, - column, - declaration: match declaration { - Some(row) => Some(format_identity(&QualifiedIdentity::Declaration( - declaration_identity(row)?, - ))), - None => None, - }, - resource: match resource { - Some(row) => Some(format_identity(&QualifiedIdentity::Resource( - resource_identity(row)?, - ))), - None => None, - }, - }) -} - -fn diagnostic_out(diagnostic: ReportDiagnostic) -> DiagnosticOutRow { - match diagnostic { - ReportDiagnostic::Index(position) => (Some(position), None), - ReportDiagnostic::Raised(facts) => (None, Some((facts.code, facts.message, facts.path))), - } -} - -pub(crate) fn raised_out(diagnostics: Vec) -> Vec { - diagnostics - .into_iter() - .map(|facts| diagnostic_out(ReportDiagnostic::Raised(facts))) - .collect() -} - -/// A JSON value as Python builds it; arrays become tuples or lists. -pub(crate) fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { - match value { - JsonValue::Null => Ok(py.None()), - JsonValue::Bool(flag) => flag.into_py_any(py), - JsonValue::Integer(integer) => match integer.to_i128() { - Some(number) => number.into_py_any(py), - None => Ok(py - .import("builtins")? - .getattr("int")? - .call1((integer.as_str(),))? - .unbind()), - }, - JsonValue::Float(number) => number.into_py_any(py), - JsonValue::String(text) => text.into_py_any(py), - JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), - JsonValue::Array(items) => { - let values: Vec = items - .into_iter() - .map(|item| json_to_python(py, item, tuples)) - .collect::>()?; - if tuples { - PyTuple::new(py, values)?.into_py_any(py) - } else { - PyList::new(py, values)?.into_py_any(py) - } - } - JsonValue::Object(entries) => { - let dictionary: Bound<'_, PyDict> = PyDict::new(py); - for (key, item) in entries { - dictionary.set_item(key, json_to_python(py, item, tuples)?)?; - } - dictionary.into_py_any(py) - } - } -} - -/// `_value` of one metadata value; an unsupported type is the error Python's serializer raises. -pub(crate) fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_none() { - return Ok(JsonValue::Null); - } - if let Ok(flag) = value.downcast::() { - return Ok(JsonValue::Bool(flag.is_true())); - } - let py: Python<'_> = value.py(); - if py - .import("enum")? - .getattr("Enum") - .and_then(|enum_type| value.is_instance(&enum_type))? - { - return python_to_json(&value.getattr("value")?); - } - if value.is_instance_of::() { - let decimal: String = value.str()?.extract()?; - return JsonInteger::parse(&decimal) - .map(JsonValue::Integer) - .ok_or_else(|| PyValueError::new_err("Unsupported scope report value: int")); - } - if value.is_instance_of::() { - return Ok(JsonValue::Float(value.extract::()?)); - } - if value.is_instance_of::() { - return Ok(JsonValue::String(value.extract()?)); - } - let pure_path = py.import("pathlib")?.getattr("PurePath")?; - if value.is_instance(&pure_path)? { - return Ok(JsonValue::String( - value.call_method0("as_posix")?.extract()?, - )); - } - if value.is_instance_of::() || value.is_instance_of::() { - let mut items: Vec = Vec::new(); - for item in value.try_iter()? { - items.push(python_to_json(&item?)?); - } - return Ok(JsonValue::Array(items)); - } - if let Ok(mapping) = value.downcast::() { - let mut entries: Vec<(String, JsonValue)> = Vec::new(); - for (key, item) in mapping.iter() { - entries.push((key.str()?.extract()?, python_to_json(&item)?)); - } - return Ok(JsonValue::Object(entries)); - } - let name: String = value.get_type().name()?.extract()?; - Err(PyValueError::new_err(format!( - "Unsupported scope report value: {name}" - ))) -} - -/// `{str(key): value}` with later duplicate keys replacing earlier values in place. -fn unique_entries(entries: Vec<(String, JsonValue)>) -> Vec<(String, JsonValue)> { - let mut unique: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); - for (key, value) in entries { - match unique.iter_mut().find(|(existing, _)| *existing == key) { - Some(entry) => entry.1 = value, - None => unique.push((key, value)), - } - } - unique -} - -pub(crate) fn declaration_out( - py: Python<'_>, - report: DeclarationReport, -) -> PyResult> { - let metadata: Vec<(String, PyObject)> = report - .metadata - .into_iter() - .map(|(key, value)| Ok((key, json_to_python(py, value, true)?))) - .collect::>()?; - Ok(( - ( - report.identity, - report.kind, - report.name, - report.owner, - ( - report.definition.path, - report.definition.line, - report.definition.column, - ), - report.scope, - report.owning_path, - ), - ( - report - .visibility - .map(|provenance| (provenance.reason, provenance.through)), - report.inaccessible_reason, - metadata, - ), - ( - report.consumers, - report.dependencies, - report.grants, - report.required_scope, - report.required_path, - report.promotion_impact, - ), - )) -} - -pub(crate) fn declaration_in(row: DeclarationRow>) -> PyResult { - let ( - (identity, kind, name, owner, (path, line, column), scope, owning_path), - (visibility, inaccessible_reason, metadata), - (consumers, dependencies, grants, required_scope, required_path, promotion_impact), - ) = row; - let metadata: Vec<(String, JsonValue)> = metadata - .into_iter() - .map(|(key, value)| Ok((key, python_to_json(&value)?))) - .collect::>()?; - Ok(DeclarationReport { - identity, - kind, - name, - owner, - definition: SourceLocation { path, line, column }, - scope, - owning_path, - visibility: visibility.map(|(reason, through)| Provenance { reason, through }), - inaccessible_reason, - metadata: unique_entries(metadata), - consumers, - dependencies, - grants, - required_scope, - required_path, - promotion_impact, - }) -} - -fn declarations_out( - py: Python<'_>, - reports: Vec, -) -> PyResult>> { - reports - .into_iter() - .map(|report| declaration_out(py, report)) - .collect() -} - -fn declarations_in( - rows: Vec>>, -) -> PyResult> { - rows.into_iter().map(declaration_in).collect() -} - -fn section_row(section: ScopeSection) -> SectionRow { - ( - section.name, - section.total, - section.returned, - section.collapsed, - section.collapsed_count, - section.truncated, - section.complete, - section.next_cursor, - section.cursor, - section.page_size, - ) -} - -fn section_in(row: SectionRow) -> ScopeSection { - let ( - name, - total, - returned, - collapsed, - collapsed_count, - truncated, - complete, - next_cursor, - cursor, - page_size, - ) = row; - ScopeSection { - name, - total, - returned, - collapsed, - collapsed_count, - truncated, - complete, - next_cursor, - cursor, - page_size, - } -} - -fn folder_row(folder: ScopeFolder) -> FolderRow { - ( - folder.path, - folder.name, - folder.descendant_count, - folder.child_count, - folder.used_count, - folder.kind_counts, - ) -} - -fn folder_in(row: FolderRow) -> ScopeFolder { - let (path, name, descendant_count, child_count, used_count, kind_counts) = row; - let mut unique: Vec<(String, i64)> = Vec::with_capacity(kind_counts.len()); - for (kind, count) in kind_counts { - match unique.iter_mut().find(|(existing, _)| *existing == kind) { - Some(entry) => entry.1 = count, - None => unique.push((kind, count)), - } - } - ScopeFolder { - path, - name, - descendant_count, - child_count, - used_count, - kind_counts: unique, - } -} - -pub(crate) fn explanation_out( - py: Python<'_>, - explanation: Explanation, -) -> PyResult> { - Ok(( - explanation - .declaration - .map(|report| declaration_out(py, report)) - .transpose()?, - explanation.complete, - )) -} - -fn explanation_in(row: ExplanationRow>) -> PyResult { - Ok(Explanation { - declaration: row.0.map(declaration_in).transpose()?, - complete: row.1, - }) -} - -pub(crate) fn move_out(py: Python<'_>, preview: MovePreview) -> PyResult> { - Ok(( - preview.resource, - preview.destination, - preview.new_ownership_root, - declarations_out(py, preview.retained)?, - declarations_out(py, preview.gained)?, - declarations_out(py, preview.lost)?, - preview.invalidated_usages, - declarations_out(py, preview.private_retained)?, - declarations_out(py, preview.relationship_retained)?, - preview.complete, - )) -} - -fn move_in(row: MoveRow>) -> PyResult { - let ( - resource, - destination, - new_ownership_root, - retained, - gained, - lost, - invalidated_usages, - private_retained, - relationship_retained, - complete, - ) = row; - Ok(MovePreview { - resource, - destination, - new_ownership_root, - retained: declarations_in(retained)?, - gained: declarations_in(gained)?, - lost: declarations_in(lost)?, - invalidated_usages, - private_retained: declarations_in(private_retained)?, - relationship_retained: declarations_in(relationship_retained)?, - complete, - }) -} - -fn resource_row(resource: ResourceReport) -> ResourceReportRow { - ( - resource.target, - resource.identity, - resource.path, - resource.prospective, - resource.directory, - resource.duplicate_count, - ) -} - -fn chain_row(entry: ScopeChainEntry) -> ChainRow { - (entry.kind, entry.path, entry.declaration_count) -} - -pub(crate) fn report_out(py: Python<'_>, report: ScopeReport) -> PyResult { - Ok(( - resource_row(report.resource), - report.scope_chain.into_iter().map(chain_row).collect(), - declarations_out(py, report.available)?, - declarations_out(py, report.used)?, - declarations_out(py, report.relationship_scope)?, - declarations_out(py, report.nearby_unavailable)?, - filters_row(&report.filters), - report.sections.into_iter().map(section_row).collect(), - report - .explanation - .map(|explanation| explanation_out(py, explanation)) - .transpose()?, - report - .move_preview - .map(|preview| move_out(py, preview)) - .transpose()?, - report.diagnostics.into_iter().map(diagnostic_out).collect(), - report.complete, - )) -} - -pub(crate) fn report_in( - row: ReportRow, DiagnosticInRow>, -) -> PyResult { - let ( - (target, identity, path, prospective, directory, duplicate_count), - chain, - available, - used, - relationship_scope, - nearby_unavailable, - filter_row, - sections, - explanation, - move_preview, - diagnostics, - complete, - ) = row; - Ok(ScopeReport { - resource: ResourceReport { - target, - identity, - path, - prospective, - directory, - duplicate_count, - }, - scope_chain: chain - .into_iter() - .map(|(kind, path, declaration_count)| ScopeChainEntry { - kind, - path, - declaration_count, - }) - .collect(), - available: declarations_in(available)?, - used: declarations_in(used)?, - relationship_scope: declarations_in(relationship_scope)?, - nearby_unavailable: declarations_in(nearby_unavailable)?, - filters: filters(filter_row), - sections: sections.into_iter().map(section_in).collect(), - explanation: explanation.map(explanation_in).transpose()?, - move_preview: move_preview.map(move_in).transpose()?, - diagnostics: diagnostics_in(diagnostics)?, - complete, - }) -} - -fn diagnostics_in(rows: Vec) -> PyResult> { - rows.into_iter() - .map(|row| Ok(ReportDiagnostic::Raised(diagnostic_facts(row)?))) - .collect() -} - -pub(crate) fn browse_out(result: BrowseResult) -> BrowseOutRow { - ( - result.folder, - result.folders.into_iter().map(folder_row).collect(), - result.diagnostics.into_iter().map(diagnostic_out).collect(), - result.complete, - ) -} - -pub(crate) fn browse_in(row: BrowseRow) -> PyResult { - let (folder, folders, diagnostics, complete) = row; - Ok(BrowseResult { - folder, - folders: folders.into_iter().map(folder_in).collect(), - diagnostics: diagnostics_in(diagnostics)?, - complete, - }) -} - -pub(crate) fn list_out(py: Python<'_>, result: ListResult) -> PyResult { - Ok(( - result.folder, - declarations_out(py, result.declarations)?, - section_row(result.section), - result.diagnostics.into_iter().map(diagnostic_out).collect(), - )) -} - -pub(crate) fn list_in(row: ListRow, DiagnosticInRow>) -> PyResult { - let (folder, declarations, section, diagnostics) = row; - Ok(ListResult { - folder, - declarations: declarations_in(declarations)?, - section: section_in(section), - diagnostics: diagnostics_in(diagnostics)?, - }) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 30b5531a80..211ba545f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -1,8 +1,10 @@ -//! `sqb scope` reports over a native scope lookup, their serialization and the metadata projection. +//! `sqb scope` reports over a `ScopeLookup`, their serialization and the metadata projection. -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::exceptions::{PyIndexError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; @@ -12,85 +14,57 @@ use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; use sqlbuild_scopes::scope_report::main::scope_metadata_projection::scope_metadata_projection; use sqlbuild_scopes::scope_report::main::serialize_scope_result::serialize_scope_result; use sqlbuild_scopes::scope_report::models::{ - ExplainRequest, FolderRequest, MoveRequest, ProjectionGrant, ProjectionIndex, - ProjectionStanding, ProjectionUsage, ReportFailure, ReportRequest, ReportSource, ReportValues, + DeclarationValues, DiagnosticFacts, ExplainRequest, FolderRequest, MoveRequest, + ProjectionIndex, ReportDiagnostic, ReportFailure, ReportRequest, ReportSource, ReportValues, ScopeResult, }; -use crate::bindings::_helpers::scopes::placement::{ - DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, - declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, -}; use crate::bindings::_helpers::scopes::scope_lookup::{ - NativeScopeLookup, grant_kind, identity_message, path_message, + NativeScopeLookup, identity_message, path_message, +}; +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, completeness, declaration_values, projection_index, read_completeness, + read_declaration, read_index, }; -use crate::bindings::_helpers::scopes::scope_report_rows::{ - BrowseOutRow, BrowseRow, CompletenessRow, DiagnosticInRow, ExplanationOutRow, FiltersRow, - ListOutRow, ListRow, MoveOutRow, ReportOutRow, ReportRow, TargetRow, ValuesRow, browse_in, - browse_out, completeness, declaration_target, declaration_values, diagnostic_facts, - explanation_out, filters, json_to_python, list_in, list_out, move_out, raised_out, report_in, - report_out, target, +use crate::bindings::_helpers::scopes::scope_report_objects::{ + browse_object, diagnostics_object, explanation_object, json_to_python, list_object, + move_object, read_declaration_target, read_filters, read_optional_target, read_result, + read_target, report_object, }; type PyObject = Py; -/// `(kind, message)` of the scope error Python raises: `path` or `identity`. -type FailureRow = Option<(&'static str, String)>; -/// Consumer, declaration, usage kind, what it came through and the enum member. -type UsageRow = ( - QualifiedRow, - QualifiedRow, - String, - Option, - Option, +/// Target, location, whether it is a directory, and the filters. +type ReportArguments<'py> = ( + Option>, + Option>, + bool, + Option>, ); -/// Resource, declaration, through and grant kind. -type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); -/// Resource, declaration, reason and what it came through. -type StandingRow = (IdentityRow, QualifiedRow, String, Option); -/// Every `ScopeIndex` tuple, with declaration values parallel to the declarations. -type ProjectionRows = ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - CompletenessRow, +/// Declaration, target, location and whether it is a directory. +type ExplainArguments<'py> = ( + Bound<'py, PyAny>, + Option>, + Option>, + bool, ); -type ReportRequestRow = (Option, Option, bool, FiltersRow); -type ExplainRequestRow = ( - (Option, String), - Option, - Option, +/// Folder, filters, target and whether the target is prospective. +type ListArguments<'py> = ( + String, + Option>, + Option>, bool, ); -type FolderRequestRow = (String, Option, bool, Option); -fn failure_row(failure: &ReportFailure) -> PyResult { - match failure { - ReportFailure::Path(rejected) => Ok(Some(("path", path_message(rejected)))), - ReportFailure::Identity(failure, text) => { - Ok(Some(("identity", identity_message(failure, text)))) - } - ReportFailure::Serialization(message) => Err(PyValueError::new_err(message.clone())), - ReportFailure::IndexOutOfRange => { - Ok(Some(("index", "tuple index out of range".to_owned()))) - } - } -} +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; -/// One lookup's reports: the lookup and every declaration's values in input order. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeScopeReports { +/// One lookup's native lookup and every declaration's values in input order. +struct LookupReports { lookup: Py, values: ReportValues, } -impl NativeScopeReports { +impl LookupReports { fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { ReportSource { records: &lookup.records, @@ -100,247 +74,256 @@ impl NativeScopeReports { } } -#[pymethods] -impl NativeScopeReports { - /// `query_scope_report`. - fn report( - &self, - py: Python<'_>, - request: ReportRequestRow, - ) -> PyResult<(Option, FailureRow)> { - let (target_row, at, directory, filter_row) = request; - let request: ReportRequest = ReportRequest { - target: target_row.map(target).transpose()?, - at, - directory, - filters: filters(filter_row), - }; - let lookup = self.lookup.get(); - match build_scope_report(self.source(lookup), &request) { - Ok(report) => Ok((Some(report_out(py, report)?), None)), - Err(failure) => Ok((None, failure_row(&failure)?)), - } +/// The reports of a `ScopeLookup`: its native lookup with its canonical declarations' values. +fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { + let index: Bound<'_, PyAny> = lookup.getattr("index")?; + let native: Py = lookup.getattr("native")?.extract()?; + let rows: Vec = index + .getattr("declarations")? + .try_iter()? + .map(|record| Ok(declaration_values(&read_declaration(&record?)?))) + .collect::>()?; + let canonical: &[usize] = &native.get().lookup.declarations; + if rows.len() != canonical.len() { + return Err(PyValueError::new_err( + "scope report values must match the lookup's declarations", + )); } + let mut declarations = vec![Default::default(); canonical.len()]; + for (position, row) in canonical.iter().zip(rows) { + declarations[*position] = row; + } + Ok(LookupReports { + lookup: native, + values: ReportValues { + declarations, + completeness: completeness(read_completeness(&index)?), + }, + }) +} - /// `explain_scope_declaration`. - fn explain( - &self, - py: Python<'_>, - request: ExplainRequestRow, - ) -> PyResult<(Option, FailureRow)> { - let (declaration, target_row, at, directory) = request; - let request: ExplainRequest = ExplainRequest { - declaration: declaration_target(declaration)?, - target: target_row.map(target).transpose()?, - at, - directory, - }; - let lookup = self.lookup.get(); - match explain_scope_declaration(self.source(lookup), &request) { - Ok((explanation, diagnostics)) => Ok(( - Some((explanation_out(py, explanation)?, raised_out(diagnostics))), - None, - )), - Err(failure) => Ok((None, failure_row(&failure)?)), +fn scope_error(py: Python<'_>, class: &str, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import(EXCEPTIONS_MODULE)? + .getattr(class)? + .call1((message,))?, + )) +} + +/// The error Python raises for a failed query. +fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { + match failure { + ReportFailure::Path(rejected) => { + scope_error(py, "InvalidScopePathError", path_message(rejected)) } + ReportFailure::Identity(failure, text) => scope_error( + py, + "InvalidQualifiedIdentityError", + identity_message(failure, text), + ), + ReportFailure::Serialization(message) => Ok(PyValueError::new_err(message.clone())), + ReportFailure::IndexOutOfRange => Ok(PyIndexError::new_err("tuple index out of range")), } +} - /// `browse_scope_folders`. - fn browse(&self, request: FolderRequestRow) -> PyResult { - let request: FolderRequest = folder_request(request)?; - let lookup = self.lookup.get(); - Ok(browse_out(browse_scope_folders( - self.source(lookup), - &request, - ))) - } +fn path_text(value: Option>) -> PyResult> { + value + .filter(|value| !value.is_none()) + .map(|value| value.str()?.extract()) + .transpose() +} + +fn raised(diagnostics: Vec) -> Vec { + diagnostics + .into_iter() + .map(ReportDiagnostic::Raised) + .collect() +} - /// `list_scope_declarations`. - fn list( - &self, - py: Python<'_>, - request: FolderRequestRow, - ) -> PyResult<(Option, FailureRow)> { - let request: FolderRequest = folder_request(request)?; - let lookup = self.lookup.get(); - match list_scope_declarations(self.source(lookup), &request) { - Ok(result) => Ok((Some(list_out(py, result)?), None)), - Err(failure) => Ok((None, failure_row(&failure)?)), +/// `query_scope_report`: one existing target or prospective path; invalid paths raise. +#[pyfunction] +fn scope_report( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ReportArguments<'_>, +) -> PyResult { + let (target, at, directory, filters) = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { + Some(filters) => filters, + None => classes.model("ScopeReportFilters")?.call0()?, + }; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ReportRequest = ReportRequest { + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + filters: read_filters(&selected)?, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match build_scope_report(reports.source(native), &request) { + Ok(report) => { + let diagnostics: Bound<'_, PyAny> = lookup.getattr("index")?.getattr("diagnostics")?; + Ok(report_object(&classes, report, (&selected, &diagnostics))?.unbind()) } + Err(failure) => Err(failure_error(py, &failure)?), } +} - /// `preview_scope_move`. - fn move_preview( - &self, - py: Python<'_>, - request: (TargetRow, String), - ) -> PyResult<(Option, FailureRow)> { - let request: MoveRequest = MoveRequest { - resource: target(request.0)?, - destination: request.1, - }; - let lookup = self.lookup.get(); - match preview_scope_move(self.source(lookup), &request) { - Ok((preview, diagnostics)) => Ok(( - Some(( - preview.map(|preview| move_out(py, preview)).transpose()?, - raised_out(diagnostics), - )), - None, - )), - Err(failure) => Ok((None, failure_row(&failure)?)), +/// `explain_scope_declaration`: one qualified declaration in an optional context. +#[pyfunction] +fn scope_explain( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ExplainArguments<'_>, +) -> PyResult<(PyObject, PyObject)> { + let (declaration, target, at, directory) = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ExplainRequest = ExplainRequest { + declaration: read_declaration_target(&classes, &declaration)?, + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match explain_scope_declaration(reports.source(native), &request) { + Ok((explanation, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + lookup.getattr("index")?.getattr("diagnostics")?; + Ok(( + explanation_object(&classes, explanation)?.unbind(), + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) } + Err(failure) => Err(failure_error(py, &failure)?), } } -fn folder_request(row: FolderRequestRow) -> PyResult { - let (folder, target_row, prospective, filter_row) = row; +fn folder_request( + classes: &ScopeClasses<'_>, + location: (String, Option>, bool), + filters: Option>, +) -> PyResult { + let (folder, target, prospective) = location; Ok(FolderRequest { folder, - target: target_row.map(target).transpose()?, + target: read_optional_target(classes, target.as_ref())?, prospective, - filters: filter_row.map(filters), + filters: filters + .filter(|filters| !filters.is_none()) + .map(|filters| read_filters(&filters)) + .transpose()?, }) } -/// The reports of `lookup`, with each canonical declaration's values and the completeness. +/// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. #[pyfunction] -fn build_native_scope_reports( - lookup: Py, - values: (Vec, CompletenessRow), -) -> PyResult { - let (rows, completeness_row) = values; - let native: &NativeScopeLookup = lookup.get(); - let canonical: &[usize] = &native.lookup.declarations; - if rows.len() != canonical.len() { - return Err(PyValueError::new_err( - "scope report values must match the lookup's declarations", - )); - } - let mut declarations = vec![Default::default(); canonical.len()]; - for (position, row) in canonical.iter().zip(rows) { - declarations[*position] = declaration_values(row)?; +fn scope_browse( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: (String, Option>, bool), +) -> PyResult { + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = folder_request(&classes, request, None)?; + let native: &NativeScopeLookup = reports.lookup.get(); + let result = browse_scope_folders(reports.source(native), &request); + let diagnostics: Bound<'_, PyAny> = lookup.getattr("index")?.getattr("diagnostics")?; + Ok(browse_object(&classes, result, &diagnostics)?.unbind()) +} + +/// `list_scope_declarations`: declarations recursively beneath one definition folder. +#[pyfunction] +fn scope_list( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + request: ListArguments<'_>, +) -> PyResult { + let (folder, filters, target, target_is_prospective) = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = + folder_request(&classes, (folder, target, target_is_prospective), filters)?; + let native: &NativeScopeLookup = reports.lookup.get(); + match list_scope_declarations(reports.source(native), &request) { + Ok(result) => { + let diagnostics: Bound<'_, PyAny> = lookup.getattr("index")?.getattr("diagnostics")?; + Ok(list_object(&classes, result, &diagnostics)?.unbind()) + } + Err(failure) => Err(failure_error(py, &failure)?), } - Ok(NativeScopeReports { - lookup, - values: ReportValues { - declarations, - completeness: completeness(completeness_row), - }, - }) } -/// `serialize_scope_report` of a `report`, `browse` or `list` result row. +/// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. #[pyfunction] -fn serialize_scope_result_row(kind: &str, row: &Bound<'_, PyAny>) -> PyResult { - use pyo3::prelude::PyAnyMethods; - let result: ScopeResult = match kind { - "report" => ScopeResult::Report(Box::new(report_in( - row.extract::, DiagnosticInRow>>()?, - )?)), - "browse" => ScopeResult::Browse(browse_in(row.extract::>()?)?), - _ => ScopeResult::List(list_in( - row.extract::, DiagnosticInRow>>()?, - )?), +fn scope_move_preview( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + resource: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<(Option, PyObject)> { + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: MoveRequest = MoveRequest { + resource: read_target(&classes, resource)?, + destination: destination.str()?.extract()?, }; + let native: &NativeScopeLookup = reports.lookup.get(); + match preview_scope_move(reports.source(native), &request) { + Ok((preview, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + lookup.getattr("index")?.getattr("diagnostics")?; + Ok(( + preview + .map(|preview| Ok::<_, PyErr>(move_object(&classes, preview)?.unbind())) + .transpose()?, + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) + } + Err(failure) => Err(failure_error(py, &failure)?), + } +} + +fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { + let result: ScopeResult = read_result(classes, result)?; serialize_scope_result(&result).map_err(|failure| match failure { ReportFailure::Serialization(message) => PyValueError::new_err(message), other => PyValueError::new_err(format!("{other:?}")), }) } -fn projection_index(rows: ProjectionRows) -> PyResult { - let ( - roots, - resources, - declarations, - values, - usages, - grants, - visibility, - inaccessible, - diagnostics, - completeness_row, - ) = rows; - let standing = - |(resource, declaration, reason, through): StandingRow| -> PyResult { - Ok(ProjectionStanding { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - reason, - through: through.map(qualified_identity).transpose()?, - }) - }; - Ok(ProjectionIndex { - roots: roots - .into_iter() - .map(ownership_root) - .collect::>()?, - resources: resources - .into_iter() - .map(resource_record) - .collect::>()?, - declarations: declarations - .into_iter() - .map(declaration_record) - .collect::>()?, - values: ReportValues { - declarations: values - .into_iter() - .map(declaration_values) - .collect::>()?, - completeness: completeness(completeness_row), - }, - usages: usages - .into_iter() - .map(|(consumer, declaration, kind, through, enum_member)| { - Ok(ProjectionUsage { - consumer: qualified_identity(consumer)?, - declaration: declaration_identity(declaration)?, - kind, - through: through.map(qualified_identity).transpose()?, - enum_member, - }) - }) - .collect::>()?, - grants: grants - .into_iter() - .map(|(resource, declaration, through, kind)| { - Ok(ProjectionGrant { - resource: resource_identity(resource)?, - declaration: declaration_identity(declaration)?, - through: qualified_identity(through)?, - kind: grant_kind(&kind)?, - }) - }) - .collect::>()?, - visibility: visibility - .into_iter() - .map(standing) - .collect::>()?, - inaccessible: inaccessible - .into_iter() - .map(standing) - .collect::>()?, - diagnostics: diagnostics - .into_iter() - .map(diagnostic_facts) - .collect::>()?, - }) +/// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. +#[pyfunction] +fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + match serialized(&classes, report) { + Err(error) if error.is_instance_of::(py) => { + let message: String = error.value(py).str()?.extract()?; + let raised: PyErr = scope_error(py, "ScopeError", message)?; + raised.set_cause(py, Some(error)); + Err(raised) + } + other => other, + } } -/// `scope_metadata_projection` of one index's rows, as the dictionary Python builds. +/// `scope_metadata_projection` of one `ScopeIndex`, as the dictionary Python builds. #[pyfunction] -fn scope_metadata_projection_rows(py: Python<'_>, rows: ProjectionRows) -> PyResult { - let index: ProjectionIndex = projection_index(rows)?; +#[pyo3(name = "scope_metadata_projection")] +fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult { + let index: ProjectionIndex = projection_index(read_index(index)?); let projection: JsonValue = scope_metadata_projection(&index); json_to_python(py, projection, false) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_native_scope_reports, module)?)?; - module.add_function(wrap_pyfunction!(serialize_scope_result_row, module)?)?; - module.add_function(wrap_pyfunction!(scope_metadata_projection_rows, module)?)?; + module.add_function(wrap_pyfunction!(scope_report, module)?)?; + module.add_function(wrap_pyfunction!(scope_explain, module)?)?; + module.add_function(wrap_pyfunction!(scope_browse, module)?)?; + module.add_function(wrap_pyfunction!(scope_list, module)?)?; + module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; + module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; + module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs new file mode 100644 index 0000000000..b21f626334 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs @@ -0,0 +1,239 @@ +//! The partial scope index of a project that failed strict discovery, with its fault diagnostics. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; +use sqlbuild_scopes::placement::models::DiagnosticOrderKey; +use sqlbuild_scopes::scope_index::models::{ + OwnershipRoot, OwnershipRootKind, ResourceIdentity, ResourceKind, ResourceRecord, +}; + +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, enum_text, field, optional_item, read_resource, root_object, tuple_attribute, +}; +use crate::bindings::_helpers::scopes::scope_validation::replace; + +/// Each authored resource root, in the order a prospective path is matched against them. +const PROSPECTIVE_ROOTS: [(ResourceKind, &str); 5] = [ + (ResourceKind::Model, "models"), + (ResourceKind::Test, "tests/unit"), + (ResourceKind::Scenario, "tests/scenarios"), + (ResourceKind::Function, "functions/sql"), + (ResourceKind::Source, "sources"), +]; +const RESOURCE_PARSE_ERROR: &str = "S019"; +const DECLARATION_PARSE_ERROR: &str = "S020"; +const MACRO_PARSE_ERROR: &str = "S021"; +const RELATIONSHIP_PARSE_ERROR: &str = "S022"; +const CONFIG_PARSE_ERROR: &str = "S023"; + +fn posix(path: &Bound<'_, PyAny>) -> PyResult { + path.call_method0("as_posix")?.extract() +} + +fn items<'py>(item: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + item.getattr(name)?.try_iter()?.collect() +} + +/// New diagnostics with their order keys. +struct Diagnostics<'py> { + classes: ScopeClasses<'py>, + objects: Vec>, + keys: Vec, +} + +impl<'py> Diagnostics<'py> { + fn push(&mut self, code: &str, message: String, path: Option) -> PyResult<()> { + let keywords: Bound<'py, PyDict> = PyDict::new(self.classes.py()); + keywords.set_item("path", path.as_deref())?; + self.objects + .push(self.classes.model("ScopeDiagnostic")?.call( + ( + self.classes.member("ScopeDiagnosticCode", code)?, + message.as_str(), + ), + Some(&keywords), + )?); + self.keys.push(DiagnosticOrderKey { + path, + line: None, + column: None, + code: code.to_owned(), + message, + }); + Ok(()) + } + + /// One `ScopeDiagnostic` per discovery fault, at the fault's POSIX path. + fn discovery_faults(&mut self, faults: &[Bound<'py, PyAny>], code: &str) -> PyResult<()> { + for fault in faults { + let path: Option = optional_item(fault, "path")? + .map(|path| posix(&path)) + .transpose()?; + self.push(code, field(fault, "message")?, path)?; + } + Ok(()) + } +} + +fn order_key(diagnostic: &Bound<'_, PyAny>) -> PyResult { + Ok(DiagnosticOrderKey { + path: field(diagnostic, "path")?, + line: field(diagnostic, "line")?, + column: field(diagnostic, "column")?, + code: enum_text(&diagnostic.getattr("code")?)?, + message: field(diagnostic, "message")?, + }) +} + +/// The resource a faulty authored file would declare, when it sits below a resource root. +fn prospective_resource(path: &Bound<'_, PyAny>) -> PyResult> { + let relative: String = posix(path)?; + for (kind, root) in PROSPECTIVE_ROOTS { + if relative.starts_with(&format!("{root}/")) { + return Ok(Some(ResourceRecord { + identity: ResourceIdentity { + kind, + name: path.getattr("stem")?.extract()?, + }, + path: relative, + ownership_root: OwnershipRoot { + path: root.to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(kind), + }, + })); + } + } + Ok(None) +} + +fn resource_object<'py>( + classes: &ScopeClasses<'py>, + record: &ResourceRecord, +) -> PyResult> { + classes.model("ResourceRecord")?.call1(( + classes.model("ResourceIdentity")?.call1(( + classes.member("ResourceKind", record.identity.kind.as_str())?, + record.identity.name.as_str(), + ))?, + record.path.as_str(), + root_object(classes, &record.ownership_root)?, + )) +} + +/// The index's resources and each faulty file's prospective resource, distinct and sorted. +fn tolerant_resources<'py>( + classes: &ScopeClasses<'py>, + index: &Bound<'py, PyAny>, + faults: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = classes.py(); + let mut records: Vec<(ResourceRecord, Bound<'py, PyAny>)> = Vec::new(); + for resource in tuple_attribute(index, "resources")?.iter() { + records.push((read_resource(&resource)?, resource)); + } + for fault in faults { + let Some(path) = optional_item(fault, "path")? else { + continue; + }; + if let Some(record) = prospective_resource(&path)? { + let object: Bound<'py, PyAny> = resource_object(classes, &record)?; + records.push((record, object)); + } + } + let mut distinct: Vec<(ResourceRecord, Bound<'py, PyAny>)> = Vec::with_capacity(records.len()); + for (record, object) in records { + if !distinct.iter().any(|(seen, _)| *seen == record) { + distinct.push((record, object)); + } + } + distinct.sort_by(|(left, _), (right, _)| { + (&left.path, left.identity.kind.as_str(), &left.identity.name).cmp(&( + &right.path, + right.identity.kind.as_str(), + &right.identity.name, + )) + }); + PyTuple::new(py, distinct.into_iter().map(|(_, object)| object)) +} + +/// The tolerant index: fault diagnostics, prospective resources and partial completeness. +#[pyfunction] +fn scope_tolerant_index( + py: Python<'_>, + index: &Bound<'_, PyAny>, + snapshot: &Bound<'_, PyAny>, + faults: (Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult> { + let (macro_faults, relationship_faults) = faults; + let resource_faults: Vec> = items(snapshot, "resource_faults")?; + let declaration_faults: Vec> = items(snapshot, "declaration_faults")?; + let snapshot_relationship_faults: Vec> = + items(snapshot, "relationship_faults")?; + let config_faults: Vec> = items(snapshot, "config_faults")?; + let macro_faults: Vec> = macro_faults.try_iter()?.collect::>()?; + let relationship_faults: Vec<(Bound<'_, PyAny>, String)> = relationship_faults + .try_iter()? + .map(|fault| fault?.extract()) + .collect::>()?; + let mut diagnostics: Diagnostics<'_> = Diagnostics { + classes: ScopeClasses::new(py)?, + objects: Vec::new(), + keys: Vec::new(), + }; + for diagnostic in tuple_attribute(index, "diagnostics")?.iter() { + diagnostics.keys.push(order_key(&diagnostic)?); + diagnostics.objects.push(diagnostic); + } + diagnostics.discovery_faults(&resource_faults, RESOURCE_PARSE_ERROR)?; + diagnostics.discovery_faults(&declaration_faults, DECLARATION_PARSE_ERROR)?; + diagnostics.discovery_faults(&snapshot_relationship_faults, RELATIONSHIP_PARSE_ERROR)?; + diagnostics.discovery_faults(&config_faults, CONFIG_PARSE_ERROR)?; + for fault in ¯o_faults { + let path: String = posix(&fault.getattr("relative_path")?)?; + diagnostics.push(MACRO_PARSE_ERROR, field(fault, "message")?, Some(path))?; + } + for (path, message) in &relationship_faults { + diagnostics.push( + RELATIONSHIP_PARSE_ERROR, + message.clone(), + Some(posix(path)?), + )?; + } + let mut prospective_faults: Vec> = resource_faults.clone(); + prospective_faults.extend(snapshot_relationship_faults.iter().cloned()); + let classes: &ScopeClasses<'_> = &diagnostics.classes; + let resources: Bound<'_, PyTuple> = tolerant_resources(classes, index, &prospective_faults)?; + let ordered: Vec> = diagnostic_order(&diagnostics.keys) + .into_iter() + .map(|position| diagnostics.objects[position].clone()) + .collect(); + let static_visibility: bool = + declaration_faults.is_empty() && config_faults.is_empty() && macro_faults.is_empty(); + let relationships: bool = + snapshot_relationship_faults.is_empty() && relationship_faults.is_empty(); + let completeness: Bound<'_, PyAny> = classes.model("ScopeCompleteness")?.call1(( + false, + static_visibility, + false, + relationships, + false, + false, + ))?; + Ok(replace( + index, + &[ + ("resources", resources.into_any()), + ("diagnostics", PyTuple::new(py, ordered)?.into_any()), + ("completeness", completeness), + ], + )? + .unbind()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_tolerant_index, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs new file mode 100644 index 0000000000..2ffafd3ecf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -0,0 +1,315 @@ +//! Compile usage merging, placement validation and move relocation over a Python `ScopeIndex`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; +use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; +use sqlbuild_scopes::placement::main::relocate_for_resource_move::relocate_for_resource_move; +use sqlbuild_scopes::placement::models::{ + DiagnosticOrderKey, PlacementCode, PlacementDiagnostic, PlacementIndex, Relocation, +}; +use sqlbuild_scopes::scope_index::models::ResourceIdentity; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::scopes::compile_usages::{ + MergeRows, SourceRow, UsageRow, merged_usage_rows, +}; +use crate::bindings::_helpers::scopes::placement::{QualifiedRow, resource_identity}; +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, enum_text, field, identity_row, optional_item, qualified_row, read_completeness, + read_placement_index, root_object, tuple_attribute, value_of, +}; +use crate::bindings::types::CompilerDetach; + +const DATACLASSES_MODULE: &str = "dataclasses"; +const RUNTIME_USAGE: usize = 2; + +/// `dataclasses.replace(item, **changes)`. +pub(crate) fn replace<'py>( + item: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = item.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in changes { + keywords.set_item(*name, value)?; + } + py.import(DATACLASSES_MODULE)? + .getattr("replace")? + .call((item,), Some(&keywords)) +} + +fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +fn usage_row(usage: &Bound<'_, PyAny>) -> PyResult { + Ok(( + qualified_row(&usage.getattr("consumer")?)?, + qualified_row(&usage.getattr("declaration")?)?, + value_of(usage, "kind")?, + optional_item(usage, "through")? + .map(|through| qualified_row(&through)) + .transpose()?, + field(usage, "enum_member")?, + )) +} + +fn dependency_row( + record: &Bound<'_, PyAny>, +) -> PyResult<(QualifiedRow, Option>)> { + let dependencies: Option> = match optional_item(record, "macro")? { + None => None, + Some(metadata) => Some( + metadata + .getattr("dependencies")? + .try_iter()? + .map(|item| qualified_row(&item?)) + .collect::>()?, + ), + }; + Ok((qualified_row(&record.getattr("identity")?)?, dependencies)) +} + +fn with_dependencies<'py>( + record: Bound<'py, PyAny>, + sources: Option>, + usages: &[Bound<'py, PyAny>], +) -> PyResult> { + let (Some(metadata), Some(sources)) = (optional_item(&record, "macro")?, sources) else { + return Ok(record); + }; + let own: Bound<'py, PyTuple> = tuple_attribute(&metadata, "dependencies")?; + let dependencies: Vec> = sources + .into_iter() + .map(|(is_own, position)| { + if is_own { + own.get_item(position) + } else { + usages[position].getattr("declaration") + } + }) + .collect::>()?; + let py: Python<'py> = record.py(); + let metadata: Bound<'py, PyAny> = replace( + &metadata, + &[("dependencies", tuple_from(py, dependencies)?)], + )?; + replace(&record, &[("macro", metadata)]) +} + +/// Python's `merged_usage_index`: compile usages merged into usages and macro dependencies. +fn merged_usage_index<'py>( + index: &Bound<'py, PyAny>, + collected: &Bound<'py, PyTuple>, +) -> PyResult> { + let py: Python<'py> = index.py(); + let records: Bound<'py, PyTuple> = tuple_attribute(index, "declarations")?; + let prior: Bound<'py, PyTuple> = tuple_attribute(index, "usages")?; + let declaration_rows: Vec<(QualifiedRow, Option>)> = records + .iter() + .map(|record| dependency_row(&record)) + .collect::>()?; + let prior_rows: Vec = prior + .iter() + .map(|usage| usage_row(&usage)) + .collect::>()?; + let collected_rows: Vec = collected + .iter() + .map(|usage| usage_row(&usage)) + .collect::>()?; + let (distinct, merged, combined): MergeRows = + merged_usage_rows(&declaration_rows, &prior_rows, &collected_rows); + let usages: Vec> = distinct + .iter() + .map(|position| collected.get_item(*position)) + .collect::>()?; + let mut all_usages: Vec> = prior.iter().collect(); + all_usages.extend(usages.iter().cloned()); + let declarations: Vec> = records + .iter() + .zip(merged) + .map(|(record, sources)| with_dependencies(record, sources, &usages)) + .collect::>()?; + let true_value: Bound<'py, PyAny> = true.into_bound_py_any(py)?; + let completeness: Bound<'py, PyAny> = replace( + &index.getattr("completeness")?, + &[ + ("runtime_usage", true_value.clone()), + ("promotion_impact", true_value), + ], + )?; + replace( + index, + &[ + ("declarations", tuple_from(py, declarations)?), + ( + "usages", + tuple_from( + py, + combined + .iter() + .map(|position| all_usages[*position].clone()) + .collect(), + )?, + ), + ("completeness", completeness), + ], + ) +} + +fn order_key(diagnostic: &Bound<'_, PyAny>) -> PyResult { + Ok(DiagnosticOrderKey { + path: field(diagnostic, "path")?, + line: field(diagnostic, "line")?, + column: field(diagnostic, "column")?, + code: enum_text(&diagnostic.getattr("code")?)?, + message: field(diagnostic, "message")?, + }) +} + +fn with_placement<'py>( + index: &Bound<'py, PyAny>, + diagnostics: Vec>, + placement: bool, +) -> PyResult> { + let py: Python<'py> = index.py(); + let completeness: Bound<'py, PyAny> = replace( + &index.getattr("completeness")?, + &[("placement", placement.into_bound_py_any(py)?)], + )?; + replace( + index, + &[ + ("diagnostics", tuple_from(py, diagnostics)?), + ("completeness", completeness), + ], + ) +} + +/// Python's `build_placement_validated_index`. +fn placement_validated_index<'py>( + index: &Bound<'py, PyAny>, + enforce_placement: bool, +) -> PyResult> { + let py: Python<'py> = index.py(); + let mut kept: Vec> = Vec::new(); + let mut keys: Vec = Vec::new(); + for diagnostic in tuple_attribute(index, "diagnostics")?.iter() { + if PlacementCode::parse(&enum_text(&diagnostic.getattr("code")?)?).is_none() { + keys.push(order_key(&diagnostic)?); + kept.push(diagnostic); + } + } + if !read_completeness(index)?[RUNTIME_USAGE] { + return with_placement(index, kept, false); + } + let classes: ScopeClasses<'py> = ScopeClasses::new(py)?; + let severity: Bound<'py, PyAny> = classes.member( + "DiagnosticSeverity", + if enforce_placement { + "error" + } else { + "warning" + }, + )?; + let placement: PlacementIndex = read_placement_index(index)?; + let found: Vec = py + .compiler_detach(|| Ok(placement_diagnostics(&placement))) + .map_err(compiler_error)?; + let records: Bound<'py, PyTuple> = tuple_attribute(index, "declarations")?; + let diagnostic_class: Bound<'py, PyAny> = classes.model("ScopeDiagnostic")?; + for item in found { + let record: Bound<'py, PyAny> = records.get_item(item.declaration)?; + let native = &placement.declarations[item.declaration]; + keys.push(DiagnosticOrderKey { + path: Some(native.path.clone()), + line: Some(native.line), + column: Some(native.column), + code: item.code.as_str().to_owned(), + message: item.message.clone(), + }); + kept.push(diagnostic_class.call1(( + classes.member("ScopeDiagnosticCode", item.code.as_str())?, + item.message, + &severity, + record.getattr("path")?, + record.getattr("line")?, + record.getattr("column")?, + record.getattr("identity")?, + ))?); + } + let ordered: Vec> = diagnostic_order(&keys) + .into_iter() + .map(|position| kept[position].clone()) + .collect(); + with_placement(index, ordered, true) +} + +/// Complete usage and placement diagnostics, merging any collected compile usages first. +#[pyfunction] +#[pyo3(signature = (index, enforce_placement, compile_usages))] +fn scope_placement_validated_index( + index: &Bound<'_, PyAny>, + enforce_placement: bool, + compile_usages: Option>, +) -> PyResult> { + let merged: Bound<'_, PyAny> = match compile_usages { + Some(collected) => merged_usage_index(index, &collected)?, + None => index.clone(), + }; + Ok(placement_validated_index(&merged, enforce_placement)?.unbind()) +} + +/// Declarations whose placement changes when `resource` moves, or `None` if unsettled. +#[pyfunction] +fn scope_relocated_declarations( + py: Python<'_>, + index: &Bound<'_, PyAny>, + resource: &Bound<'_, PyAny>, + destination: &str, +) -> PyResult>> { + if !read_completeness(index)?[RUNTIME_USAGE] { + return Ok(None); + } + let placement: PlacementIndex = read_placement_index(index)?; + let resource: ResourceIdentity = resource_identity(identity_row(resource)?)?; + let relocations: Option> = py + .compiler_detach(|| { + Ok(relocate_for_resource_move( + &placement, + &resource, + destination, + )) + }) + .map_err(compiler_error)?; + let Some(relocations) = relocations else { + return Ok(None); + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let records: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let mut moved: Vec> = Vec::with_capacity(relocations.len()); + for item in relocations { + let owning_path: Bound<'_, PyAny> = item.owning_path.into_bound_py_any(py)?; + moved.push(replace( + &records.get_item(item.declaration)?, + &[ + ("path", item.path.into_bound_py_any(py)?), + ("scope", classes.member("ScopeKind", item.scope.as_str())?), + ( + "ownership_root", + root_object(&classes, &item.ownership_root)?, + ), + ("owning_path", owning_path), + ], + )?); + } + Ok(Some(PyTuple::new(py, moved)?.unbind())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; + module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs new file mode 100644 index 0000000000..20f038e1f5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -0,0 +1,239 @@ +//! Static declaration visibility of scope targets and authored paths over a `ScopeLookup`. + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::models::{ + OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, +}; + +use crate::bindings::_helpers::scopes::scope_lookup::{ + NativeScopeLookup, ResolvedRows, TargetRows, path_message, scope_path_error, +}; +use crate::bindings::_helpers::scopes::scope_objects::{ + ScopeClasses, qualified_row, read_resource, tuple_attribute, +}; + +type PyObject = Py; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const CURRENT_PATH: &str = "."; + +fn native_lookup(lookup: &Bound<'_, PyAny>) -> PyResult> { + lookup.getattr("native")?.extract() +} + +/// Python's `query_target`: the resources or declarations `target` names. +fn target_query<'py>( + classes: &ScopeClasses<'py>, + lookup: &Bound<'py, PyAny>, + target: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let identity: bool = target.is_instance(&classes.model("ResourceIdentity")?)? + || target.is_instance(&classes.model("DeclarationIdentity")?)?; + let raw: String = if identity { + String::new() + } else { + target.str()?.extract()? + }; + let row = if identity { + Some(qualified_row(target)?) + } else { + None + }; + let (declarations, positions, identity_error, path_error): TargetRows = + native_lookup(lookup)?.get().target(row, &raw)?; + if let Some(message) = identity_error { + return Err(PyErr::from_value( + py.import(EXCEPTIONS_MODULE)? + .getattr("InvalidQualifiedIdentityError")? + .call1((message,))?, + )); + } + if let Some(message) = path_error { + return Err(scope_path_error(py, message)?); + } + let value: Bound<'py, PyAny> = if identity { + target.clone() + } else { + raw.into_bound_py_any(py)? + }; + let index: Bound<'py, PyAny> = lookup.getattr("index")?; + let records: Bound<'py, PyTuple> = tuple_attribute( + &index, + if declarations { + "declarations" + } else { + "resources" + }, + )?; + let matched: Vec> = positions + .iter() + .map(|position| records.get_item(*position)) + .collect::>()?; + let matched: Bound<'py, PyTuple> = PyTuple::new(py, matched)?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item( + if declarations { + "declaration_matches" + } else { + "matches" + }, + matched, + )?; + classes + .model("ScopeTargetQuery")? + .call((value,), Some(&keywords)) +} + +/// `query_target`: the `ScopeTargetQuery` of `target`; invalid text raises the scope error. +#[pyfunction] +fn scope_query_target( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, +) -> PyResult { + Ok(target_query(&ScopeClasses::new(py)?, lookup, target)?.unbind()) +} + +fn resolved( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + resource: &ResourceRecord, +) -> PyResult { + let rows: ResolvedRows = native_lookup(lookup)?.get().resolve(py, resource)?; + if let Some(message) = rows.2 { + return Err(scope_path_error(py, message)?); + } + Ok(rows) +} + +/// Every visible fact of each matched resource, and its inaccessible declarations. +#[pyfunction] +fn scope_resolve_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + target: &Bound<'_, PyAny>, + with_reasons: bool, +) -> PyResult { + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let query: Bound<'_, PyAny> = target_query(&classes, lookup, target)?; + let index: Bound<'_, PyAny> = lookup.getattr("index")?; + let identities: Bound<'_, PyTuple> = + tuple_attribute(&lookup.getattr("visibility_index")?, "identities")?; + let declarations: Bound<'_, PyTuple> = tuple_attribute(&index, "declarations")?; + let grants: Bound<'_, PyTuple> = tuple_attribute(&index, "grants")?; + let visible_class: Bound<'_, PyAny> = classes.model("VisibilityRecord")?; + let inaccessible_class: Bound<'_, PyAny> = classes.model("InaccessibleRecord")?; + let mut visible: Vec> = Vec::new(); + let mut inaccessible: Vec> = Vec::new(); + for resource in tuple_attribute(&query, "matches")?.iter() { + let (resource_visible, resource_inaccessible, _) = + resolved(py, lookup, &read_resource(&resource)?)?; + let owner: Bound<'_, PyAny> = resource.getattr("identity")?; + for (position, reason, grant) in resource_visible { + let through: Option> = grant + .map(|grant| grants.get_item(grant)?.getattr("through")) + .transpose()?; + visible.push(visible_class.call1(( + &owner, + identities.get_item(position)?, + classes.member("VisibilityReason", reason)?, + through, + ))?); + } + for (position, reason) in resource_inaccessible { + let identity: Bound<'_, PyAny> = + declarations.get_item(position)?.getattr("identity")?; + inaccessible.push(if with_reasons { + inaccessible_class.call1(( + &owner, + identity, + classes.member("InaccessibleReason", reason)?, + ))? + } else { + identity + }); + } + } + let result: &str = if with_reasons { + "VisibilityResolution" + } else { + "DeclarationVisibility" + }; + Ok(classes + .model(result)? + .call1(( + query, + PyTuple::new(py, visible)?, + PyTuple::new(py, inaccessible)?, + ))? + .unbind()) +} + +/// Visible and inaccessible declarations for an authored path, including declaration files. +#[pyfunction] +fn scope_resolve_path_visibility( + py: Python<'_>, + lookup: &Bound<'_, PyAny>, + path: &Bound<'_, PyAny>, +) -> PyResult<(Py, Py)> { + let text: String = path.str()?.extract()?; + let normalized: String = match normalize_scope_path(&text) { + Ok(normalized) => normalized, + Err(failure) => { + let rejected: RejectedPath = RejectedPath { + failure, + path: text, + }; + return Err(scope_path_error(py, path_message(&rejected))?); + } + }; + let resource: ResourceRecord = ResourceRecord { + identity: ResourceIdentity { + kind: ResourceKind::Model, + name: format!(""), + }, + path: normalized, + ownership_root: OwnershipRoot { + path: CURRENT_PATH.to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(ResourceKind::Model), + }, + }; + let (visible, inaccessible, _) = resolved(py, lookup, &resource)?; + let declarations: Bound<'_, PyTuple> = + tuple_attribute(&lookup.getattr("index")?, "declarations")?; + let pick = |positions: Vec| -> PyResult> { + let records: Vec> = positions + .into_iter() + .map(|position| declarations.get_item(position)) + .collect::>()?; + Ok(PyTuple::new(py, records)?.unbind()) + }; + Ok(( + pick( + visible + .into_iter() + .map(|(position, _, _)| position) + .collect(), + )?, + pick( + inaccessible + .into_iter() + .map(|(position, _)| position) + .collect(), + )?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; + module.add_function(wrap_pyfunction!(scope_resolve_visibility, module)?)?; + module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs new file mode 100644 index 0000000000..1546c8daee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs @@ -0,0 +1,140 @@ +//! Reject authored SQL tests and scenarios whose CTE names cannot be read, at the authored text. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_analysis::compiler::main::sql_test_classification::extract_sql_tests; +use sqlbuild_analysis::compiler::models::{SqlTestExtractionFailure, SqlTestRequest}; +use sqlbuild_analysis::sql_test_attachment::main::authored_test_offset::authored_test_offset; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; +const MODEL_MODE: &str = "model"; + +/// One authored SQL body to read: its text, file contents, relative path and body span. +struct AuthoredBody { + sql: String, + contents: String, + relative_path: String, + /// `str(relative_path)`, the label extraction errors name the file by. + file_label: String, + span: Option<(usize, usize)>, +} + +impl AuthoredBody { + fn read( + owner: &Bound<'_, PyAny>, + sql: &Bound<'_, PyAny>, + file: &Bound<'_, PyAny>, + ) -> PyResult { + Ok(Self { + sql: sql.extract()?, + contents: file.getattr("contents")?.extract()?, + relative_path: file + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?, + file_label: file.getattr("relative_path")?.str()?.extract()?, + span: owner.getattr("sql_body_span")?.extract()?, + }) + } +} + +/// `CompileInputError(message, help=help)` at the authored offset when it is exact. +fn located_error(py: Python<'_>, failure: &SqlTestExtractionFailure, body: &AuthoredBody) -> PyErr { + let offset: Option = failure.token.as_ref().and_then(|(_, offset)| { + authored_test_offset(&body.contents, &body.sql, body.span, *offset) + }); + let message: String = match offset { + Some(offset) => { + let position: TextPosition = text_position(&body.contents, offset); + format!( + "{}\n --> {}:{}:{}", + failure.message, body.relative_path, position.line, position.column + ) + } + None => failure.message.clone(), + }; + let raised = || -> PyResult { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", failure.help.as_deref())?; + let error: Bound<'_, PyAny> = py + .import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call((message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + }; + raised().unwrap_or_else(|error| error) +} + +/// Raise the located error of the first body whose authored CTEs cannot be read. +fn check_bodies( + py: Python<'_>, + bodies: &[AuthoredBody], + scenario: bool, + syntax: &LexicalSyntax, +) -> PyResult<()> { + if bodies.is_empty() { + return Ok(()); + } + let requests: Vec = bodies + .iter() + .map(|body| SqlTestRequest { + sql: body.sql.clone(), + file_label: body.file_label.clone(), + mode: MODEL_MODE.to_owned(), + raw: false, + authored: true, + scenario, + }) + .collect(); + match extract_sql_tests(&requests, syntax) { + Ok(_) => Ok(()), + Err(failure) => Err(located_error(py, &failure, &bodies[failure.index])), + } +} + +/// Reject the first authored test block, then scenario, whose CTE names cannot be read. +#[pyfunction] +fn validate_authored_sql_test_ctes( + py: Python<'_>, + test_files: Bound<'_, PyAny>, + scenario_files: Bound<'_, PyAny>, + syntax: LexicalSyntaxInput, +) -> Result<(), PyErr> { + let syntax: LexicalSyntax = syntax.into(); + let mut blocks: Vec = Vec::new(); + for test_file in test_files.try_iter()? { + let test_file: Bound<'_, PyAny> = test_file?; + for block in test_file.getattr("blocks")?.try_iter()? { + let block: Bound<'_, PyAny> = block?; + blocks.push(AuthoredBody::read( + &block, + &block.getattr("sql_body")?, + &test_file, + )?); + } + } + check_bodies(py, &blocks, false, &syntax)?; + let mut scenarios: Vec = Vec::new(); + for scenario_file in scenario_files.try_iter()? { + let scenario_file: Bound<'_, PyAny> = scenario_file?; + scenarios.push(AuthoredBody::read( + &scenario_file, + &scenario_file.getattr("sql_body")?, + &scenario_file, + )?); + } + check_bodies(py, &scenarios, true, &syntax) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_authored_sql_test_ctes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 75d847c142..9150680c73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod assembly; +pub(crate) mod authored_ctes; pub(crate) mod planning; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index ac702ccbfd..d2e33df977 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::sql_test_glue::{assembly, planning}; +use crate::bindings::_helpers::sql_test_glue::{assembly, authored_ctes, planning}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + authored_ctes::register(module)?; planning::register(module)?; assembly::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 83f06928db..98e4ac6e68 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,9 +4,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, analysis_session, attachments, boundary, cache, compile_outputs, compiled_project, - contracts, discovery, graph, lineage, model_config, project_assembly, refactor, render, rules, - scopes, semantic_checks, sql_test_glue, sqltext, type_system, + analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, + compiled_project, contracts, discovery, graph, lineage, model_config, project_assembly, + refactor, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -56,5 +56,6 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { compile_outputs::registration::register(module)?; graph::registration::register(module)?; compiled_project::registration::register(module)?; + compile_driver::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs b/crates/sqlbuild-scopes/src/placement/_helpers/declaration_directory.rs similarity index 96% rename from crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs rename to crates/sqlbuild-scopes/src/placement/_helpers/declaration_directory.rs index cc8d87f169..9aed373875 100644 --- a/crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs +++ b/crates/sqlbuild-scopes/src/placement/_helpers/declaration_directory.rs @@ -5,7 +5,7 @@ use crate::scope_index::models::{DeclarationKind, ScopeKind}; /// Python's `declaration_directory`. #[must_use] -pub fn declaration_directory( +pub(crate) fn declaration_directory( kind: DeclarationKind, scope: ScopeKind, path: Option<&str>, diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs b/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs index 9799141027..57f1827422 100644 --- a/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs +++ b/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod anchors; +pub(crate) mod declaration_directory; pub(crate) mod posix_paths; pub(crate) mod required; pub(crate) mod roles; diff --git a/crates/sqlbuild-scopes/src/placement/main/diagnostic_order.rs b/crates/sqlbuild-scopes/src/placement/main/diagnostic_order.rs new file mode 100644 index 0000000000..0c9a70d256 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/diagnostic_order.rs @@ -0,0 +1,20 @@ +//! The canonical order of scope diagnostics. + +use crate::placement::models::DiagnosticOrderKey; + +/// Positions of `keys` in Python's stable sort by path, line, column, code and message. +#[must_use] +pub fn diagnostic_order(keys: &[DiagnosticOrderKey]) -> Vec { + let mut positions: Vec = (0..keys.len()).collect(); + positions.sort_by_key(|position| { + let key: &DiagnosticOrderKey = &keys[*position]; + ( + key.path.as_deref().unwrap_or_default(), + key.line.unwrap_or_default(), + key.column.unwrap_or_default(), + key.code.as_str(), + key.message.as_str(), + ) + }); + positions +} diff --git a/crates/sqlbuild-scopes/src/placement/main/mod.rs b/crates/sqlbuild-scopes/src/placement/main/mod.rs index 7b79db2f39..c605776667 100644 --- a/crates/sqlbuild-scopes/src/placement/main/mod.rs +++ b/crates/sqlbuild-scopes/src/placement/main/mod.rs @@ -1,4 +1,4 @@ -pub mod declaration_directory; +pub mod diagnostic_order; pub mod placement_diagnostics; pub mod relocate_for_resource_move; pub mod required_declaration_placements; diff --git a/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs b/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs index 69cb00cfbf..2eff50bd04 100644 --- a/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs +++ b/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs @@ -3,9 +3,9 @@ use crate::placement::_helpers::anchors::{ AnchorSets, UsagesByDeclaration, anchor_sets, usages_by_declaration, }; +use crate::placement::_helpers::declaration_directory::declaration_directory; use crate::placement::_helpers::required::{owner_uses_directly, required_placement}; use crate::placement::_helpers::roles::scope_label; -use crate::placement::main::declaration_directory::declaration_directory; use crate::placement::models::{ PlacementCode, PlacementDiagnostic, PlacementIndex, PlacementUsage, RequiredPlacement, }; diff --git a/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs b/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs index 2ff103097f..1327dc38a0 100644 --- a/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs +++ b/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs @@ -5,8 +5,8 @@ use std::collections::{HashMap, HashSet}; use crate::placement::_helpers::anchors::{ AnchorSets, UsagesByDeclaration, anchor_sets, usages_by_declaration, }; +use crate::placement::_helpers::declaration_directory::declaration_directory; use crate::placement::_helpers::required::required_placement; -use crate::placement::main::declaration_directory::declaration_directory; use crate::placement::main::placement_diagnostics::placement_diagnostics; use crate::placement::models::{PlacementIndex, Relocation, RequiredPlacement}; use crate::scope_index::models::{ diff --git a/crates/sqlbuild-scopes/src/placement/models.rs b/crates/sqlbuild-scopes/src/placement/models.rs index 25a1b7417c..bab319791d 100644 --- a/crates/sqlbuild-scopes/src/placement/models.rs +++ b/crates/sqlbuild-scopes/src/placement/models.rs @@ -32,6 +32,19 @@ pub enum PlacementCode { } impl PlacementCode { + /// The placement code with Python's `ScopeDiagnosticCode` value `code`. + pub fn parse(code: &str) -> Option { + [ + Self::LocalNeededByDescendant, + Self::OverBroadInherited, + Self::RequiresGlobalPlacement, + Self::UnusedDeclaration, + Self::OverBroadGlobal, + ] + .into_iter() + .find(|item| item.as_str() == code) + } + /// Python's `ScopeDiagnosticCode` value. pub fn as_str(self) -> &'static str { match self { @@ -69,3 +82,13 @@ pub struct Relocation { pub ownership_root: OwnershipRoot, pub owning_path: Option, } + +/// The fields Python orders scope diagnostics by. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DiagnosticOrderKey { + pub path: Option, + pub line: Option, + pub column: Option, + pub code: String, + pub message: String, +} diff --git a/crates/sqlbuild-scopes/src/placement/tests/helpers.rs b/crates/sqlbuild-scopes/src/placement/tests/helpers.rs index e99edb192f..a32c9a15ab 100644 --- a/crates/sqlbuild-scopes/src/placement/tests/helpers.rs +++ b/crates/sqlbuild-scopes/src/placement/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::placement::models::{PlacementIndex, PlacementUsage}; +use crate::placement::models::{DiagnosticOrderKey, PlacementIndex, PlacementUsage}; use crate::scope_index::models::{ DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, QualifiedIdentity, ResourceIdentity, ResourceKind, ResourceRecord, ScopeKind, @@ -108,3 +108,30 @@ pub(super) fn owned_relocations( .map(|(position, path, owning)| (*position, (*path).to_owned(), owning.map(str::to_owned))) .collect() } + +fn order_key( + path: Option<&str>, + line: Option, + code: &str, + message: &str, +) -> DiagnosticOrderKey { + DiagnosticOrderKey { + path: path.map(str::to_owned), + line, + column: None, + code: code.to_owned(), + message: message.to_owned(), + } +} + +/// Diagnostics without a path or line first, then by path, line, code and message. +pub(super) fn mixed_diagnostics() -> Vec { + vec![ + order_key(Some("models/b.sql"), Some(2), "S010", "b"), + order_key(Some("models/a.sql"), Some(9), "S010", "a"), + order_key(None, None, "S019", "parse"), + order_key(Some(""), Some(0), "S019", "parse"), + order_key(Some("models/a.sql"), Some(9), "S007", "z"), + order_key(Some("models/b.sql"), None, "S010", "b"), + ] +} diff --git a/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs b/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs index cfecc3a23f..86b9b74014 100644 --- a/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs +++ b/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs @@ -1,12 +1,14 @@ -use crate::placement::main::declaration_directory::declaration_directory; +use crate::placement::_helpers::declaration_directory::declaration_directory; +use crate::placement::main::diagnostic_order::diagnostic_order; use crate::placement::main::placement_diagnostics::placement_diagnostics; use crate::placement::main::relocate_for_resource_move::relocate_for_resource_move; use crate::placement::models::Relocation; use crate::placement::tests::helpers::{ - chained_macros, descendant_models, model, owned_relocations, + chained_macros, descendant_models, mixed_diagnostics, model, owned_relocations, }; use crate::placement::tests::test_types::{ - DeclarationDirectoryTestCase, PlacementDiagnosticsTestCase, RelocationTestCase, + DeclarationDirectoryTestCase, DiagnosticOrderTestCase, PlacementDiagnosticsTestCase, + RelocationTestCase, }; use crate::scope_index::models::{DeclarationKind, ScopeKind}; @@ -147,3 +149,20 @@ fn given_resource_move_when_relocating_then_chained_local_declarations_follow() ); } } + +#[test] +fn given_diagnostics_when_ordering_then_python_sort_key_and_stability_hold() { + let test_cases = [DiagnosticOrderTestCase { + description: "missing paths and lines sort as empty and zero; equal keys keep their order", + keys: mixed_diagnostics, + expected_order: &[2, 3, 4, 1, 5, 0], + }]; + for test_case in test_cases { + assert_eq!( + diagnostic_order(&(test_case.keys)()), + test_case.expected_order, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/placement/tests/test_types.rs b/crates/sqlbuild-scopes/src/placement/tests/test_types.rs index 070617d7c9..755b469832 100644 --- a/crates/sqlbuild-scopes/src/placement/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/placement/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::placement::models::PlacementIndex; +use crate::placement::models::{DiagnosticOrderKey, PlacementIndex}; use crate::scope_index::models::{DeclarationKind, ScopeKind}; /// A usage-complete index and the `(declaration, code, message)` diagnostics it raises. @@ -27,3 +27,10 @@ pub(super) struct RelocationTestCase { pub(super) destination: &'static str, pub(super) expected_relocations: Option<&'static [RelocationCase]>, } + +/// Diagnostic order keys and the stable order Python sorts them into. +pub(super) struct DiagnosticOrderTestCase { + pub(super) description: &'static str, + pub(super) keys: fn() -> Vec, + pub(super) expected_order: &'static [usize], +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/macro_source_digest.rs b/crates/sqlbuild-scopes/src/scope_index/main/macro_source_digest.rs new file mode 100644 index 0000000000..1ac83b2653 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/macro_source_digest.rs @@ -0,0 +1,12 @@ +//! The source digest of a loaded macro. + +use sha2::{Digest, Sha256}; + +/// Lowercase hex SHA-256 of the UTF-8 macro source, as Python's `hashlib` gives it. +#[must_use] +pub fn macro_source_digest(source: &str) -> String { + Sha256::digest(source.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs index 6e5a72df95..083babc7d0 100644 --- a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -4,6 +4,7 @@ pub mod classify_resource; pub mod explain_report_declaration; pub mod format_identity; pub mod identity_repr; +pub mod macro_source_digest; pub mod normalize_scope_path; pub mod parse_identity; pub mod path_visible; diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs index 8a8e696f2d..4bfad639e1 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -4,6 +4,8 @@ mod build_scope_index; mod classify_resource; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_macro_source_digest.rs"] +mod macro_source_digest; #[path = "tests/test_parse_identity.rs"] mod parse_identity; #[path = "tests/test_path_visible.rs"] diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_macro_source_digest.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_macro_source_digest.rs new file mode 100644 index 0000000000..31d0c5636a --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_macro_source_digest.rs @@ -0,0 +1,31 @@ +use crate::scope_index::main::macro_source_digest::macro_source_digest; +use crate::scope_index::tests::test_types::MacroDigestTestCase; + +#[test] +fn given_macro_source_when_digesting_then_python_sha256_hex_matches() { + let test_cases = [ + MacroDigestTestCase { + description: "empty source", + source: "", + expected_digest: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + MacroDigestTestCase { + description: "a one-function macro module", + source: "def cents(value: str) -> str:\n return f\"{value} * 100\"\n", + expected_digest: "a1b0650d0b624a37526376fb5059b1d9b5c790c4d27b9ee561f4f6a318f0db9e", + }, + MacroDigestTestCase { + description: "non-ASCII text digests its UTF-8 bytes", + source: "caf\u{e9}", + expected_digest: "850f7dc43910ff890f8879c0ed26fe697c93a067ad93a7d50f466a7028a9bf4e", + }, + ]; + for test_case in test_cases { + assert_eq!( + macro_source_digest(test_case.source), + test_case.expected_digest, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs index 3d7ff5f2ab..25c945a4cc 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -108,3 +108,10 @@ pub(super) struct ParseIdentityTestCase { crate::scope_index::models::IdentityFailure, >, } + +/// Macro source text and its digest. +pub(super) struct MacroDigestTestCase { + pub(super) description: &'static str, + pub(super) source: &'static str, + pub(super) expected_digest: &'static str, +} diff --git a/fensu.toml b/fensu.toml index 7e73547683..de395eefcc 100644 --- a/fensu.toml +++ b/fensu.toml @@ -321,6 +321,7 @@ crate_names = [ "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", + "sqlbuild-pysource", "sqlbuild-python", "sqlbuild-refactor", "sqlbuild-render", diff --git a/python-boundary.toml b/python-boundary.toml index a823f96233..53b2f8ecfa 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 141910 +remaining_lines = 137187 [[stays]] path = "executor" @@ -219,3 +219,7 @@ note = "A native lint failure is reported as a lint error result, not recomputed [[handled_native_errors]] location = "rule_engine/_helpers/run/rules.py::_cached_sql_findings" note = "An undecodable cached finding is a cache miss; the rules run again natively." + +[[handled_native_errors]] +location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" +note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 53fabed117..4ee7605459 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,12 +1,19 @@ """Private SQLBuild native engine bindings.""" from collections.abc import Callable, Mapping, Sequence -from pathlib import Path -from typing import Any, TypedDict - +from pathlib import Path, PurePath +from typing import Any, Literal, TypedDict, overload + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts +from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild.cli.compile.models import CompileAnalysis +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( + CompileAnalysisSelection, CompileAuditInput, CompiledAudit, CompiledFunction, @@ -33,10 +40,14 @@ from sqlbuild.compiler.compile.models import ( InferredColumn, LoadedMacro, MacroContext, + StaticMacroExport, + StaticMacroFault, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledModelView, + CompiledProjectView, + CompileInputsView, FunctionLanguage, ) from sqlbuild.compiler.discovery.models import ( @@ -53,12 +64,35 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSourceFile, DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, + DiscoveredSqlScenarioFile, + DiscoveredSqlTestFile, EnumDeclaration, ModelSchemaDeclaration, + TolerantScopeDiscovery, ) +from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability +from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.references.models import LiteralSqlRelation from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord +from sqlbuild.compiler.scopes.models import ( + DeclarationExplanation, + DeclarationIdentity, + DeclarationRecord, + DeclarationVisibility, + MovePreview, + ResourceIdentity, + ScopeBrowseResult, + ScopeDiagnostic, + ScopeIndex, + ScopeListResult, + ScopeLookup, + ScopeReport, + ScopeReportFilters, + ScopeTargetQuery, + UsageRecord, + VisibilityResolution, +) +from sqlbuild.compiler.scopes.types import JsonValue, ScopeQueryTarget from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssemblyRequest, @@ -67,6 +101,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -881,6 +916,84 @@ def parse_schema_document( def parse_sources_document( loaded: object, path: str, classes: dict[str, object] ) -> tuple[SourceEntry, ...]: ... +def analyze_compile_project( + *, + project_dir: Path, + no_sql_validation: bool, + selected_target: str | None, + lineage_mode: CompileLineageMode, + cli_vars: dict[str, object] | None, + profile_flags: CompileProfileFlags, + analysis_selection: CompileAnalysisSelection, + status: TransientStatusReporter | None, + prepared_artifacts: PreparedCompileArtifacts | None = None, +) -> CompileAnalysis: ... +def build_compiled_project( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None = None, + no_sql_validation: bool = False, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + cli_vars: dict[str, object] | None = None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + resolved_connection: dict[str, object] | None = None, + analysis_selection: CompileAnalysisSelection | None = None, +) -> CompiledProjectView: ... +def build_project_graph_with_analysis_selection( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + analysis_selection: CompileAnalysisSelection, + selected_target: str | None = None, + no_sql_validation: bool = False, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + cli_vars: dict[str, object] | None = None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + on_progress: Callable[[str], None] | None = None, +) -> ProjectGraph: ... +def build_project_graph( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None = None, + no_sql_validation: bool = False, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + cli_vars: dict[str, object] | None = None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + on_progress: Callable[[str], None] | None = None, + no_cache: bool = False, +) -> ProjectGraph: ... +def run_compile_command( + request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None +) -> int: ... +def validate_authored_sql_test_ctes( + test_files: tuple[DiscoveredSqlTestFile, ...], + scenario_files: tuple[DiscoveredSqlScenarioFile, ...], + syntax: dict[str, object], +) -> None: ... +def validate_hook_sql_syntax( + value: object, + hook_name: str, + model: tuple[str, Path], + parsing: tuple[dict[str, str] | None, str | None] = (None, None), +) -> None: ... +def validate_sql_syntax( + query_sql: str, + model: tuple[str, Path], + parsing: tuple[dict[str, str] | None, str | None] = (None, None), +) -> None: ... +def hook_sql_statements(value: object) -> list[str]: ... +def declared_relations( + inputs: CompileInputsView, +) -> tuple[ + dict[str, dict[str, str]], + dict[str, dict[str, InferredNullability]], + dict[str, dict[str, str]], +]: ... def externally_described_models( inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] ) -> list[Path]: ... @@ -946,97 +1059,85 @@ class NativeDeclarationContexts: ) -> tuple[Any, bool]: ... class NativeScopeIndex: - def records( - self, - ) -> tuple[ - list[str], - list[tuple[str, str, str, str | None, str | None]], - list[int], - list[int], - list[tuple[int, int]], - list[tuple[str, str, str | None, int | None, int | None, int | None, int | None]], - bool, - ]: ... + @property + def index(self) -> ScopeIndex: ... + @property + def has_scoped_relationship_declarations(self) -> bool: ... def grant_relationships( self, - files: list[tuple[str, list[tuple[str, str, str]]]], - scenarios: list[tuple[str, str, str]], - settings: dict[str, object], - ) -> tuple[list[tuple[str, str, int, str | int, str]] | None, list[tuple[str, str]]]: ... - -def scope_placement_diagnostics( - rows: tuple[list[Any], list[Any], list[Any]], -) -> list[tuple[int, str, str]]: ... -def scope_relocations( - rows: tuple[list[Any], list[Any], list[Any]], - resource: tuple[str, str], - destination: str, -) -> list[tuple[int, str, str, tuple[str, str, str | None], str | None]] | None: ... -def scope_declaration_directory(kind: str, scope: str, path: str | None) -> str: ... - -type _ScopeQualified = tuple[str, str, tuple[str, str] | None] -type _ScopeRoot = tuple[str, str, str | None] -type _ScopeResource = tuple[tuple[str, str], str, _ScopeRoot] -type _ScopeGroups = list[list[int]] - -class NativeScopeLookup: - def target( - self, identity: _ScopeQualified | None, text: str, / - ) -> tuple[bool, list[int], str | None, str | None]: ... - def resolve( - self, resource: _ScopeResource, / - ) -> tuple[list[tuple[int, str, int | None]], list[tuple[int, str]], str | None]: ... - -def build_native_scope_lookup( - rows: tuple[ - list[_ScopeRoot], - list[_ScopeResource], - list[tuple[_ScopeQualified, str, int, int, str, _ScopeRoot, str | None]], - list[tuple[_ScopeQualified, _ScopeQualified, str, _ScopeQualified | None]], - list[tuple[tuple[str, str], _ScopeQualified, _ScopeQualified, str]], - list[tuple[str | None, int | None, int | None, str]], - ], + *, + discovered_inputs: DiscoveredProjectInputs, + syntax: SqlLexicalSyntax, + scan_store: tuple[SqlTestScanStore | None, str], + ) -> tuple[tuple[Path, str], ...]: ... + def index_with_relationships(self) -> ScopeIndex: ... + +def open_native_scope_index( + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro], + static_macros: tuple[StaticMacroExport, ...], /, -) -> tuple[ - NativeScopeLookup | None, - tuple[ - tuple[ - tuple[list[int], list[int], list[int], list[int], list[int], list[int]], - tuple[ - _ScopeGroups, - _ScopeGroups, - _ScopeGroups, - _ScopeGroups, - _ScopeGroups, - _ScopeGroups, - list[str], - ], - tuple[ - list[int], - list[tuple[tuple[str, str], list[int]]], - list[tuple[str, list[int]]], - list[tuple[str, list[int]]], - ], - ] - | None, - str | None, +) -> NativeScopeIndex: ... +def scope_tolerant_index( + index: ScopeIndex, + snapshot: TolerantScopeDiscovery, + faults: tuple[tuple[StaticMacroFault, ...], tuple[tuple[Path, str], ...]], + /, +) -> ScopeIndex: ... +def scope_placement_validated_index( + index: ScopeIndex, + enforce_placement: bool, + compile_usages: tuple[UsageRecord, ...] | None, +) -> ScopeIndex: ... +def scope_relocated_declarations( + index: ScopeIndex, resource: ResourceIdentity, destination: str +) -> tuple[DeclarationRecord, ...] | None: ... + +class NativeScopeLookup: ... + +def scope_query_target(lookup: ScopeLookup, target: ScopeQueryTarget, /) -> ScopeTargetQuery: ... +@overload +def scope_resolve_visibility( + lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[True], / +) -> VisibilityResolution: ... +@overload +def scope_resolve_visibility( + lookup: ScopeLookup, target: ScopeQueryTarget, with_reasons: Literal[False], / +) -> DeclarationVisibility: ... +def scope_resolve_path_visibility( + lookup: ScopeLookup, path: str | PurePath, / +) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: ... +def build_scope_lookup(index: ScopeIndex) -> ScopeLookup: ... +def scope_report( + lookup: ScopeLookup, + request: tuple[ScopeQueryTarget | None, str | PurePath | None, bool, ScopeReportFilters | None], + /, +) -> ScopeReport: ... +def scope_explain( + lookup: ScopeLookup, + request: tuple[ + str | DeclarationIdentity, + str | PurePath | ResourceIdentity | None, + str | PurePath | None, + bool, ], -]: ... - -class NativeScopeReports: - def report(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... - def explain(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... - def browse(self, request: tuple[Any, ...], /) -> Any: ... - def list(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... - def move_preview(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... - -def build_native_scope_reports( - lookup: NativeScopeLookup, - values: tuple[list[Any], tuple[bool, bool, bool, bool, bool, bool]], /, -) -> NativeScopeReports: ... -def serialize_scope_result_row(kind: str, row: object, /) -> str: ... -def scope_metadata_projection_rows(rows: tuple[Any, ...], /) -> Any: ... +) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: ... +def scope_browse( + lookup: ScopeLookup, + request: tuple[str, str | PurePath | ResourceIdentity | None, bool], + /, +) -> ScopeBrowseResult: ... +def scope_list( + lookup: ScopeLookup, + request: tuple[str, ScopeReportFilters | None, str | PurePath | ResourceIdentity | None, bool], + /, +) -> ScopeListResult: ... +def scope_move_preview( + lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath +) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: ... +def serialize_scope_report(report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: ... +def scope_metadata_projection(index: ScopeIndex) -> dict[str, JsonValue]: ... def effective_config_target_name( project_config: object, local_config: object, selected: str | None, / ) -> tuple[str | None, str | None]: ... @@ -1124,39 +1225,14 @@ def scope_index_fingerprint( inputs: tuple[list[str], list[str], list[str], list[str], str], /, ) -> str | None: ... -def read_scope_index_cache(path: Path, fingerprint: str, /) -> Any: ... -def write_scope_index_cache(location: tuple[Path, str], fingerprint: str, rows: Any, /) -> bool: ... +def read_scope_index_cache(path: Path, fingerprint: str, /) -> ScopeIndex | None: ... +def write_scope_index_cache( + location: tuple[Path, str], fingerprint: str, index: ScopeIndex, / +) -> bool: ... def scope_schema_usages( schemas: list[tuple[str, str | None, list[tuple[str, str | None]]]], records: list[tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]], ) -> tuple[list[tuple[int, int]], list[tuple[list[str], str, str, str]], str | None]: ... -def scope_merged_usages( - declarations: list[ - tuple[ - tuple[str, str, tuple[str, str] | None], - list[tuple[str, str, tuple[str, str] | None]] | None, - ] - ], - prior: list[Any], - collected: list[Any], -) -> tuple[list[int], list[list[tuple[bool, int]] | None], list[int]]: ... -def build_native_scope_index( - resources: list[tuple[str, str, str, str]], - declarations: list[ - tuple[ - str, - str, - tuple[str, str] | None, - str, - int, - str, - str | None, - str, - str | None, - list[tuple[str, str, tuple[str, str] | None]], - ] - ], -) -> NativeScopeIndex | None: ... class NativeConfigError: class_name: str @@ -1282,6 +1358,16 @@ def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... def macro_call_names(sql: str, nested: bool, /) -> tuple[list[str] | None, str | None]: ... +def inventory_macro_modules( + sources: list[ + tuple[str, str, list[str], list[str] | None, list[str] | None, str, str, str | None] + ], + /, +) -> tuple[ + list[tuple[str, int, list[str], int, str, list[str]]], + list[tuple[int, str]], + list[tuple[int, str, list[str]]] | None, +]: ... def splice_macro_calls( sql: str, sites: list[tuple[int, int]], outputs: list[str] ) -> tuple[str, list[tuple[int, int, int, int]]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index 59bab771b1..148a3b2d0b 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( enforce_model_execution_limit, @@ -33,7 +34,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.clone_with_options import run_clone_pipeline_with_options -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.main.unit_test_selection import split_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/collection.py b/src/sqlbuild/cli/commands/_helpers/compile/collection.py deleted file mode 100644 index 7bfe2bb3c7..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/collection.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Pause cyclic garbage collection while one compile builds its long-lived project state.""" - -from __future__ import annotations - -import gc -from collections.abc import Iterator -from contextlib import contextmanager - - -@contextmanager -def paused_cyclic_collection() -> Iterator[None]: - """Skip full-heap cycle scans of long-lived compile state, then restore the prior state.""" - - was_enabled: bool = gc.isenabled() - gc.disable() - try: - yield - finally: - if was_enabled: - gc.enable() diff --git a/src/sqlbuild/cli/commands/_helpers/compile/dag.py b/src/sqlbuild/cli/commands/_helpers/compile/dag.py deleted file mode 100644 index 26350613ee..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/dag.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Compile command DAG artifact helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands.constants import EMPTY_DAG_PATH - - -def resolve_compile_dag_path(*, project_dir: Path, dag_path: str) -> Path: - """Resolve the optional compile DAG artifact path.""" - - if dag_path == EMPTY_DAG_PATH: - return project_dir / "target" / "sqlbuild_dag.json" - path: Path = Path(dag_path) - if path.is_absolute(): - return path - return project_dir / path diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py deleted file mode 100644 index a1618c6426..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Compile pipeline phases: analysis, manifest, DAG, and artifact writing.""" - -from __future__ import annotations - -import time -from dataclasses import replace -from functools import partial -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.compile.lineage import ( - build_compile_lineage, - compile_analysis_lineage_mode, -) -from sqlbuild.cli.commands._helpers.compile.status import ( - complete_compile_phase, - elapsed_ms, - start_compile_phase, -) -from sqlbuild.cli.commands._helpers.compile.target_writer import ( - static_sql_test_planning_diagnostics, - write_static_compile_target, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.classes.native_artifact_batch import NativeArtifactBatch -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.models import ( - CompileAnalysis, - CompileWriteResult, -) -from sqlbuild.cli.entry.models import CompileProfileFlags -from sqlbuild.cli.output.models import ( - WrittenTarget, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, - CompiledObjectKey, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts -from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.models import ProjectColumnLineage -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.main.selected_graph import ( - build_project_graph_with_analysis_selection, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.rule_engine.classes.early_sql_lint import EarlySqlLint -from sqlbuild.rule_engine.constants import RULE_FIX_AVAILABLE_NOTE -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress -from sqlbuild.rule_engine.main.run_rules import run_rules -from sqlbuild.rule_engine.models import RulesConfig, RulesRunResult -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) - - -def analyze_compile_project( - *, - project_dir: Path, - no_sql_validation: bool, - selected_target: str | None, - lineage_mode: CompileLineageMode, - cli_vars: dict[str, object] | None, - profile_flags: CompileProfileFlags, - analysis_selection: CompileAnalysisSelection, - status: TransientStatusReporter | None, - prepared_artifacts: PreparedCompileArtifacts | None = None, -) -> CompileAnalysis: - """Discover, compile, and validate the project into one analysis result.""" - - with EarlySqlLint( - enabled=not (analysis_selection.select or analysis_selection.exclude) - ) as early_lint: - return _analyze_compile_project( - project_dir=project_dir, - no_sql_validation=no_sql_validation, - selected_target=selected_target, - lineage_mode=lineage_mode, - cli_vars=cli_vars, - profile_flags=profile_flags, - analysis_selection=analysis_selection, - status=status, - early_lint=early_lint, - prepared_artifacts=prepared_artifacts, - ) - - -def _analyze_compile_project( - *, - project_dir: Path, - no_sql_validation: bool, - selected_target: str | None, - lineage_mode: CompileLineageMode, - cli_vars: dict[str, object] | None, - profile_flags: CompileProfileFlags, - analysis_selection: CompileAnalysisSelection, - status: TransientStatusReporter | None, - early_lint: EarlySqlLint, - prepared_artifacts: PreparedCompileArtifacts | None = None, -) -> CompileAnalysis: - select: tuple[str, ...] = analysis_selection.select - exclude: tuple[str, ...] = analysis_selection.exclude - - discover_start: float = time.monotonic() - _ = start_compile_phase(status=status, message="Discovering project...") - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=project_dir, - extract_output_column_locations=False, - sql_analysis_enabled_override=( - False if profile_flags.skip_discovery_sql_analysis else None - ), - ) - reject_unit_test_selectors(select=select, exclude=exclude, discovered_inputs=discovered_inputs) - discover_ms: int = elapsed_ms(discover_start) - _ = complete_compile_phase( - status=status, message=f"Discovered project. ({discover_ms / 1000:.2f}s)" - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project_dir=project_dir, - ) - graph_start: float = time.monotonic() - _ = start_compile_phase(status=status, message="Compiling project graph...") - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - graph: ProjectGraph = build_project_graph_with_analysis_selection( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - skip_column_inference=profile_flags.skip_column_inference, - column_lineage_mode=compile_analysis_lineage_mode(lineage_mode), - cli_vars=cli_vars, - analysis_selection=replace( - analysis_selection, - on_inputs_ready=partial( - early_lint.start, dialect=adapter.sql_analysis_dialect() or "generic" - ), - ), - ) - graph_ms: int = elapsed_ms(graph_start) - _ = complete_compile_phase( - status=status, message=f"Compiled project graph. ({graph_ms / 1000:.2f}s)" - ) - graph_failed: bool = any(diagnostic.is_error for diagnostic in graph.project.diagnostics) - early_lint_wait_ms: int = early_lint.stop() if graph_failed else 0 - lineage_start: float = time.monotonic() - _ = start_compile_phase(status=status, message="Analyzing column lineage...") - lineage: ProjectColumnLineage | None = build_compile_lineage( - graph=graph, - dialect=adapter.sql_analysis_dialect(), - mode=lineage_mode, - ) - lineage_ms: int = elapsed_ms(lineage_start) - _ = complete_compile_phase( - status=status, message=f"Analyzed column lineage. ({lineage_ms / 1000:.2f}s)" - ) - contracts_start: float = time.monotonic() - contracts_cpu_start: float = time.thread_time() - contract_result: ContractValidationResult - if profile_flags.skip_contracts: - contract_result = ContractValidationResult(diagnostics=()) - else: - _ = start_compile_phase(status=status, message="Validating model contracts...") - contract_result = evaluate_model_contracts( - project=graph.project, - dialect=adapter.sql_analysis_dialect(), - ) - contract_ms: int = elapsed_ms(contracts_start) - contract_cpu_ms: int = int((time.thread_time() - contracts_cpu_start) * 1000) - if not profile_flags.skip_contracts: - _ = complete_compile_phase( - status=status, message=f"Validated model contracts. ({contract_ms / 1000:.2f}s)" - ) - selected_keys: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=select, exclude=exclude - ) - rules_result: RulesRunResult = RulesRunResult( - findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0 - ) - core_diagnostics: tuple[CompilerDiagnostic, ...] = ( - *graph.project.diagnostics, - *contract_result.diagnostics, - ) - if not graph_failed: - rules_config: RulesConfig = load_rules_config(project_dir=project_dir) - if prepared_artifacts is not None and not any(item.is_error for item in core_diagnostics): - prepared_artifacts.start( - project=graph.project, - adapter=adapter, - target_dir=project_dir / "target", - stage_artifacts=bool(rules_config.select), - ) - _ = start_compile_phase(status=status, message="Evaluating built-in and custom rules...") - rules_start: float = time.monotonic() - rules_result = run_rules( - graph=graph, - discovered_inputs=discovered_inputs, - config=rules_config, - project_dir=project_dir, - dialect=adapter.sql_analysis_dialect() or "generic", - selected_keys=selected_keys if select or exclude else None, - prepared_sql=early_lint.preparation, - expansion_reuse=early_lint.expansion_reuse, - no_sql_analysis=no_sql_validation, - ) - _ = complete_compile_phase( - status=status, - message=format_rules_progress( - elapsed_seconds=time.monotonic() - rules_start, result=rules_result - ), - ) - early_lint_wait_ms = early_lint.stop() - rule_diagnostics: tuple[CompilerDiagnostic, ...] = tuple( - CompilerDiagnostic( - phase=DiagnosticPhase.RULE, - severity=DiagnosticSeverity.ERROR, - code=fault.code, - message=fault.message, - path=fault.path, - line=fault.line, - column=fault.column, - help=fault.remediation, - notes=(RULE_FIX_AVAILABLE_NOTE,) if fault.fixable else (), - affected_rules=fault.affected_rules, - ) - for fault in rules_result.findings - ) - return CompileAnalysis( - discovered_inputs=discovered_inputs, - adapter=adapter, - graph=graph, - selected_keys=selected_keys, - lineage=lineage, - diagnostics=(*core_diagnostics, *rule_diagnostics), - discover_ms=discover_ms, - graph_ms=graph_ms, - lineage_ms=lineage_ms, - contract_ms=contract_ms, - contract_cpu_ms=contract_cpu_ms, - built_in_rules_ms=rules_result.built_in_ms, - custom_rules_ms=rules_result.custom_ms, - custom_rules_cpu_ms=rules_result.custom_cpu_ms, - early_lint_wait_ms=early_lint_wait_ms, - rule_cache_hits=rules_result.cache_hits, - rule_cache_misses=rules_result.cache_misses, - skipped_type_proof_rules=rules_result.skipped_type_proof_rules, - ) - - -def build_compile_manifest_payload( - *, - manifest: bool, - analysis: CompileAnalysis, - status: TransientStatusReporter | None, -) -> dict[str, object] | None: - """Build the manifest payload when requested.""" - - if not manifest: - return None - from sqlbuild.compiler.manifest.main.build_graph_manifest import build_graph_manifest - - manifest_start: float = time.monotonic() - _ = start_compile_phase(status=status, message="Building manifest...") - manifest_payload: dict[str, object] = build_graph_manifest( - project=analysis.graph.project, - plan_output=None, - project_name=analysis.discovered_inputs.project_config.name, - adapter_type=resolve_effective_adapter_name( - project_config=analysis.discovered_inputs.project_config, - local_config=analysis.discovered_inputs.local_config, - ), - graph=analysis.graph.native, - ) - _ = complete_compile_phase( - status=status, message=f"Built manifest. ({time.monotonic() - manifest_start:.2f}s)" - ) - return manifest_payload - - -def write_compile_dag_artifact( - *, - dag_path: str | None, - project_dir: Path, - analysis: CompileAnalysis, - status: TransientStatusReporter | None, -) -> None: - """Write the DAG JSON artifact when a DAG path is requested.""" - - if dag_path is None: - return - from sqlbuild.cli.commands._helpers.compile.dag import resolve_compile_dag_path - from sqlbuild.compiler.dag.main.build import build_dag_json - from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph - from sqlbuild.compiler.python_nodes.models import PythonNodeGraph - - dag_start: float = time.monotonic() - _ = start_compile_phase(status=status, message="Writing DAG artifact...") - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=analysis.discovered_inputs - ) - resolved_dag_path: Path = resolve_compile_dag_path( - project_dir=project_dir, - dag_path=dag_path, - ) - dag_json: str = build_dag_json( - graph=analysis.graph, - project_name=analysis.discovered_inputs.project_config.name, - python_graph=python_graph, - ) - dag_batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=False) - dag_batch.queue(path=resolved_dag_path, contents=dag_json.encode("utf-8")) - dag_batch.flush() - _ = complete_compile_phase( - status=status, message=f"Wrote DAG artifact. ({time.monotonic() - dag_start:.2f}s)" - ) - - -def compile_sql_test_planning_diagnostics( - *, - project_dir: Path, - analysis: CompileAnalysis, - prepared_artifacts: PreparedCompileArtifacts | None, -) -> tuple[CompilerDiagnostic, ...]: - """Report SQL test planning errors when artifacts are withheld, reusing staged planning.""" - - try: - staged: tuple[CompilerDiagnostic, ...] | None = ( - prepared_artifacts.planning_diagnostics() if prepared_artifacts is not None else None - ) - if staged is not None: - return staged - return static_sql_test_planning_diagnostics( - target_dir=project_dir / "target", - adapter=analysis.adapter, - project=analysis.graph.project, - ) - except (CompileInputError, PlannerInputError) as error: - return ( - CompilerDiagnostic( - phase=DiagnosticPhase.TEST, - severity=DiagnosticSeverity.ERROR, - code=error.code, - message=str(error), - help=error.help, - location=error.location if isinstance(error, SqlTestReferenceError) else None, - ), - ) - - -def write_compile_artifacts( - *, - profile_skip_write: bool, - project_dir: Path, - analysis: CompileAnalysis, - manifest_payload: dict[str, object] | None, - status: TransientStatusReporter | None, - prepared_artifacts: PreparedCompileArtifacts | None = None, -) -> CompileWriteResult: - """Write compiled artifacts to target/ and report the written counts.""" - - write_start: float = time.monotonic() - target_dir: Path = project_dir / "target" - written: WrittenTarget - if profile_skip_write: - written = WrittenTarget( - model_count=0, - seed_count=0, - function_count=0, - audit_count=0, - test_count=0, - target_dir=target_dir, - ) - else: - _ = start_compile_phase(status=status, message="Writing compiled artifacts...") - staged: WrittenTarget | None = ( - prepared_artifacts.publish(target_dir=target_dir, manifest=manifest_payload) - if prepared_artifacts is not None - else None - ) - written = ( - staged - if staged is not None - else write_static_compile_target( - target_dir=target_dir, - adapter=analysis.adapter, - project=analysis.graph.project, - manifest=manifest_payload, - ) - ) - write_ms: int = elapsed_ms(write_start) - if not profile_skip_write: - _ = complete_compile_phase( - status=status, message=f"Wrote compiled artifacts. ({write_ms / 1000:.2f}s)" - ) - return CompileWriteResult(written=written, write_ms=write_ms) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/status.py b/src/sqlbuild/cli/commands/_helpers/compile/status.py deleted file mode 100644 index 67913469a2..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/status.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Interactive compile status helpers.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - - -def elapsed_ms(start: float) -> int: - return int((time.monotonic() - start) * 1000) - - -def start_compile_status(*, json_output: bool, no_color: bool) -> TransientStatusReporter | None: - """Create an interactive-only compile status reporter.""" - - if json_output: - return None - if not hasattr(sys.stdout, "isatty") or not sys.stdout.isatty(): - return None - return TransientStatusReporter( - stream=sys.stdout, - use_color=not no_color, - ) - - -def start_compile_phase(*, status: TransientStatusReporter | None, message: str) -> None: - if status is not None: - status.start(message) - - -def complete_compile_phase(*, status: TransientStatusReporter | None, message: str) -> None: - if status is not None: - status.complete(message=message) diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py index d780189e70..363cc8cbad 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py @@ -106,9 +106,9 @@ def _prepare_graph( ) if cached is not None: return cached, None, parsed_depth + from sqlbuild._native import build_project_graph from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.discovery.main.discover import discover_project_inputs - from sqlbuild.compiler.pipeline.main.graph import build_project_graph discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=project_dir, diff --git a/src/sqlbuild/cli/commands/_helpers/lint/selection.py b/src/sqlbuild/cli/commands/_helpers/lint/selection.py index 1fe414da86..ef0b456211 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/selection.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -13,7 +14,6 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py index f4c9ec2639..20c6166e52 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.cli.commands._helpers.compile.pipeline import analyze_compile_project +from sqlbuild._native import analyze_compile_project from sqlbuild.cli.commands._helpers.compile.target_writer import ( static_sql_test_planning_diagnostics, ) diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index 8ae03adbdc..752fc1e08b 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -7,6 +7,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/src/sqlbuild/cli/commands/main/execution/_freshness.py b/src/sqlbuild/cli/commands/main/execution/_freshness.py index af1df800ba..6713ecbef8 100644 --- a/src/sqlbuild/cli/commands/main/execution/_freshness.py +++ b/src/sqlbuild/cli/commands/main/execution/_freshness.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.observe import ( observe_source_freshness_for_command, @@ -31,7 +32,6 @@ from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py index 5f90a9a9a9..08e25a3c72 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ b/src/sqlbuild/cli/commands/main/inspection/_contract.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import build_compiled_project from sqlbuild.cli.commands._helpers.contract.progress import ( inspect_contracts_with_progress, write_contract_updates_with_progress, @@ -29,7 +30,6 @@ from sqlbuild.compiler.contract_adoption.types import ContractAction from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.main.project_graph import build_project_graph_from_compiled_project from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py deleted file mode 100644 index 6552c8e429..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ /dev/null @@ -1,266 +0,0 @@ -"""CLI compile command entry point.""" - -from __future__ import annotations - -import time -from dataclasses import replace -from pathlib import Path - -from sqlbuild.cli.commands._helpers.compile.collection import paused_cyclic_collection -from sqlbuild.cli.commands._helpers.compile.dag import resolve_compile_dag_path -from sqlbuild.cli.commands._helpers.compile.output import ( - format_compile_json, - format_compile_text, -) -from sqlbuild.cli.commands._helpers.compile.pipeline import ( - analyze_compile_project, - build_compile_manifest_payload, - compile_sql_test_planning_diagnostics, - write_compile_artifacts, - write_compile_dag_artifact, -) -from sqlbuild.cli.commands._helpers.compile.semantic_notice import semantic_coverage_notice -from sqlbuild.cli.commands._helpers.compile.status import elapsed_ms, start_compile_status -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.models import ( - CompileAnalysis, - CompileCommandResult, - CompileWriteResult, -) -from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES -from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings -from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.cli.entry.models import CompileCommandRequest -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic -from sqlbuild.compiler.compile.types import DiagnosticPhase -from sqlbuild.compiler.profiling.main.collect import collect_compile_timings -from sqlbuild.compiler.profiling.models import CompileTimingCollector -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules - - -def run_compile( - *, request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None -) -> int: - """Execute the compile command.""" - - total_start: float = time.monotonic() if reuse_attempt is None else reuse_attempt.started - effective_request: CompileCommandRequest = ( - request if request.project_dir is not None else replace(request, project_dir=Path.cwd()) - ) - attempt: CompileReuseAttempt = ( - CompileReuseAttempt( - outcome=CompileReuseOutcome.BYPASS, - project_dir=Path(effective_request.project_dir or Path.cwd()), - started=total_start, - ) - if reuse_attempt is None - else reuse_attempt - ) - output: RecordedCompileOutput = RecordedCompileOutput() - status: TransientStatusReporter | None = start_compile_status( - json_output=request.json_output, - no_color=request.no_color, - ) - with paused_cyclic_collection(): - try: - with ( - collect_compile_timings() as detailed_timings, - COMPILE_INPUT_READS.recording() as input_reads, - COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, - PreparedCompileArtifacts( - enabled=not request.profile_flags.skip_write - ) as prepared_artifacts, - ): - result: CompileCommandResult = _run_compile_with_status( - request=effective_request, - total_start=total_start, - status=status, - detailed_timings=detailed_timings, - prepared_artifacts=prepared_artifacts, - output=output, - reuse_timings_ms=compile_reuse_timings(attempt=attempt), - ) - finally: - if status is not None: - status.close() - try: - write_reusable_compile( - attempt=attempt, - output=output, - exit_code=result.exit_code, - input_reads=input_reads, - artifact_writes=artifact_writes, - compile_cache_enabled=result.compile_cache_enabled, - artifacts_written=result.artifacts_written, - dag_artifact_path=result.dag_artifact_path, - json_output=request.json_output, - ) - finally: - output.print_stdout() - return result.exit_code - - -def _run_compile_with_status( - *, - request: CompileCommandRequest, - total_start: float, - status: TransientStatusReporter | None, - detailed_timings: CompileTimingCollector, - prepared_artifacts: PreparedCompileArtifacts, - output: RecordedCompileOutput, - reuse_timings_ms: dict[str, int], -) -> CompileCommandResult: - """Execute compile after the optional interactive status reporter is initialized.""" - - project_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() - json_output: bool = request.json_output - manifest: bool = request.manifest - no_color: bool = request.no_color - lineage_mode: CompileLineageMode = request.lineage_mode - prepared_artifacts.remove_abandoned_staging(target_dir=project_dir / "target") - analysis: CompileAnalysis = analyze_compile_project( - project_dir=project_dir, - no_sql_validation=request.no_sql_validation, - selected_target=request.selected_target, - lineage_mode=lineage_mode, - cli_vars=request.cli_vars, - profile_flags=request.profile_flags, - analysis_selection=CompileAnalysisSelection( - select=request.select, exclude=request.exclude, no_cache=request.no_cache - ), - status=status, - prepared_artifacts=prepared_artifacts, - ) - rules_failed: bool = any( - diagnostic.is_error and diagnostic.phase is DiagnosticPhase.RULE - for diagnostic in analysis.diagnostics - ) - manifest_payload: dict[str, object] | None = build_compile_manifest_payload( - manifest=manifest and not rules_failed, - analysis=analysis, - status=status, - ) - write_compile_dag_artifact( - dag_path=None if rules_failed else request.dag_path, - project_dir=project_dir, - analysis=analysis, - status=status, - ) - write_result: CompileWriteResult = write_compile_artifacts( - profile_skip_write=request.profile_flags.skip_write or rules_failed, - project_dir=project_dir, - analysis=analysis, - manifest_payload=manifest_payload, - status=status, - prepared_artifacts=prepared_artifacts, - ) - timings_ms: dict[str, int] = { - "discover_ms": analysis.discover_ms, - "graph_ms": analysis.graph_ms, - "lineage_ms": analysis.lineage_ms, - "contracts_ms": analysis.contract_ms, - "contracts_cpu_ms": analysis.contract_cpu_ms, - "built_in_rules_ms": analysis.built_in_rules_ms, - "custom_rules_ms": analysis.custom_rules_ms, - "custom_rules_cpu_ms": analysis.custom_rules_cpu_ms, - "early_lint_wait_ms": analysis.early_lint_wait_ms, - "rule_cache_hits": analysis.rule_cache_hits, - "rule_cache_misses": analysis.rule_cache_misses, - "write_ms": write_result.write_ms, - **detailed_timings.as_milliseconds(), - **reuse_timings_ms, - "total_ms": elapsed_ms(total_start), - } - withheld_test_diagnostics: tuple[CompilerDiagnostic, ...] = ( - compile_sql_test_planning_diagnostics( - project_dir=project_dir, analysis=analysis, prepared_artifacts=prepared_artifacts - ) - if rules_failed and not request.profile_flags.skip_write - else () - ) - diagnostics: tuple[CompilerDiagnostic, ...] = ( - *analysis.diagnostics, - *write_result.written.diagnostics, - *withheld_test_diagnostics, - ) - exit_code: int = 1 if any(diagnostic.is_error for diagnostic in diagnostics) else 0 - - if status is not None: - status.close() - skipped_rules_note: str | None = format_skipped_type_proof_rules( - codes=analysis.skipped_type_proof_rules - ) - if skipped_rules_note is not None: - output.print_stderr_line(f"note: {skipped_rules_note}") - result: CompileCommandResult = _command_result( - request=request, - project_dir=project_dir, - exit_code=exit_code, - rules_failed=rules_failed, - analysis=analysis, - ) - - if json_output: - notice: str | None = semantic_coverage_notice( - project=analysis.graph.project, - selected_keys=analysis.selected_keys, - sql_validation_enabled=not request.no_sql_validation, - ) - if notice: - output.print_stderr_line(notice) - output.record_stdout( - format_compile_json( - graph=analysis.graph, - written=write_result.written, - manifest=manifest, - timings_ms=timings_ms, - lineage=analysis.lineage, - lineage_mode=lineage_mode, - diagnostics=diagnostics, - selected_keys=analysis.selected_keys, - sql_validation_enabled=not request.no_sql_validation, - ) - ) - return result - - output.record_stdout( - format_compile_text( - graph=analysis.graph, - written=write_result.written, - manifest=manifest, - lineage=analysis.lineage, - lineage_mode=lineage_mode, - diagnostics=diagnostics, - selected_keys=analysis.selected_keys, - use_color=(not no_color) and supports_color(), - sql_validation_enabled=not request.no_sql_validation, - ) - ) - return result - - -def _command_result( - *, - request: CompileCommandRequest, - project_dir: Path, - exit_code: int, - rules_failed: bool, - analysis: CompileAnalysis, -) -> CompileCommandResult: - return CompileCommandResult( - exit_code=exit_code, - compile_cache_enabled=analysis.graph.project.compile_cache_dir is not None, - artifacts_written=not (rules_failed or request.profile_flags.skip_write), - dag_artifact_path=( - None - if rules_failed or request.dag_path is None - else resolve_compile_dag_path(project_dir=project_dir, dag_path=request.dag_path) - ), - ) diff --git a/src/sqlbuild/cli/commands/main/project/_dag.py b/src/sqlbuild/cli/commands/main/project/_dag.py index 7a910a01cb..f770c9a6da 100644 --- a/src/sqlbuild/cli/commands/main/project/_dag.py +++ b/src/sqlbuild/cli/commands/main/project/_dag.py @@ -5,6 +5,7 @@ import json from pathlib import Path +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, @@ -13,7 +14,6 @@ from sqlbuild.compiler.dag.main.build import build_dag_json from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph from sqlbuild.compiler.python_nodes.models import PythonNodeGraph diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py index b9acae3e44..8dda17e2f1 100644 --- a/src/sqlbuild/cli/compile/main/run.py +++ b/src/sqlbuild/cli/compile/main/run.py @@ -14,9 +14,9 @@ def run_compile(request: CompileCommandRequest) -> int: attempt: CompileReuseAttempt = attempt_compile_reuse(request=_reuse_request(request=request)) if attempt.outcome is CompileReuseOutcome.HIT and attempt.exit_code is not None: return attempt.exit_code - from sqlbuild.cli.commands.main.project._compile import run_compile as run_full_compile + from sqlbuild._native import run_compile_command - return run_full_compile(request=request, reuse_attempt=attempt) + return run_compile_command(request, attempt) def _reuse_request(*, request: CompileCommandRequest) -> CompileReuseRequest: diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index dc8357891e..4fde249c68 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -18,16 +18,6 @@ from sqlbuild.compiler.planner.models import NativeSqlTestArtifact -@dataclass(frozen=True) -class CompileCommandResult: - """Outcome of one full compile that decides whether it can be stored for reuse.""" - - exit_code: int - compile_cache_enabled: bool - artifacts_written: bool - dag_artifact_path: Path | None - - @dataclass(frozen=True) class CompileAnalysis: """Compiled project analysis shared by compile output phases.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py index e700fb00fa..130efa4093 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.analysis.validation import hook_sql_statements +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, ) @@ -54,7 +54,7 @@ def _syntax_checks( placeholders: dict[str, str] | None = model_placeholders(model_input) statements: list[str] = [] for hook_name in _HOOK_NAMES: - statements.extend(hook_sql_statements(model_input.config.values.get(hook_name))) + statements.extend(_native.hook_sql_statements(model_input.config.values.get(hook_name))) if model_name not in analysis_succeeded: statements.append( cursor_intrinsics_analysis_sql( diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py index a256b85574..6265087cef 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py @@ -1,145 +1,18 @@ -"""Polyglot-backed SQL syntax validation.""" +"""Help for SQL the parser rejects.""" from __future__ import annotations -from pathlib import Path - -import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import SQL_ANALYSIS_OPT_OUT_ENTRY -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.constants import ( PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, SQL_ANALYSIS_CONFIG_KEY, ) -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help -def validate_sql_syntax( - *, - query_sql: str, - model_name: str, - file_path: Path, - placeholders: dict[str, str] | None = None, - dialect: str | None = None, -) -> None: - """Validate that the model query SQL is parseable by Polyglot.""" - - error_message: str | None = _native.sql_syntax_error(query_sql, placeholders, dialect) - if error_message is None: - return - raise CompileInputError( - f"SQL syntax error in model '{model_name}' ({file_path}): {error_message}", - help=sql_analysis_opt_out_help(model_owned=True), - ) from None - - -def validate_hook_sql_syntax( - *, - value: object, - hook_name: str, - model_name: str, - file_path: Path, - placeholders: dict[str, str] | None = None, - hook_label: str | None = None, - dialect: str | None = None, -) -> None: - """Validate hook SQL strings recursively inside supported hook container shapes.""" - - if isinstance(value, str): - effective_label: str = hook_label or hook_name - _validate_hook_sql_with_message( - query_sql=value, - error_prefix=( - f"Polyglot could not parse model '{model_name}' {effective_label} ({file_path})" - ), - placeholders=placeholders, - dialect=dialect, - ) - return - if isinstance(value, SqlHookEntry): - entry_label: str = ( - f'{hook_name} sql("{value.name}")' - if value.name is not None - else f'{hook_name} inline_sql("...")' - ) - validate_hook_sql_syntax( - value=value.statement, - hook_name=hook_name, - model_name=model_name, - file_path=value.relative_path or file_path, - placeholders=placeholders, - hook_label=hook_label or entry_label, - dialect=dialect, - ) - return - if isinstance(value, PythonHookEntry): - return - if isinstance(value, list | tuple): - hook_index: int - item: object - for hook_index, item in enumerate(value): - item_label: str | None = None - if isinstance(item, SqlHookEntry): - item_label = ( - f'{hook_name}[{hook_index}] sql("{item.name}")' - if item.name is not None - else f'{hook_name}[{hook_index}] inline_sql("...")' - ) - elif isinstance(item, str): - item_label = f'{hook_name}[{hook_index}] inline_sql("...")' - validate_hook_sql_syntax( - value=item, - hook_name=hook_name, - model_name=model_name, - file_path=file_path, - placeholders=placeholders, - hook_label=item_label, - dialect=dialect, - ) - - -def hook_sql_statements(value: object) -> tuple[str, ...]: - """Return the hook SQL strings `validate_hook_sql_syntax` validates, in its order.""" - - if isinstance(value, str): - return (value,) - if isinstance(value, SqlHookEntry): - return hook_sql_statements(value.statement) - statements: list[str] = [] - if isinstance(value, list | tuple): - for item in value: - statements.extend(hook_sql_statements(item)) - return tuple(statements) - - -def _validate_hook_sql_with_message( - *, - query_sql: str, - error_prefix: str, - placeholders: dict[str, str] | None, - dialect: str | None, -) -> None: - """Validate a complete hook execution payload with Polyglot.""" - - error_message: str | None = _native.sql_syntax_error(query_sql, placeholders, dialect) - if error_message is not None: - _raise_sql_validation_error( - error_prefix=error_prefix, error_message=error_message, model_owned=True - ) - - -def _raise_sql_validation_error( - *, error_prefix: str, error_message: str, model_owned: bool -) -> None: - raise CompileInputError( - f"{error_prefix}: {error_message}", help=sql_analysis_opt_out_help(model_owned=model_owned) - ) from None - - def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: """Help for SQL the parser rejects: the exact header, project and single-run opt-outs.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index f077f6f686..052823009c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -24,10 +24,6 @@ from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( model_placeholders as _model_placeholders, ) -from sqlbuild.compiler.compile._helpers.analysis.validation import ( - validate_hook_sql_syntax, - validate_sql_syntax, -) from sqlbuild.compiler.compile._helpers.assembly.hand_built_models import ( hand_built_compiled_model, ) @@ -35,13 +31,6 @@ known_declared_types, known_function_names, ) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - build_column_nullability_by_table, - build_complete_binding_schemas, -) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - build_declared_column_types as _build_column_types_by_table, -) from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( @@ -141,10 +130,12 @@ def assemble_compiled_project( sql_analysis_enabled: bool = ( inputs.effective_settings.sql_analysis and not skip_column_inference ) - column_nullability_by_table: dict[str, dict[str, InferredNullability]] = ( - build_column_nullability_by_table(inputs) + column_types_by_table: dict[str, dict[str, str]] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + complete_binding_schemas: dict[str, dict[str, str]] + column_types_by_table, column_nullability_by_table, complete_binding_schemas = ( + _native.declared_relations(inputs) ) - column_types_by_table: dict[str, dict[str, str]] = _build_column_types_by_table(inputs) dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( _build_dynamic_families_by_table(inputs) ) @@ -170,7 +161,6 @@ def assemble_compiled_project( }, }, ) - complete_binding_schemas: dict[str, dict[str, str]] = build_complete_binding_schemas(inputs) binding_catalog: Any = create_binding_catalog( dialect=profile.sql_analysis_dialect or "generic", quoted_ignore_case=profile.quoted_identifiers_ignore_case, @@ -438,13 +428,11 @@ def _assemble_compiled_model( placeholders: dict[str, str] | None = _model_placeholders(model_input) if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: for hook_name in ("pre_hooks", "post_hooks"): - validate_hook_sql_syntax( - value=model_input.config.values.get(hook_name), - hook_name=hook_name, - model_name=model_name, - file_path=model_input.model_file.file_path, - placeholders=placeholders, - dialect=profile.sql_analysis_dialect, + _native.validate_hook_sql_syntax( + model_input.config.values.get(hook_name), + hook_name, + (model_name, model_input.model_file.file_path), + (placeholders, profile.sql_analysis_dialect), ) session_analysis: tuple[_native.NativeModelAnalysisSession, int] | None = None fallback: list[ColumnRow] | None = None @@ -461,12 +449,10 @@ def _assemble_compiled_model( ) if model_name not in analyses.succeeded: if model_input.sql_validation_enabled and not syntax_validated: - validate_sql_syntax( - query_sql=analysis_query_sql, - model_name=model_name, - file_path=model_input.model_file.file_path, - placeholders=placeholders, - dialect=profile.sql_analysis_dialect, + _native.validate_sql_syntax( + analysis_query_sql, + (model_name, model_input.model_file.file_path), + (placeholders, profile.sql_analysis_dialect), ) fallback = infer_native_query_column_rows( queries=( @@ -478,12 +464,10 @@ def _assemble_compiled_model( column_nullability_by_table=column_nullability_by_table, )[0][1] elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: - validate_sql_syntax( - query_sql=analysis_query_sql, - model_name=model_name, - file_path=model_input.model_file.file_path, - placeholders=placeholders, - dialect=profile.sql_analysis_dialect, + _native.validate_sql_syntax( + analysis_query_sql, + (model_name, model_input.model_file.file_path), + (placeholders, profile.sql_analysis_dialect), ) macro_deps: tuple[str, ...] = model_input.macro_deps or find_macro_call_names( model_input.macro_source_sql diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index ed20062ef3..77c5a9d064 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -10,17 +10,14 @@ infer_native_expression_source_shapes, ) from sqlbuild.compiler.compile._helpers.analysis.reference_names import table_function_analysis_name -from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.models import ( CompileSqlReference, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompileInputsView, ModelInput, ) -from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.constants import ( SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, SNAPSHOT_DEFAULT_VALID_TO_COLUMN, @@ -33,9 +30,6 @@ from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SchemaColumn, - SourceColumnEntry, SourceEntry, ) @@ -120,120 +114,6 @@ def inferred_binding_shape( ) -def build_declared_column_types(inputs: CompileInputsView) -> dict[str, dict[str, str]]: - """Collect declared type hints independently of closed-schema authority.""" - facts: dict[str, dict[str, str]] = {} - for model_input in inputs.model_inputs: - if model_input.schema_entry is not None: - facts[model_input.schema_entry.name] = { - column.name: column.type or "UNKNOWN" for column in model_input.schema_entry.columns - } - for seed_input in inputs.seed_inputs: - facts[seed_input.schema_entry.name] = { - column.name: column.type or "UNKNOWN" for column in seed_input.schema_entry.columns - } - for source_input in inputs.source_inputs: - facts[source_input.source_entry.name] = { - column.name: column.type or "UNKNOWN" for column in source_input.source_entry.columns - } - for function_input in inputs.sql_function_inputs: - if function_input.return_columns: - facts[table_function_analysis_name(function_input.name)] = { - column.name: column.type for column in function_input.return_columns - } - return facts - - -def build_column_nullability_by_table( - inputs: CompileInputsView, -) -> dict[str, dict[str, InferredNullability]]: - """Return what each model, seed, source and table function declares about column nulls.""" - - facts: dict[str, dict[str, InferredNullability]] = {} - for model_input in inputs.model_inputs: - if model_input.schema_entry is None: - continue - facts[model_input.schema_entry.name] = _schema_column_nullability( - model_input.schema_entry.columns - ) - for seed_input in inputs.seed_inputs: - facts[seed_input.schema_entry.name] = _schema_column_nullability( - seed_input.schema_entry.columns - ) - for source_input in inputs.source_inputs: - facts[source_input.source_entry.name] = _source_column_nullability( - source_input.source_entry.columns - ) - for function_input in inputs.sql_function_inputs: - if not function_input.return_columns: - continue - facts[table_function_analysis_name(function_input.name)] = { - column.name: InferredNullability.UNKNOWN for column in function_input.return_columns - } - return facts - - -def _schema_column_nullability( - columns: tuple[SchemaColumn, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _source_column_nullability( - columns: tuple[SourceColumnEntry, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _declared_column_nullability( - *, - nullable: bool | None, - audits: tuple[SchemaAuditInstance, ...], -) -> InferredNullability: - if nullable is False: - return InferredNullability.NON_NULL - if any(audit.definition_name == NOT_NULL_AUDIT_NAME for audit in audits): - return InferredNullability.NON_NULL - return InferredNullability.UNKNOWN - - -def build_complete_binding_schemas(inputs: CompileInputsView) -> dict[str, dict[str, str]]: - """Collect authoritative declared interfaces before SQL inference.""" - schemas: dict[str, dict[str, str]] = {} - for model in inputs.model_inputs: - if ( - model.config.values.get("contract") == ContractPolicy.ENFORCED - and model.schema_entry is not None - and model.schema_entry.columns - and not model.schema_entry.dynamic_columns - ): - schemas[model.model_file.file_path.stem] = { - column.name: column.type or "UNKNOWN" for column in model.schema_entry.columns - } - for source in inputs.source_inputs: - if source.source_entry.contract == ContractPolicy.ENFORCED and source.source_entry.columns: - schemas[source.source_entry.name] = { - column.name: column.type or "UNKNOWN" for column in source.source_entry.columns - } - for seed in inputs.seed_inputs: - if seed.schema_entry.columns: - schemas[seed.schema_entry.name] = { - column.name: column.type or "UNKNOWN" for column in seed.schema_entry.columns - } - for function in inputs.sql_function_inputs: - if function.return_columns: - schemas[table_function_analysis_name(function.name)] = { - column.name: column.type for column in function.return_columns - } - return schemas - - def semantic_shapes( *, project: CompiledProjectView, @@ -305,18 +185,3 @@ def semantic_shapes( break pending = remaining return shapes - - -def referenced_model_names( - *, model_input: ModelInput, available_names: frozenset[str] | None = None -) -> tuple[str, ...]: - return tuple( - sorted( - { - reference.ref_name - for reference in model_input.references - if reference.ref_kind == SqlReferenceKind.REF - and (available_names is None or reference.ref_name in available_names) - } - ) - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 3fb7baa190..6874fa1934 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -7,13 +7,10 @@ from pathlib import Path from types import CodeType +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.render.declarations import ( build_declaration_scope_resolver, ) -from sqlbuild.compiler.compile._helpers.sql_tests.extraction_errors import ( - validate_authored_scenario_cte_names, - validate_authored_test_cte_names, -) from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -22,7 +19,6 @@ LoadedMacro, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index @@ -41,11 +37,10 @@ def build_declaration_scope( scan_cache: SqlTestScanCache = SqlTestScanCache(cache_dir=compile_cache_dir) - validate_authored_test_cte_names( - test_files=discovered_inputs.test_files, syntax=sql_lexical_syntax - ) - validate_authored_scenario_cte_names( - scenario_files=discovered_inputs.scenario_files, syntax=sql_lexical_syntax + _native.validate_authored_sql_test_ctes( + discovered_inputs.test_files, + discovered_inputs.scenario_files, + sql_lexical_syntax.native_mapping, ) return _build_native_declaration_scope( discovered_inputs=discovered_inputs, @@ -64,7 +59,7 @@ def _build_native_declaration_scope( ) -> DeclarationScopeBuild: """Build and validate the scope index natively, with its relationship grants and lookup.""" - native: NativeScopeIndex = open_native_scope_index( + native: _native.NativeScopeIndex = open_native_scope_index( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) try: @@ -82,7 +77,7 @@ def _build_native_declaration_scope( if faults: raise CompileInputError(faults[0][1]) index: ScopeIndex = native.index_with_relationships() - lookup: ScopeLookup = native.lookup(index=index) + lookup: ScopeLookup = _native.build_scope_lookup(index) return DeclarationScopeBuild( loaded_macros=loaded_macros, index=index, diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index 856d796075..a976798ca4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -42,18 +42,6 @@ def call_site_sql_references( ) -def merge_call_site_references( - *, - references: tuple[CompileSqlReference, ...], - argument_references: tuple[CompileSqlReference, ...], -) -> tuple[CompileSqlReference, ...]: - """Add typed macro arguments to the references found in expanded SQL, once each.""" - - if not argument_references: - return references - return tuple(dict.fromkeys((*references, *argument_references))) - - def typed_reference_kinds() -> dict[str, SqlResourceRefKind]: """Each typed reference function, such as `__ref`, and the resource kind it names.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index 6151d667e1..e342978177 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -62,7 +62,7 @@ ResourceIdentity, UsageRecord, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind, UsageKind +from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, UsageKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import RELATION_RENDERER, SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -70,31 +70,18 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_STAR_IMPORT_NAME: str = "*" _SYNTHETIC_PACKAGE_PREFIX: str = "_sqlbuild_project_macros_" _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() @dataclass(frozen=True) -class _MacroModuleAnalysis: +class _MacroModuleLoad: file: DiscoveredMacroFile - name: str tree: ast.Module - exports: tuple[str, ...] - dependencies: dict[str, tuple[DeclarationIdentity, ...]] module_dependencies: tuple[str, ...] -@dataclass(frozen=True) -class _MacroAnalysisInputs: - files: dict[str, DiscoveredMacroFile] - trees: dict[str, ast.Module] - exports: dict[str, frozenset[str]] - module_names: frozenset[str] - package_names: frozenset[str] - - class _MacroDeclarationValues(Mapping[str, object]): def __init__( self, @@ -179,28 +166,25 @@ def load_project_macros(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[st if not macro_files: return {} - try: - analyses: dict[str, _MacroModuleAnalysis] = _analyze_project_macro_modules( - macro_files=macro_files - ) - except CompileInputError: - fault_inventory: StaticMacroInventory = _inventory_project_macros_tolerantly( - macro_files=macro_files - ) - raise CompileInputError(fault_inventory.faults[0].message) from None - inventory: StaticMacroInventory = _inventory_from_analyses( - macro_files=macro_files, analyses=analyses + trees: tuple[ast.Module | str, ...] = tuple( + _parse_macro_module(macro_file=macro_file) for macro_file in macro_files ) - if inventory.faults: + inventory: StaticMacroInventory + loads: dict[str, _MacroModuleLoad] | None + inventory, loads = _native_macro_inventory(macro_files=macro_files, trees=trees) + if inventory.faults or loads is None: raise CompileInputError(inventory.faults[0].message) - modules: dict[str, ModuleType] = _load_macro_modules(analyses=analyses) + modules: dict[str, ModuleType] = _load_macro_modules(analyses=loads) export_dependencies: dict[tuple[Path, str], tuple[DeclarationIdentity, ...]] = { (item.relative_path, item.name): item.dependencies for item in inventory.exports } loaded_macros: dict[str, LoadedMacro] = {} - for macro_file in macro_files: - module: ModuleType = modules[_macro_module_name(macro_file)] + module_name: str + load: _MacroModuleLoad + for module_name, load in loads.items(): + macro_file: DiscoveredMacroFile = load.file + module: ModuleType = modules[module_name] attribute_name: str for attribute_name in dir(module): if attribute_name.startswith("_"): @@ -233,709 +217,85 @@ def load_project_macros(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[st return loaded_macros -def _macro_inventory_fault( - *, error: CompileInputError, macro_files: tuple[DiscoveredMacroFile, ...] -) -> StaticMacroFault: - relative_path: Path = next( - (item.relative_path for item in macro_files if item.relative_path.as_posix() in str(error)), - macro_files[0].relative_path if macro_files else Path("macros"), - ) - return StaticMacroFault(relative_path=relative_path, message=str(error)) - - -def _inventory_project_macros_tolerantly( - *, macro_files: tuple[DiscoveredMacroFile, ...] -) -> StaticMacroInventory: - remaining: list[DiscoveredMacroFile] = list(macro_files) - faults: list[StaticMacroFault] = [] - while remaining: - current: tuple[DiscoveredMacroFile, ...] = tuple(remaining) - try: - analyses: dict[str, _MacroModuleAnalysis] = _analyze_project_macro_modules( - macro_files=current - ) - except CompileInputError as error: - collisions: tuple[tuple[str, tuple[DiscoveredMacroFile, ...]], ...] = ( - _macro_module_path_collisions(macro_files=current) - ) - if collisions: - collided_paths: set[Path] = set() - for module_name, files in collisions: - paths: str = " and ".join(str(file.relative_path) for file in files) - for file in files: - collided_paths.add(file.relative_path) - faults.append( - StaticMacroFault( - relative_path=file.relative_path, - message=( - f"Macro module path collision for '{module_name}': {paths}" - ), - ) - ) - remaining = [item for item in remaining if item.relative_path not in collided_paths] - continue - fault: StaticMacroFault = _macro_inventory_fault(error=error, macro_files=current) - faults.append(fault) - remaining = [item for item in remaining if item.relative_path != fault.relative_path] - continue - inventory: StaticMacroInventory = _inventory_from_analyses( - macro_files=current, analyses=analyses - ) - return StaticMacroInventory( - exports=inventory.exports, - faults=_deduplicated_macro_faults((*faults, *inventory.faults)), - ) - return StaticMacroInventory(faults=_deduplicated_macro_faults(tuple(faults))) - - -def _macro_module_path_collisions( - *, macro_files: tuple[DiscoveredMacroFile, ...] -) -> tuple[tuple[str, tuple[DiscoveredMacroFile, ...]], ...]: - grouped: dict[str, list[DiscoveredMacroFile]] = {} - for file in macro_files: - grouped.setdefault(_macro_module_name(file), []).append(file) - collisions: list[tuple[str, tuple[DiscoveredMacroFile, ...]]] = [] - for module_name in sorted(grouped): - files: list[DiscoveredMacroFile] = grouped[module_name] - if len(files) > 1: - collisions.append((module_name, tuple(files))) - return tuple(collisions) - - -def _inventory_from_analyses( - *, - macro_files: tuple[DiscoveredMacroFile, ...], - analyses: dict[str, _MacroModuleAnalysis], -) -> StaticMacroInventory: - exports: dict[str, StaticMacroExport] = {} - collided_names: set[str] = set() - faults: list[StaticMacroFault] = [] - for macro_file in macro_files: - analysis: _MacroModuleAnalysis = analyses[_macro_module_name(macro_file)] - for statement in analysis.tree.body: - if not isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef): - continue - if statement.name.startswith("_"): - continue - item: StaticMacroExport = _static_macro_export( - macro_file=macro_file, analysis=analysis, statement=statement - ) - existing: StaticMacroExport | None = exports.pop(item.name, None) - if existing is not None: - collided_names.add(item.name) - faults.append( - StaticMacroFault( - relative_path=item.relative_path, - message=( - f"Macro name collision for '{item.name}': " - f"{existing.relative_path} and {item.relative_path}" - ), - ) - ) - elif item.name in collided_names: - faults.append( - StaticMacroFault( - relative_path=item.relative_path, - message=f"Macro name collision for '{item.name}' at {item.relative_path}", - ) - ) - else: - exports[item.name] = item - return StaticMacroInventory( - exports=tuple( - sorted(exports.values(), key=lambda item: (item.name, item.relative_path.as_posix())) - ), - faults=tuple(faults), - ) - - def _static_macro_export( *, - macro_file: DiscoveredMacroFile, - analysis: _MacroModuleAnalysis, - statement: ast.FunctionDef | ast.AsyncFunctionDef, + row: tuple[str, int, list[str], int, str, list[str]], + macro_files: tuple[DiscoveredMacroFile, ...], ) -> StaticMacroExport: - arguments: ast.arguments = statement.args + name, file_index, parameters, line, source_digest, dependencies = row return StaticMacroExport( - name=statement.name, - relative_path=macro_file.relative_path, - parameters=tuple(argument.arg for argument in (*arguments.posonlyargs, *arguments.args)) - + ((arguments.vararg.arg,) if arguments.vararg is not None else ()) - + tuple(argument.arg for argument in arguments.kwonlyargs) - + ((arguments.kwarg.arg,) if arguments.kwarg is not None else ()), - line=statement.lineno, - source_digest=hashlib.sha256(macro_file.contents.encode()).hexdigest(), - dependencies=analysis.dependencies[statement.name], + name=name, + relative_path=macro_files[file_index].relative_path, + parameters=tuple(parameters), + line=line, + source_digest=source_digest, + dependencies=tuple( + DeclarationIdentity(DeclarationKind.MACRO, dependency) for dependency in dependencies + ), ) -def _deduplicated_macro_faults( - faults: tuple[StaticMacroFault, ...], -) -> tuple[StaticMacroFault, ...]: - return tuple(dict.fromkeys(faults)) +def _parse_macro_module(*, macro_file: DiscoveredMacroFile) -> ast.Module | str: + """CPython's tree of a macro module, or the message of its syntax error.""" - -def _macro_module_name(macro_file: DiscoveredMacroFile) -> str: - return ".".join(macro_file.relative_path.with_suffix("").parts) - - -def _parse_macro_module(*, macro_file: DiscoveredMacroFile) -> ast.Module: try: - module_ast: ast.Module = ast.parse(macro_file.contents, filename=str(macro_file.file_path)) + return ast.parse(macro_file.contents, filename=str(macro_file.file_path)) except SyntaxError as error: - raise CompileInputError( - f"Failed to parse macros from '{macro_file.file_path}': {error}" - ) from error - - for statement in module_ast.body: - public_declaration_name: str | None = _public_non_function_declaration_name(statement) - if public_declaration_name is not None: - raise CompileInputError( - f"Macro module '{macro_file.relative_path}' declaration " - f"'{public_declaration_name}' must be underscore-private" - ) - return module_ast - - -def _analyze_project_macro_modules( - *, macro_files: tuple[DiscoveredMacroFile, ...] -) -> dict[str, _MacroModuleAnalysis]: - """Analyze every authored module and dependency edge before executing any of them.""" - - inputs: _MacroAnalysisInputs = _macro_analysis_inputs(macro_files=macro_files) - analyses: dict[str, _MacroModuleAnalysis] = { - module_name: _analyze_macro_module(module_name=module_name, inputs=inputs) - for module_name in inputs.trees - } - return _validated_macro_analyses(analyses=analyses) - - -def _analyze_macro_module( - *, module_name: str, inputs: _MacroAnalysisInputs -) -> _MacroModuleAnalysis: - tree: ast.Module = inputs.trees[module_name] - file: DiscoveredMacroFile = inputs.files[module_name] - _validate_no_nested_project_imports( - tree=tree, - module_name=module_name, - file=file, - module_names=inputs.module_names, - package_names=inputs.package_names, - ) - imported, module_dependencies = _analyze_macro_imports( - tree=tree, module_name=module_name, inputs=inputs - ) - functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = { - statement.name: statement - for statement in tree.body - if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef) - } - shadowed_imports: set[str] = set(functions).intersection(imported) - if shadowed_imports: - shadowed_name: str = sorted(shadowed_imports)[0] - raise CompileInputError( - f"Macro module '{file.relative_path}' defines function '{shadowed_name}' over an " - "imported macro binding; imported macros must not be replaced" - ) - direct_dependencies, helper_calls, runtime_node_ids = _analyze_macro_calls( - functions=functions, imported=imported, file=file - ) - _validate_local_call_cycles( - exports=inputs.exports[module_name], helper_calls=helper_calls, file=file - ) - _validate_call_use_locations( - tree=tree, - imported=imported, - local_functions=frozenset(functions), - runtime_node_ids=runtime_node_ids, - file=file, - ) - return _MacroModuleAnalysis( - file=file, - name=module_name, - tree=tree, - exports=tuple(sorted(inputs.exports[module_name])), - dependencies=_export_dependencies( - exports=inputs.exports[module_name], - direct_dependencies=direct_dependencies, - helper_calls=helper_calls, - ), - module_dependencies=module_dependencies, - ) - - -def _macro_analysis_inputs(*, macro_files: tuple[DiscoveredMacroFile, ...]) -> _MacroAnalysisInputs: - files: dict[str, DiscoveredMacroFile] = {} - for item in macro_files: - module_name: str = _macro_module_name(item) - existing: DiscoveredMacroFile | None = files.get(module_name) - if existing is not None: - raise CompileInputError( - f"Macro module path collision for '{module_name}': " - f"{existing.relative_path} and {item.relative_path}" - ) - files[module_name] = item - trees: dict[str, ast.Module] = { - name: _parse_macro_module(macro_file=item) for name, item in files.items() - } - exports: dict[str, frozenset[str]] = { - name: _module_exports(tree=tree) for name, tree in trees.items() - } - return _MacroAnalysisInputs( - files=files, - trees=trees, - exports=exports, - module_names=frozenset(files), - package_names=frozenset(_macro_package_name(file) for file in files.values()), - ) - - -def _module_exports(*, tree: ast.Module) -> frozenset[str]: - exports: set[str] = set() - for statement in tree.body: - if isinstance( - statement, ast.FunctionDef | ast.AsyncFunctionDef - ) and not statement.name.startswith("_"): - exports.add(statement.name) - return frozenset(exports) - - -def _analyze_macro_imports( - *, - tree: ast.Module, - module_name: str, - inputs: _MacroAnalysisInputs, -) -> tuple[dict[str, DeclarationIdentity], tuple[str, ...]]: - imported: dict[str, DeclarationIdentity] = {} - module_dependencies: list[str] = [] - file: DiscoveredMacroFile = inputs.files[module_name] - for statement in tree.body: - if isinstance(statement, ast.Import): - local_names: list[str] = [ - name - for name in _imported_module_names( - statement=statement, current_module_name=module_name - ) - if _is_project_macro_import( - imported_name=name, - module_names=inputs.module_names, - package_names=inputs.package_names, - ) - ] - if local_names: - raise CompileInputError( - f"Macro module '{file.relative_path}' must use 'from ... import ...' for " - "project macro imports; module imports are not supported" - ) - continue - if not isinstance(statement, ast.ImportFrom): - continue - resolved_module: str = _resolve_import_from_module( - statement=statement, current_module_name=module_name - ) - if resolved_module not in inputs.files: - if _is_project_macro_import( - imported_name=resolved_module, - module_names=inputs.module_names, - package_names=inputs.package_names, - ): - raise CompileInputError( - f"Macro module '{file.relative_path}' imports project package " - f"'{resolved_module}' instead of a macro module" - ) - continue - module_dependencies.append(resolved_module) - imported = _record_imported_macro_bindings( - statement=statement, - importer=file, - imported_file=inputs.files[resolved_module], - exported_names=inputs.exports[resolved_module], - imported=imported, - resolved_module=resolved_module, - ) - return imported, tuple(dict.fromkeys(module_dependencies)) - - -def _record_imported_macro_bindings( - *, - statement: ast.ImportFrom, - importer: DiscoveredMacroFile, - imported_file: DiscoveredMacroFile, - exported_names: frozenset[str], - imported: dict[str, DeclarationIdentity], - resolved_module: str, -) -> dict[str, DeclarationIdentity]: - updated: dict[str, DeclarationIdentity] = dict(imported) - for alias in statement.names: - if alias.name == _STAR_IMPORT_NAME: - raise CompileInputError( - f"Macro module '{importer.relative_path}' must not use star imports for project " - "macros" - ) - if alias.name not in exported_names: - raise CompileInputError( - f"Macro module '{importer.relative_path}' cannot import '{alias.name}' from " - f"project macro module '{resolved_module}'; the name is not an exported macro" - ) - _validate_macro_import_visibility( - importer=importer, imported=imported_file, name=alias.name - ) - binding: str = alias.asname or alias.name - if binding in updated: - raise CompileInputError( - f"Macro module '{importer.relative_path}' binds project macro import '{binding}' " - "more than once" + return f"Failed to parse macros from '{macro_file.file_path}': {error}" + + +def _native_macro_inventory( + *, macro_files: tuple[DiscoveredMacroFile, ...], trees: tuple[ast.Module | str, ...] +) -> tuple[StaticMacroInventory, dict[str, _MacroModuleLoad] | None]: + """The native static inventory and, when every module analysed cleanly, the load plan.""" + + export_rows, fault_rows, module_rows = _native.inventory_macro_modules( + [ + ( + str(macro_file.relative_path), + macro_file.relative_path.as_posix(), + list(macro_file.relative_path.parts), + None + if macro_file.declaration_root is None + else list(macro_file.declaration_root.parts), + None if macro_file.owning_path is None else list(macro_file.owning_path.parts), + macro_file.scope_kind.value, + macro_file.contents, + tree if isinstance(tree, str) else None, ) - updated[binding] = DeclarationIdentity(DeclarationKind.MACRO, alias.name) - return updated - - -def _analyze_macro_calls( - *, - functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef], - imported: dict[str, DeclarationIdentity], - file: DiscoveredMacroFile, -) -> tuple[ - dict[str, tuple[DeclarationIdentity, ...]], - dict[str, tuple[str, ...]], - frozenset[int], -]: - direct_dependencies: dict[str, tuple[DeclarationIdentity, ...]] = {} - helper_calls: dict[str, tuple[str, ...]] = {} - runtime_node_ids: set[int] = set() - for function_name, function in functions.items(): - runtime_nodes: tuple[ast.AST, ...] = _runtime_function_nodes(function=function) - runtime_node_ids.update(id(node) for node in runtime_nodes) - dependencies, called_helpers = _analyze_function_calls( - function=function, - runtime_nodes=runtime_nodes, - functions=frozenset(functions), - imported=imported, - file=file, - ) - direct_dependencies[function_name] = dependencies - helper_calls[function_name] = called_helpers - return direct_dependencies, helper_calls, frozenset(runtime_node_ids) - - -def _analyze_function_calls( - *, - function: ast.FunctionDef | ast.AsyncFunctionDef, - runtime_nodes: tuple[ast.AST, ...], - functions: frozenset[str], - imported: dict[str, DeclarationIdentity], - file: DiscoveredMacroFile, -) -> tuple[tuple[DeclarationIdentity, ...], tuple[str, ...]]: - bound_names: set[str] = _function_bound_names(function=function, runtime_nodes=runtime_nodes) - shadowed_imports: set[str] = bound_names.intersection(imported) - if shadowed_imports: - shadowed_name: str = sorted(shadowed_imports)[0] - raise CompileInputError( - f"Macro module '{file.relative_path}' shadows imported macro '{shadowed_name}'; " - "imported macros must be called directly without shadowing" - ) - dependencies: list[DeclarationIdentity] = [] - called_helpers: list[str] = [] - direct_call_nodes: set[int] = { - id(node.func) - for node in runtime_nodes - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) - } - for node in runtime_nodes: - if isinstance(node, ast.Name) and node.id in imported and id(node) not in direct_call_nodes: - raise CompileInputError( - f"Macro module '{file.relative_path}' uses imported macro '{node.id}' " - "dynamically; call imported macros directly" - ) - if ( - isinstance(node, ast.Name) - and node.id in functions - and id(node) not in direct_call_nodes - ): - raise CompileInputError( - f"Macro module '{file.relative_path}' uses local helper '{node.id}' dynamically; " - "call local macros and helpers directly" - ) - if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name): - continue - if node.func.id in imported: - dependencies.append(imported[node.func.id]) - elif node.func.id in functions: - if node.func.id in bound_names: - raise CompileInputError( - f"Macro module '{file.relative_path}' shadows local helper '{node.func.id}'; " - "static macro dependency analysis cannot resolve this call" - ) - called_helpers.append(node.func.id) - return tuple(dict.fromkeys(dependencies)), tuple(dict.fromkeys(called_helpers)) - - -def _function_bound_names( - *, function: ast.FunctionDef | ast.AsyncFunctionDef, runtime_nodes: tuple[ast.AST, ...] -) -> set[str]: - names: set[str] = { - argument.arg - for argument in ( - *function.args.posonlyargs, - *function.args.args, - *function.args.kwonlyargs, - ) - } - if function.args.vararg is not None: - names.add(function.args.vararg.arg) - if function.args.kwarg is not None: - names.add(function.args.kwarg.arg) - names.update( - node.id - for node in runtime_nodes - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + for macro_file, tree in zip(macro_files, trees, strict=True) + ] ) - return names - - -def _validate_call_use_locations( - *, - tree: ast.Module, - imported: dict[str, DeclarationIdentity], - local_functions: frozenset[str], - runtime_node_ids: frozenset[int], - file: DiscoveredMacroFile, -) -> None: - unsupported: ast.Name | None = next( - ( - node - for node in ast.walk(tree) - if isinstance(node, ast.Name) - and node.id in imported - and id(node) not in runtime_node_ids + inventory: StaticMacroInventory = StaticMacroInventory( + exports=tuple( + _static_macro_export(row=row, macro_files=macro_files) for row in export_rows ), - None, - ) - if unsupported is not None: - raise CompileInputError( - f"Macro module '{file.relative_path}' uses imported macro '{unsupported.id}' outside " - "an ordinary function body; call imported macros directly from module-level macro or " - "helper functions" - ) - unsupported_local: ast.Name | None = next( - ( - node - for node in ast.walk(tree) - if isinstance(node, ast.Name) - and node.id in local_functions - and id(node) not in runtime_node_ids + faults=tuple( + StaticMacroFault(relative_path=macro_files[file_index].relative_path, message=message) + for file_index, message in fault_rows ), - None, ) - if unsupported_local is not None: - raise CompileInputError( - f"Macro module '{file.relative_path}' uses local helper " - f"'{unsupported_local.id}' outside an ordinary function body; call local macros and " - "helpers directly from module-level macro or helper functions" - ) - - -def _macro_package_name(file: DiscoveredMacroFile) -> str: - root: Path = file.declaration_root or Path(file.relative_path.parts[0]) - return ".".join(root.parts) - - -def _runtime_function_nodes( - *, function: ast.FunctionDef | ast.AsyncFunctionDef -) -> tuple[ast.AST, ...]: - """Return function-body nodes without entering dynamically nested scopes.""" - - nodes: list[ast.AST] = [] - pending: list[ast.AST] = list(reversed(function.body)) - while pending: - node: ast.AST = pending.pop() - nodes.append(node) - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef): - continue - pending.extend(reversed(tuple(ast.iter_child_nodes(node)))) - return tuple(nodes) - - -def _export_dependencies( - *, - exports: frozenset[str], - direct_dependencies: dict[str, tuple[DeclarationIdentity, ...]], - helper_calls: dict[str, tuple[str, ...]], -) -> dict[str, tuple[DeclarationIdentity, ...]]: - dependencies: dict[str, tuple[DeclarationIdentity, ...]] = {} - for export in exports: - found: list[DeclarationIdentity] = [] - pending: list[str] = [export] - visited: set[str] = set() - while pending: - function_name: str = pending.pop(0) - if function_name in visited: - continue - visited.add(function_name) - found.extend(direct_dependencies[function_name]) - for called_function in helper_calls[function_name]: - if called_function in exports: - found.append(DeclarationIdentity(DeclarationKind.MACRO, called_function)) - pending.append(called_function) - dependencies[export] = tuple(dict.fromkeys(found)) - return dependencies - - -def _validate_local_call_cycles( - *, - exports: frozenset[str], - helper_calls: dict[str, tuple[str, ...]], - file: DiscoveredMacroFile, -) -> None: - completed: frozenset[str] = frozenset() - for export in sorted(exports): - completed = _visit_local_call( - name=export, - helper_calls=helper_calls, - file=file, - active=(), - completed=completed, - ) - - -def _visit_local_call( - *, - name: str, - helper_calls: dict[str, tuple[str, ...]], - file: DiscoveredMacroFile, - active: tuple[str, ...], - completed: frozenset[str], -) -> frozenset[str]: - if name in active: - cycle: tuple[str, ...] = (*active[active.index(name) :], name) - raise CompileInputError(f"Macro call cycle in '{file.relative_path}': {' -> '.join(cycle)}") - if name in completed: - return completed - updated: frozenset[str] = completed - for dependency in helper_calls[name]: - updated = _visit_local_call( - name=dependency, - helper_calls=helper_calls, - file=file, - active=(*active, name), - completed=updated, - ) - return updated.union((name,)) - - -def _validated_macro_analyses( - *, analyses: dict[str, _MacroModuleAnalysis] -) -> dict[str, _MacroModuleAnalysis]: - _validate_import_cycles(analyses=analyses) - return analyses - - -def _resolve_import_from_module(*, statement: ast.ImportFrom, current_module_name: str) -> str: - base: str = statement.module or "" - if not statement.level: - return base - package_parts: list[str] = current_module_name.split(".")[:-1] - if statement.level > len(package_parts): - raise CompileInputError( - f"Relative project macro import in '{current_module_name}' escapes its top-level " - "package" - ) - retained: int = len(package_parts) - statement.level + 1 - parts: list[str] = package_parts[: max(retained, 0)] - if base: - parts.extend(base.split(".")) - return ".".join(parts) - - -def _validate_no_nested_project_imports( - *, - tree: ast.Module, - module_name: str, - file: DiscoveredMacroFile, - module_names: frozenset[str], - package_names: frozenset[str], -) -> None: - top_level_imports: set[ast.Import | ast.ImportFrom] = { - statement for statement in tree.body if isinstance(statement, ast.Import | ast.ImportFrom) - } - for node in ast.walk(tree): - if not isinstance(node, ast.Import | ast.ImportFrom) or node in top_level_imports: - continue - imported_names: tuple[str, ...] = _imported_module_names( - statement=node, current_module_name=module_name - ) - if any( - _is_project_macro_import( - imported_name=name, - module_names=module_names, - package_names=package_names, - ) - for name in imported_names - ): - raise CompileInputError( - f"Macro module '{file.relative_path}' contains a nested project macro import; " - "project macro imports must be static and module-level" - ) - - -def _validate_macro_import_visibility( - *, importer: DiscoveredMacroFile, imported: DiscoveredMacroFile, name: str -) -> None: - visible: bool = imported.scope_kind is ScopeKind.GLOBAL - importer_parent: Path = importer.owning_path or importer.relative_path.parent - owner: Path = imported.owning_path or Path(".") - if imported.scope_kind is ScopeKind.LOCAL: - visible = importer_parent == owner - elif imported.scope_kind is ScopeKind.INHERITED: - visible = importer_parent == owner or owner in importer_parent.parents - if not visible: - raise CompileInputError( - f"Macro module '{importer.relative_path}' cannot import macro '{name}' from " - f"'{imported.relative_path}': the imported macro is not visible from the importer scope" - ) - - -def _validate_import_cycles(*, analyses: dict[str, _MacroModuleAnalysis]) -> None: - completed: frozenset[str] = frozenset() - for name in sorted(analyses): - completed = _visit_macro_import( - name=name, analyses=analyses, active=(), completed=completed + if module_rows is None: + return inventory, None + loads: dict[str, _MacroModuleLoad] = {} + for file_index, module_name, module_dependencies in module_rows: + tree: ast.Module | str = trees[file_index] + if isinstance(tree, str): + raise CompileInputError(tree) + loads[module_name] = _MacroModuleLoad( + file=macro_files[file_index], + tree=tree, + module_dependencies=tuple(module_dependencies), ) + return inventory, loads -def _visit_macro_import( - *, - name: str, - analyses: dict[str, _MacroModuleAnalysis], - active: tuple[str, ...], - completed: frozenset[str], -) -> frozenset[str]: - if name in active: - cycle: tuple[str, ...] = (*active[active.index(name) :], name) - raise CompileInputError( - f"Macro import cycle in '{analyses[name].file.relative_path}': {' -> '.join(cycle)}" - ) - if name in completed: - return completed - updated: frozenset[str] = completed - for dependency in analyses[name].module_dependencies: - updated = _visit_macro_import( - name=dependency, - analyses=analyses, - active=(*active, name), - completed=updated, - ) - return updated.union((name,)) - - -def _load_macro_modules(*, analyses: dict[str, _MacroModuleAnalysis]) -> dict[str, ModuleType]: +def _load_macro_modules(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, ModuleType]: with _MACRO_MODULE_LOAD_LOCK: return _load_macro_modules_locked(analyses=analyses) -def _load_macro_modules_locked( - *, analyses: dict[str, _MacroModuleAnalysis] -) -> dict[str, ModuleType]: +def _load_macro_modules_locked(*, analyses: dict[str, _MacroModuleLoad]) -> dict[str, ModuleType]: digest: str = hashlib.sha256( "\0".join( f"{name}\0{analyses[name].file.file_path}\0{analyses[name].file.contents}" @@ -979,7 +339,7 @@ def _load_macro_modules_locked( heapq.heapify(ready) while ready: name: str = heapq.heappop(ready) - analysis: _MacroModuleAnalysis = analyses[name] + analysis: _MacroModuleLoad = analyses[name] synthetic_name: str = f"{root}.{name}" module: ModuleType = ModuleType(synthetic_name) module.__file__ = str(analysis.file.file_path) @@ -1039,73 +399,6 @@ def _rewrite_absolute_project_imports( return ast.Module(body=body, type_ignores=tree.type_ignores) -def _matches_macro_module(*, imported_name: str, module_names: frozenset[str]) -> bool: - module_name: str - for module_name in module_names: - if module_name == imported_name or module_name.startswith(f"{imported_name}."): - return True - return False - - -def _is_project_macro_import( - *, - imported_name: str, - module_names: frozenset[str], - package_names: frozenset[str], -) -> bool: - return _matches_macro_module(imported_name=imported_name, module_names=module_names) or any( - imported_name == package or imported_name.startswith(f"{package}.") - for package in package_names - ) - - -def _imported_module_names(*, statement: ast.stmt, current_module_name: str) -> tuple[str, ...]: - if isinstance(statement, ast.Import): - return tuple(alias.name for alias in statement.names) - if not isinstance(statement, ast.ImportFrom): - return () - - base_module_name: str = statement.module or "" - if statement.level: - current_package_parts: list[str] = current_module_name.split(".")[:-1] - retained_part_count: int = len(current_package_parts) - statement.level + 1 - base_parts: list[str] = current_package_parts[: max(retained_part_count, 0)] - if base_module_name: - base_parts.extend(base_module_name.split(".")) - base_module_name = ".".join(base_parts) - names: list[str] = [] - if base_module_name: - names.append(base_module_name) - alias: ast.alias - for alias in statement.names: - if alias.name == _STAR_IMPORT_NAME: - continue - imported_parts: list[str] = [base_module_name] if base_module_name else [] - imported_parts.append(alias.name) - names.append(".".join(imported_parts)) - return tuple(names) - - -def _public_non_function_declaration_name(statement: ast.stmt) -> str | None: - if isinstance(statement, ast.ClassDef) and not statement.name.startswith("_"): - return statement.name - if isinstance(statement, ast.TypeAlias) and isinstance(statement.name, ast.Name): - return statement.name.id if not statement.name.id.startswith("_") else None - assignment_names: tuple[str, ...] = () - if isinstance(statement, ast.Assign): - names: list[str] = [] - target: ast.expr - for target in statement.targets: - node: ast.AST - for node in ast.walk(target): - if isinstance(node, ast.Name): - names.append(node.id) - assignment_names = tuple(names) - elif isinstance(statement, ast.AnnAssign) and isinstance(statement.target, ast.Name): - assignment_names = (statement.target.id,) - return next((name for name in assignment_names if not name.startswith("_")), None) - - def expand_sql_macros( *, sql: str, @@ -1259,13 +552,12 @@ def _inventory_project_macros( ) -> StaticMacroInventory: """Validate and inventory project macro exports without executing authored code.""" - try: - analyses: dict[str, _MacroModuleAnalysis] = _analyze_project_macro_modules( - macro_files=macro_files - ) - except CompileInputError: - return _inventory_project_macros_tolerantly(macro_files=macro_files) - return _inventory_from_analyses(macro_files=macro_files, analyses=analyses) + inventory: StaticMacroInventory + inventory, _loads = _native_macro_inventory( + macro_files=macro_files, + trees=tuple(_parse_macro_module(macro_file=macro_file) for macro_file in macro_files), + ) + return inventory def find_macro_call_names(sql: str) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py deleted file mode 100644 index 8cb95bab87..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Point SQL-test extraction errors and malformed calls at the authored text they are about.""" - -from __future__ import annotations - -import sys -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.sql_tests.native import ( - authored_sql_test_cte_batch, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_TAB_SIZE: int = 8 -_LINE_BREAK: str = "\n" -_CARRIAGE_RETURN: str = "\r" -_TAB: str = "\t" - - -@dataclass(frozen=True) -class _CleanedLine: - """One line of `inspect.cleandoc` output and where it comes from in the authored text.""" - - authored_start: int - authored_text: str - removed_columns: int - - -def validate_authored_test_cte_names( - *, test_files: tuple[DiscoveredSqlTestFile, ...], syntax: SqlLexicalSyntax -) -> None: - """Raise the located CTE-name error of the first authored test block that has one.""" - - blocks: list[tuple[DiscoveredSqlTestFile, DiscoveredSqlTestBlock]] = [] - test_file: DiscoveredSqlTestFile - for test_file in test_files: - blocks.extend((test_file, test_block) for test_block in test_file.blocks) - try: - _ = authored_sql_test_cte_batch( - tests=tuple( - (test_block.sql_body, str(test_file.relative_path)) - for test_file, test_block in blocks - ), - syntax=syntax, - ) - except SqlTestExtractionError as error: - failed_file, failed_block = blocks[error.test_index] - raise located_extraction_error( - error=error, test_file=failed_file, test_block=failed_block, sql=failed_block.sql_body - ) from None - - -def validate_authored_scenario_cte_names( - *, scenario_files: tuple[DiscoveredSqlScenarioFile, ...], syntax: SqlLexicalSyntax -) -> None: - """Raise the located CTE-name or materialization error of the first scenario that has one.""" - - try: - _ = authored_sql_test_cte_batch( - tests=tuple( - (scenario_file.sql_body, str(scenario_file.relative_path)) - for scenario_file in scenario_files - ), - syntax=syntax, - scenarios=True, - ) - except SqlTestExtractionError as error: - failed: DiscoveredSqlScenarioFile = scenario_files[error.test_index] - offset: int | None = ( - _authored_offset( - contents=failed.contents, - authored=failed.sql_body, - span=failed.sql_body_span, - body_offset=error.token_offset, - ) - if error.token_offset is not None - else None - ) - raise _located_error( - error=error, contents=failed.contents, relative_path=failed.relative_path, offset=offset - ) from None - - -def located_extraction_error( - *, - error: SqlTestExtractionError, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - sql: str, -) -> CompileInputError: - """Return the error at the file line and column of its offending text when that is exact.""" - - offset: int | None = ( - authored_file_offset( - test_file=test_file, test_block=test_block, body_offset=error.token_offset - ) - if error.token_offset is not None and sql == test_block.sql_body - else None - ) - return _located_error( - error=error, - contents=test_file.contents, - relative_path=test_file.relative_path, - offset=offset, - ) - - -def _located_error( - *, error: SqlTestExtractionError, contents: str, relative_path: Path, offset: int | None -) -> CompileInputError: - if offset is None: - return CompileInputError(error.message, code=error.code, help=error.help) - line: int = contents.count(_LINE_BREAK, 0, offset) + 1 - column: int = offset - (contents.rfind(_LINE_BREAK, 0, offset) + 1) + 1 - return CompileInputError( - f"{error.message}\n --> {relative_path.as_posix()}:{line}:{column}", - code=error.code, - help=error.help, - ) - - -def authored_file_offset( - *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock, body_offset: int -) -> int | None: - """Map an offset in a block's cleaned SQL back to the file, or None if it cannot be exact.""" - - return _authored_offset( - contents=test_file.contents, - authored=test_block.sql_body, - span=test_block.sql_body_span, - body_offset=body_offset, - ) - - -def _authored_offset( - *, contents: str, authored: str, span: tuple[int, int] | None, body_offset: int -) -> int | None: - if span is None: - return None - lines: tuple[_CleanedLine, ...] | None = _cleaned_lines( - authored=authored, span=span, contents=contents - ) - if lines is None: - return None - body: str = authored - line_index: int = body.count(_LINE_BREAK, 0, body_offset) - column: int = body_offset - (body.rfind(_LINE_BREAK, 0, body_offset) + 1) - cleaned: _CleanedLine = lines[line_index] - return cleaned.authored_start + _authored_index( - text=cleaned.authored_text, expanded_column=cleaned.removed_columns + column - ) - - -def _cleaned_lines( - *, authored: str, span: tuple[int, int], contents: str -) -> tuple[_CleanedLine, ...] | None: - text: str = contents[span[0] : span[1]] - raw_lines: list[str] = text.split(_LINE_BREAK) - expanded: list[str] = [line.expandtabs(_TAB_SIZE) for line in raw_lines] - indents: list[int] = [len(line) - len(line.lstrip()) for line in expanded[1:] if line.strip()] - margin: int = min(indents, default=sys.maxsize) - starts: list[int] = [] - cursor: int = span[0] - for line in raw_lines: - starts.append(cursor) - cursor += len(line) + 1 - removed: list[int] = [ - len(expanded[0]) - len(expanded[0].lstrip()), - *([margin if margin < sys.maxsize else 0] * (len(expanded) - 1)), - ] - kept: list[int] = list(range(len(expanded))) - while kept and not expanded[kept[-1]][removed[kept[-1]] :]: - kept.pop() - while kept and not expanded[kept[0]][removed[kept[0]] :]: - kept.pop(0) - cleaned: str = _LINE_BREAK.join(expanded[index][removed[index] :] for index in kept) - if cleaned != authored: - return None - return tuple( - _CleanedLine( - authored_start=starts[index], - authored_text=raw_lines[index], - removed_columns=removed[index], - ) - for index in kept - ) - - -def _authored_index(*, text: str, expanded_column: int) -> int: - position: int = 0 - column: int = 0 - index: int - character: str - for index, character in enumerate(text): - width: int = _TAB_SIZE - column % _TAB_SIZE if character == _TAB else 1 - if expanded_column < position + width: - return index - position += width - column = 0 if character == _CARRIAGE_RETURN else column + width - return len(text) diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py deleted file mode 100644 index 1ab0b45a23..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Native boundary for SQL-test extraction before and after expansion, under adapter rules.""" - -from __future__ import annotations - -from typing import Protocol, cast - -import orjson - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import ( - CompileInputError, - SqlTestExtractionError, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -class _NativeSqlTestModule(Protocol): - def extract_sql_tests_json(self, request_json: str) -> str: ... - - -_SYNTAX_FIELDS: dict[str, str] = { - "backslash_escape_quotes": "backslashEscapeQuotes", - "escape_string_prefix": "escapeStringPrefix", - "raw_string_prefix": "rawStringPrefix", - "triple_quoted_strings": "tripleQuotedStrings", - "nested_block_comments": "nestedBlockComments", - "line_comment_prefixes": "lineCommentPrefixes", -} -_AUTHORED_CTE_LENGTH: int = 3 - - -def authored_sql_test_cte_batch( - *, tests: tuple[tuple[str, str], ...], syntax: SqlLexicalSyntax, scenarios: bool = False -) -> tuple[tuple[tuple[str, int, str], ...], ...]: - """Read many authored blocks or scenarios at once; a CTE error names one by `test_index`.""" - - if not tests: - return () - results: list[object] = _native_batch( - tests=[ - { - "sql": sql, - "fileLabel": file_label, - "mode": "model", - "authored": True, - "scenario": scenarios, - } - for sql, file_label in tests - ], - syntax=syntax, - ) - return tuple(_authored_ctes(item) for item in results) - - -def _authored_ctes(item: object) -> tuple[tuple[str, int, str], ...]: - ctes: object = cast(dict[str, object], item).get("ctes") if isinstance(item, dict) else None - if not isinstance(ctes, list): - raise CompileInputError("native SQL test extraction returned an invalid CTE response") - return tuple(_authored_cte(value) for value in ctes) - - -def _authored_cte(value: object) -> tuple[str, int, str]: - if not ( - isinstance(value, list) - and len(value) == _AUTHORED_CTE_LENGTH - and isinstance(value[0], str) - and isinstance(value[1], int) - and isinstance(value[2], str) - ): - raise CompileInputError("native SQL test extraction returned an invalid CTE") - return value[0], value[1], value[2] - - -def _native_batch(*, tests: list[dict[str, object]], syntax: SqlLexicalSyntax) -> list[object]: - native_syntax: dict[str, object] = syntax.native_mapping - request_json: str = orjson.dumps( - { - "syntax": {_SYNTAX_FIELDS[key]: value for key, value in native_syntax.items()}, - "tests": tests, - } - ).decode() - try: - response: object = orjson.loads( - cast(_NativeSqlTestModule, _native).extract_sql_tests_json(request_json) - ) - except ValueError as error: - raise CompileInputError(str(error)) from None - if isinstance(response, dict) and isinstance(response.get("error"), dict): - raise _extraction_error(cast(dict[str, object], response["error"])) - results: object = response.get("tests") if isinstance(response, dict) else None - if not isinstance(results, list) or len(results) != len(tests): - raise CompileInputError("native SQL test extraction returned an invalid batch response") - return cast(list[object], results) - - -def _extraction_error(error: dict[str, object]) -> SqlTestExtractionError: - message: object = error.get("message") - help_text: object = error.get("help") - index: object = error.get("index") - token: object = error.get("token") - token_offset: object = error.get("tokenOffset") - return SqlTestExtractionError( - message if isinstance(message, str) else "native SQL test extraction failed", - help=help_text if isinstance(help_text, str) else None, - test_index=index if isinstance(index, int) else 0, - token=token if isinstance(token, str) else None, - token_offset=token_offset if isinstance(token_offset, int) else None, - ) diff --git a/src/sqlbuild/compiler/contract_adoption/main/write.py b/src/sqlbuild/compiler/contract_adoption/main/write.py index 426aee7550..ae108c3b71 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/write.py @@ -5,6 +5,7 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, @@ -26,7 +27,6 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.main.write_atomically import write_atomically diff --git a/src/sqlbuild/compiler/manifest/main/build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py similarity index 100% rename from src/sqlbuild/compiler/manifest/main/build_graph_manifest.py rename to src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py diff --git a/src/sqlbuild/compiler/pipeline/_helpers/clone.py b/src/sqlbuild/compiler/pipeline/_helpers/clone.py index 5127f9eb66..aa31ac89d8 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/clone.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/clone.py @@ -2,11 +2,11 @@ from __future__ import annotations +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompileAnalysisSelection from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/diff.py b/src/sqlbuild/compiler/pipeline/_helpers/diff.py index a2959bd7e5..2b2884a36d 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/diff.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/diff.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -16,7 +17,6 @@ PATH_SELECTOR_PREFIX, PATH_SEPARATORS, ) -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver diff --git a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py index 983e2dae84..274d369125 100644 --- a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py +++ b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py @@ -5,6 +5,7 @@ import time from collections.abc import Callable +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config @@ -14,7 +15,6 @@ from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( resolve_compile_analysis_selection, ) -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.models import CompiledProjectPhaseResult, CompilePipelineOptions from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py deleted file mode 100644 index 133f17c259..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Build a compiled project with target defaults and adapter-aware validation.""" - -from __future__ import annotations - -from functools import partial - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.main._assemble_project import assemble_project -from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import ( - CompileAdapterContext, - CompileAnalysisSelection, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.project_with import project_with -from sqlbuild.compiler.contracts.main.promotion_conflicts import promotion_conflict_diagnostics -from sqlbuild.compiler.discovery.constants import ( - LOCAL_CONFIG_FILENAME, - PROJECT_CONFIG_FILENAME, - TABLE_PROMOTION_MODE_SETTING_KEY, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier -from sqlbuild.compiler.frontier.types import CompilerStage -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline._helpers.analysis_selection import resolve_analysis_model_names -from sqlbuild.compiler.pipeline._helpers.target_defaults import apply_target_defaults -from sqlbuild.compiler.pipeline._helpers.target_validation import ( - validate_managed_loader_target_isolation, - validate_managed_write_schemas, - validate_named_target_schema_strategy, - validate_project_targets, -) -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( - resolve_effective_collection_rendering, -) - - -def build_compiled_project( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None = None, - no_sql_validation: bool = False, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - cli_vars: dict[str, object] | None = None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - resolved_connection: dict[str, object] | None = None, - analysis_selection: CompileAnalysisSelection | None = None, -) -> CompiledProjectView: - """Build one compiled project with adapter defaults and target validation applied.""" - - return compile_frontier( - until=CompilerStage.COMPILED_PROJECT, - python_stage=partial( - _build_python_compiled_project, - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - resolved_connection=resolved_connection, - analysis_selection=analysis_selection, - ), - ) - - -def _build_python_compiled_project( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None, - no_sql_validation: bool, - skip_column_inference: bool, - column_lineage_mode: ColumnLineageMode, - cli_vars: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - resolved_connection: dict[str, object] | None, - analysis_selection: CompileAnalysisSelection | None, -) -> CompiledProjectView: - validate_named_target_schema_strategy( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - ) - validate_managed_loader_target_isolation( - discovered_inputs=discovered_inputs, - adapter=adapter, - resolved_connection=resolved_connection, - ) - no_cache: bool = analysis_selection is not None and analysis_selection.no_cache - with record_compile_timing("attachment_ms"): - compile_inputs: CompileInputsView = _frontier_compile_inputs( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - resolved_connection=resolved_connection, - external_sql_reference_resolver=external_sql_reference_resolver, - no_cache=no_cache, - ) - if analysis_selection is not None and analysis_selection.on_inputs_ready is not None: - analysis_selection.on_inputs_ready(inputs=compile_inputs) - inference_profile: ExpressionInferenceProfile = adapter.expression_inference_profile() - analysis_model_names: frozenset[str] | None = resolve_analysis_model_names( - compile_inputs=compile_inputs, - inference_profile=inference_profile, - selection=analysis_selection, - ) - project: CompiledProjectView = assemble_project( - inputs=compile_inputs, - inference_profile=inference_profile, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - analysis_cache_dir=compile_inputs.compile_cache_dir, - analysis_model_names=analysis_model_names, - ) - validate_managed_write_schemas(adapter=adapter, project=project) - project = apply_target_defaults( - project=project, - default_schema=project.effective_target_schema or adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - ) - validate_project_targets( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project=project, - ) - return project_with( - project=project, - diagnostics=( - *project.diagnostics, - *promotion_conflict_diagnostics( - project=project, - adapter_default=adapter.default_table_promotion_mode(), - settings_file=( - LOCAL_CONFIG_FILENAME - if TABLE_PROMOTION_MODE_SETTING_KEY - in discovered_inputs.local_config.setting_overrides - else PROJECT_CONFIG_FILENAME - ), - ), - ), - ) - - -def _frontier_compile_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None, - no_sql_validation: bool, - cli_vars: dict[str, object] | None, - resolved_connection: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - no_cache: bool, -) -> CompileInputsView: - return compile_frontier( - until=CompilerStage.COMPILE_PROJECT_INPUTS, - python_stage=partial( - build_compile_inputs, - discovered_inputs=discovered_inputs, - adapter_context=CompileAdapterContext( - value_renderer=adapter, - collection_rendering=resolve_effective_collection_rendering( - project_config=discovered_inputs.project_config, - declaration_override=None, - ), - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - sql_lexical_syntax=adapter.sql_lexical_syntax, - ), - selected_target=selected_target, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=True, - cli_vars=cli_vars, - resolved_connection=resolved_connection, - external_sql_reference_resolver=external_sql_reference_resolver, - no_cache=no_cache, - ), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/graph.py b/src/sqlbuild/compiler/pipeline/main/graph.py deleted file mode 100644 index 873ef8fd46..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/graph.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Build the static compiled project dependency graph.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompileAnalysisSelection -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.main.selected_graph import ( - build_project_graph_with_analysis_selection, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def build_project_graph( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - selected_target: str | None = None, - no_sql_validation: bool = False, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - cli_vars: dict[str, object] | None = None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - on_progress: Callable[[str], None] | None = None, - no_cache: bool = False, -) -> ProjectGraph: - """Build the static dependency graph for a compiled project.""" - - return build_project_graph_with_analysis_selection( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - on_progress=on_progress, - analysis_selection=CompileAnalysisSelection(no_cache=no_cache), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/project.py b/src/sqlbuild/compiler/pipeline/main/project.py index 50a337c728..2d4cb068fc 100644 --- a/src/sqlbuild/compiler/pipeline/main/project.py +++ b/src/sqlbuild/compiler/pipeline/main/project.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver diff --git a/src/sqlbuild/compiler/pipeline/main/selected_graph.py b/src/sqlbuild/compiler/pipeline/main/selected_graph.py deleted file mode 100644 index 883e0ce786..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/selected_graph.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Build a project graph with explicit deep-analysis selection.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def build_project_graph_with_analysis_selection( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - analysis_selection: CompileAnalysisSelection, - selected_target: str | None = None, - no_sql_validation: bool = False, - skip_column_inference: bool = False, - column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, - cli_vars: dict[str, object] | None = None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - on_progress: Callable[[str], None] | None = None, -) -> ProjectGraph: - """Build a static graph with an explicit deep-analysis selection.""" - - if on_progress is not None: - on_progress("Compiling project...") - compile_start: float = time.monotonic() - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=selected_target, - no_sql_validation=no_sql_validation, - skip_column_inference=skip_column_inference, - column_lineage_mode=column_lineage_mode, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - analysis_selection=analysis_selection, - ) - if on_progress is not None: - on_progress(f"Compiled project. ({time.monotonic() - compile_start:.2f}s)") - return build_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/profiling/main/collect.py b/src/sqlbuild/compiler/profiling/main/_collect.py similarity index 100% rename from src/sqlbuild/compiler/profiling/main/collect.py rename to src/sqlbuild/compiler/profiling/main/_collect.py diff --git a/src/sqlbuild/compiler/scopes/_helpers/builder.py b/src/sqlbuild/compiler/scopes/_helpers/builder.py index 898763cd9f..f7e35f2595 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/builder.py +++ b/src/sqlbuild/compiler/scopes/_helpers/builder.py @@ -2,35 +2,18 @@ from __future__ import annotations -from dataclasses import replace from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM from sqlbuild.compiler.compile.main._static_macro_inventory import inventory_project_macros from sqlbuild.compiler.compile.models import StaticMacroInventory from sqlbuild.compiler.discovery.main._scope_snapshot import discover_scope_snapshot -from sqlbuild.compiler.discovery.models import DiscoveryFileFault, TolerantScopeDiscovery -from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex +from sqlbuild.compiler.discovery.models import TolerantScopeDiscovery from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.models import ( - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeCompleteness, - ScopeDiagnostic, - ScopeIndex, -) -from sqlbuild.compiler.scopes.types import ResourceKind, ScopeDiagnosticCode +from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_ROOTS: dict[ResourceKind, str] = { - ResourceKind.MODEL: "models", - ResourceKind.TEST: "tests/unit", - ResourceKind.SCENARIO: "tests/scenarios", - ResourceKind.FUNCTION: "functions/sql", - ResourceKind.SOURCE: "sources", -} - def build_tolerant_scope_index( *, project_dir: Path, sql_lexical_syntax: SqlLexicalSyntax @@ -41,7 +24,7 @@ def build_tolerant_scope_index( macro_inventory: StaticMacroInventory = inventory_project_macros( macro_files=snapshot.discovered_inputs.macro_files ) - native: NativeScopeIndex = open_native_scope_index( + native: _native.NativeScopeIndex = open_native_scope_index( discovered_inputs=snapshot.discovered_inputs, loaded_macros={}, static_macros=macro_inventory.exports, @@ -51,112 +34,8 @@ def build_tolerant_scope_index( syntax=sql_lexical_syntax, scan_store=(None, SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM), ) - index: ScopeIndex = native.index_with_relationships() - diagnostics: tuple[ScopeDiagnostic, ...] = _tolerant_diagnostics( - snapshot=snapshot, - macro_inventory=macro_inventory, - relationship_faults=relationship_faults, - ) - prospective: tuple[ResourceRecord, ...] = tuple( - record - for fault in (*snapshot.resource_faults, *snapshot.relationship_faults) - if fault.path is not None - if (record := _prospective_resource(path=fault.path)) is not None - ) - return replace( - index, - resources=tuple( - sorted( - dict.fromkeys((*index.resources, *prospective)), - key=_resource_key, - ) - ), - diagnostics=tuple( - sorted( - (*index.diagnostics, *diagnostics), - key=_diagnostic_key, - ) - ), - completeness=ScopeCompleteness( - discovery=False, - static_visibility=not ( - snapshot.declaration_faults or snapshot.config_faults or macro_inventory.faults - ), - runtime_usage=False, - relationships=not snapshot.relationship_faults and not relationship_faults, - placement=False, - promotion_impact=False, - ), - ) - - -def _tolerant_diagnostics( - *, - snapshot: TolerantScopeDiscovery, - macro_inventory: StaticMacroInventory, - relationship_faults: tuple[tuple[Path, str], ...], -) -> tuple[ScopeDiagnostic, ...]: - diagnostics: list[ScopeDiagnostic] = [] - diagnostics.extend( - _discovery_diagnostic(fault=fault, code=ScopeDiagnosticCode.RESOURCE_PARSE_ERROR) - for fault in snapshot.resource_faults - ) - diagnostics.extend( - _discovery_diagnostic(fault=fault, code=ScopeDiagnosticCode.DECLARATION_PARSE_ERROR) - for fault in snapshot.declaration_faults - ) - diagnostics.extend( - _discovery_diagnostic(fault=fault, code=ScopeDiagnosticCode.RELATIONSHIP_PARSE_ERROR) - for fault in snapshot.relationship_faults - ) - diagnostics.extend( - _discovery_diagnostic(fault=fault, code=ScopeDiagnosticCode.CONFIG_PARSE_ERROR) - for fault in snapshot.config_faults - ) - diagnostics.extend( - ScopeDiagnostic( - ScopeDiagnosticCode.MACRO_PARSE_ERROR, - fault.message, - path=fault.relative_path.as_posix(), - ) - for fault in macro_inventory.faults + return _native.scope_tolerant_index( + native.index_with_relationships(), + snapshot, + (macro_inventory.faults, relationship_faults), ) - diagnostics.extend( - ScopeDiagnostic( - ScopeDiagnosticCode.RELATIONSHIP_PARSE_ERROR, - message, - path=path.as_posix(), - ) - for path, message in relationship_faults - ) - return tuple(diagnostics) - - -def _discovery_diagnostic( - *, fault: DiscoveryFileFault, code: ScopeDiagnosticCode -) -> ScopeDiagnostic: - return ScopeDiagnostic( - code, - fault.message, - path=fault.path.as_posix() if fault.path is not None else None, - ) - - -def _prospective_resource(*, path: Path) -> ResourceRecord | None: - relative: str = path.as_posix() - for kind, root in _ROOTS.items(): - if relative.startswith(f"{root}/"): - return ResourceRecord( - ResourceIdentity(kind, path.stem), - relative, - OwnershipRoot(root, resource_kind=kind), - ) - return None - - -def _resource_key(record: ResourceRecord) -> tuple[str, str, str]: - return (record.path, record.identity.kind.value, record.identity.name) - - -def _diagnostic_key(item: ScopeDiagnostic) -> tuple[str, int, int, str, str]: - return (item.path or "", item.line or 0, item.column or 0, item.code.value, item.message) diff --git a/src/sqlbuild/compiler/scopes/_helpers/cache.py b/src/sqlbuild/compiler/scopes/_helpers/cache.py index ef413b0e97..4c50f61f6b 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/cache.py +++ b/src/sqlbuild/compiler/scopes/_helpers/cache.py @@ -4,7 +4,6 @@ from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Any import sqlbuild._native as _native from sqlbuild.compiler.discovery.constants import ( @@ -13,44 +12,13 @@ PROJECT_CONFIG_FILENAME, ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row from sqlbuild.compiler.scopes.constants import ( GLOBAL_DECLARATION_DIRECTORIES, GLOBAL_NAMED_DECLARATION_DIRECTORIES, SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME, ) -from sqlbuild.compiler.scopes.models import ( - ConstantMetadata, - DeclarationIdentity, - DeclarationRecord, - EnumMemberMetadata, - EnumMetadata, - GrantRecord, - InaccessibleRecord, - MacroMetadata, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeCompleteness, - ScopeDiagnostic, - ScopeIndex, - UsageRecord, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - DiagnosticSeverity, - GrantKind, - InaccessibleReason, - NativeQualifiedRow, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - UsageKind, - VisibilityReason, -) +from sqlbuild.compiler.scopes.models import ScopeIndex _CONFIG_FILES: tuple[str, ...] = ( PROJECT_CONFIG_FILENAME, @@ -65,7 +33,6 @@ "python", "seeds", ) -_RESOURCE_KINDS: frozenset[str] = frozenset(kind.value for kind in ResourceKind) def scope_index_fingerprint(*, project_dir: Path) -> str | None: @@ -86,248 +53,20 @@ def scope_index_fingerprint(*, project_dir: Path) -> str | None: def read_cached_scope_index(*, project_dir: Path, fingerprint: str) -> ScopeIndex | None: """Read a verified index; every storage or validation fault is a cache miss.""" - rows: Any = _native.read_scope_index_cache( + return _native.read_scope_index_cache( compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME, fingerprint, ) - if rows is None: - return None - try: - return _index(rows) - except ValueError: - return None def write_cached_scope_index(*, project_dir: Path, fingerprint: str, index: ScopeIndex) -> None: """Atomically write one complete canonical scope index, ignoring I/O faults.""" - if not index.completeness.complete: - return _ = _native.write_scope_index_cache( (compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME), fingerprint, - _rows(index), - ) - - -def _rows(index: ScopeIndex) -> Any: - completeness: ScopeCompleteness = index.completeness - return ( - [root_row(root=root) for root in index.ownership_roots], - [ - (_resource_row(record.identity), record.path, root_row(root=record.ownership_root)) - for record in index.resources - ], - [_declaration_row(record) for record in index.declarations], - [ - ( - qualified_row(identity=item.consumer), - qualified_row(identity=item.declaration), - item.kind.value, - _optional_row(item.through), - item.enum_member, - ) - for item in index.usages - ], - [ - ( - _resource_row(item.resource), - qualified_row(identity=item.declaration), - qualified_row(identity=item.through), - item.kind.value, - ) - for item in index.grants - ], - [ - ( - _resource_row(item.resource), - qualified_row(identity=item.declaration), - item.reason.value, - _optional_row(item.through), - ) - for item in index.visibility - ], - [ - ( - _resource_row(item.resource), - qualified_row(identity=item.declaration), - item.reason.value, - ) - for item in index.inaccessible - ], - [ - ( - item.code.value, - item.message, - item.severity.value, - item.path, - item.line, - item.column, - _optional_row(item.declaration), - None if item.resource is None else _resource_row(item.resource), - ) - for item in index.diagnostics - ], - ( - completeness.discovery, - completeness.static_visibility, - completeness.runtime_usage, - completeness.relationships, - completeness.placement, - completeness.promotion_impact, - ), - ) - - -def _declaration_row(record: DeclarationRecord) -> Any: - return ( - ( - qualified_row(identity=record.identity), - record.path, - record.line, - record.column, - record.scope.value, - root_row(root=record.ownership_root), - record.owning_path, - ), - None - if record.macro is None - else ( - list(record.macro.parameters), - [qualified_row(identity=item) for item in record.macro.dependencies], - record.macro.source_digest, - ), - None - if record.enum is None - else ([member.name for member in record.enum.members], record.enum.scalar_type), - None - if record.constant is None - else ( - record.constant.logical_type, - record.constant.collection_kind, - record.constant.item_count, - record.constant.nullable, - record.constant.render_as, - ), - ) - - -def _resource_row(identity: ResourceIdentity) -> tuple[str, str]: - return identity.kind.value, identity.name - - -def _optional_row(identity: ResourceIdentity | DeclarationIdentity | None) -> object: - return None if identity is None else qualified_row(identity=identity) - - -def _index(rows: Any) -> ScopeIndex: - roots, resources, declarations, usages, grants, visibility, inaccessible, diagnostics, done = ( - rows + index, ) - return ScopeIndex( - ownership_roots=tuple(map(_root, roots)), - resources=tuple( - ResourceRecord(_resource(identity), path, _root(root)) - for identity, path, root in resources - ), - declarations=tuple(map(_declaration, declarations)), - usages=tuple( - UsageRecord( - _identity(consumer), - _declaration_identity(declaration), - UsageKind(kind), - None if through is None else _identity(through), - enum_member, - ) - for consumer, declaration, kind, through, enum_member in usages - ), - grants=tuple( - GrantRecord( - _resource(resource), - _declaration_identity(declaration), - _identity(through), - GrantKind(kind), - ) - for resource, declaration, through, kind in grants - ), - visibility=tuple( - VisibilityRecord( - _resource(resource), - _declaration_identity(declaration), - VisibilityReason(reason), - None if through is None else _identity(through), - ) - for resource, declaration, reason, through in visibility - ), - inaccessible=tuple( - InaccessibleRecord( - _resource(resource), _declaration_identity(declaration), InaccessibleReason(reason) - ) - for resource, declaration, reason in inaccessible - ), - diagnostics=tuple(map(_diagnostic, diagnostics)), - completeness=ScopeCompleteness(*done), - ) - - -def _declaration(row: Any) -> DeclarationRecord: - (identity, path, line, column, scope, root, owning_path), macro, enum, constant = row - return DeclarationRecord( - identity=_declaration_identity(identity), - path=path, - line=line, - column=column, - scope=ScopeKind(scope), - ownership_root=_root(root), - owning_path=owning_path, - macro=None - if macro is None - else MacroMetadata(tuple(macro[0]), tuple(map(_declaration_identity, macro[1])), macro[2]), - enum=None - if enum is None - else EnumMetadata(tuple(map(EnumMemberMetadata, enum[0])), enum[1]), - constant=None if constant is None else ConstantMetadata(*constant), - ) - - -def _diagnostic(row: Any) -> ScopeDiagnostic: - code, message, severity, path, line, column, declaration, resource = row - return ScopeDiagnostic( - ScopeDiagnosticCode(code), - message, - DiagnosticSeverity(severity), - path, - line, - column, - None if declaration is None else _declaration_identity(declaration), - None if resource is None else _resource(resource), - ) - - -def _root(row: tuple[str, str, str | None]) -> OwnershipRoot: - path, kind, resource_kind = row - return OwnershipRoot( - path, - OwnershipRootKind(kind), - None if resource_kind is None else ResourceKind(resource_kind), - ) - - -def _resource(row: tuple[str, str]) -> ResourceIdentity: - return ResourceIdentity(ResourceKind(row[0]), row[1]) - - -def _declaration_identity(row: NativeQualifiedRow) -> DeclarationIdentity: - kind, name, owner = row - return DeclarationIdentity( - DeclarationKind(kind), name, None if owner is None else _resource(owner) - ) - - -def _identity(row: NativeQualifiedRow) -> ResourceIdentity | DeclarationIdentity: - if row[2] is None and row[0] in _RESOURCE_KINDS: - return _resource((row[0], row[1])) - return _declaration_identity(row) def _sqlbuild_version() -> str: diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py deleted file mode 100644 index 04bb489b58..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Canonical scope lookups, ordered and grouped by the native scope lookup.""" - -from __future__ import annotations - -from collections.abc import Callable, Hashable, Sequence -from types import MappingProxyType - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibilityIndex, - GrantRecord, - ResourceIdentity, - ResourceRecord, - ScopeIndex, - ScopeLookup, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ResourceKind - -type _Groups = list[list[int]] -type _Mappings = tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups, list[str]] -type _Visibility = tuple[ - list[int], - list[tuple[tuple[str, str], list[int]]], - list[tuple[str, list[int]]], - list[tuple[str, list[int]]], -] - - -def build_lookup(*, index: ScopeIndex) -> ScopeLookup: - """The native lookup of `index`; a path the lookup rejects raises `InvalidScopePathError`.""" - - native: _native.NativeScopeLookup | None - native, (rows, error) = _native.build_native_scope_lookup( - ( - [root_row(root=root) for root in index.ownership_roots], - [ - ( - (record.identity.kind.value, record.identity.name), - record.path, - root_row(root=record.ownership_root), - ) - for record in index.resources - ], - [ - ( - qualified_row(identity=record.identity), - record.path, - record.line, - record.column, - record.scope.value, - root_row(root=record.ownership_root), - record.owning_path, - ) - for record in index.declarations - ], - [ - ( - qualified_row(identity=usage.consumer), - qualified_row(identity=usage.declaration), - usage.kind.value, - None if usage.through is None else qualified_row(identity=usage.through), - ) - for usage in index.usages - ], - [ - ( - (grant.resource.kind.value, grant.resource.name), - qualified_row(identity=grant.declaration), - qualified_row(identity=grant.through), - grant.kind.value, - ) - for grant in index.grants - ], - [(item.path, item.line, item.column, item.code.value) for item in index.diagnostics], - ) - ) - if rows is None: - raise InvalidScopePathError(error or "") - (roots, resources, declarations, usages, grants, diagnostics), mappings, visibility = rows - canonical: ScopeIndex = ScopeIndex( - ownership_roots=tuple(index.ownership_roots[position] for position in roots), - resources=tuple(index.resources[position] for position in resources), - declarations=tuple(index.declarations[position] for position in declarations), - usages=tuple(index.usages[position] for position in usages), - grants=tuple(index.grants[position] for position in grants), - visibility=index.visibility, - inaccessible=index.inaccessible, - diagnostics=tuple(index.diagnostics[position] for position in diagnostics), - completeness=index.completeness, - ) - return project_lookup( - index=canonical, - records=(index.resources, index.declarations, index.usages, index.grants), - mappings=mappings, - visibility=visibility, - native=native, - ) - - -def project_lookup( - *, - index: ScopeIndex, - records: tuple[ - Sequence[ResourceRecord], - Sequence[DeclarationRecord], - Sequence[UsageRecord], - Sequence[GrantRecord], - ], - mappings: _Mappings, - visibility: _Visibility, - native: object | None, -) -> ScopeLookup: - """Project native lookup groups over `records` as a `ScopeLookup` of canonical `index`.""" - - resources, declarations, usages, grants = records - global_positions, private_positions, local_positions, inherited_positions = visibility - return ScopeLookup( - index=index, - resources=_mapping(groups=mappings[0], records=resources, key=_resource_identity), - resources_by_path=_path_mapping(groups=mappings[1], records=resources, paths=mappings[6]), - declarations=_mapping(groups=mappings[2], records=declarations, key=_declaration_identity), - usages_by_consumer=_mapping(groups=mappings[3], records=usages, key=_usage_consumer), - usages_by_declaration=_mapping(groups=mappings[4], records=usages, key=_usage_declaration), - grants_by_resource=_mapping(groups=mappings[5], records=grants, key=_grant_resource), - visibility_index=DeclarationVisibilityIndex( - identities=tuple(record.identity for record in index.declarations), - global_positions=tuple(global_positions), - private_positions=MappingProxyType( - { - ResourceIdentity(ResourceKind(kind), name): tuple(positions) - for (kind, name), positions in private_positions - } - ), - local_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in local_positions} - ), - inherited_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in inherited_positions} - ), - ), - native=native, - ) - - -def _resource_identity(record: ResourceRecord) -> ResourceIdentity: - return record.identity - - -def _declaration_identity(record: DeclarationRecord) -> DeclarationIdentity: - return record.identity - - -def _usage_consumer(record: UsageRecord) -> ResourceIdentity | DeclarationIdentity: - return record.consumer - - -def _usage_declaration(record: UsageRecord) -> DeclarationIdentity: - return record.declaration - - -def _grant_resource(record: GrantRecord) -> ResourceIdentity: - return record.resource - - -def _path_mapping( - *, groups: _Groups, records: Sequence[ResourceRecord], paths: list[str] -) -> MappingProxyType[str, tuple[ResourceRecord, ...]]: - items: list[tuple[str, tuple[ResourceRecord, ...]]] = [] - for group in groups: - items.append((paths[group[0]], tuple(records[member] for member in group))) - return MappingProxyType(dict(items)) - - -def _mapping[Record, Key: Hashable]( - *, groups: _Groups, records: Sequence[Record], key: Callable[[Record], Key] -) -> MappingProxyType[Key, tuple[Record, ...]]: - items: list[tuple[Key, tuple[Record, ...]]] = [] - for group in groups: - grouped: tuple[Record, ...] = tuple(records[member] for member in group) - items.append((key(grouped[0]), grouped)) - return MappingProxyType(dict(items)) diff --git a/src/sqlbuild/compiler/scopes/_helpers/placement.py b/src/sqlbuild/compiler/scopes/_helpers/placement.py deleted file mode 100644 index bf749ea471..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/placement.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Validate declaration usage and exact lexical placement with the native placement rules.""" - -from __future__ import annotations - -from dataclasses import replace - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - OwnershipRoot, - ResourceIdentity, - ScopeDiagnostic, - ScopeIndex, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - DiagnosticSeverity, - NativePlacementRows, - NativeQualifiedRow, - NativeRootRow, - NativeUsageKeyRow, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, -) - -_PLACEMENT_CODES: frozenset[ScopeDiagnosticCode] = frozenset( - { - ScopeDiagnosticCode.UNUSED_DECLARATION, - ScopeDiagnosticCode.LOCAL_NEEDED_BY_DESCENDANT, - ScopeDiagnosticCode.OVER_BROAD_INHERITED, - ScopeDiagnosticCode.REQUIRES_GLOBAL_PLACEMENT, - ScopeDiagnosticCode.OVER_BROAD_GLOBAL, - } -) - - -def merged_usage_index(*, index: ScopeIndex, collected: tuple[UsageRecord, ...]) -> ScopeIndex: - """The index with compile usages merged into its usages and macro dependencies.""" - - distinct, merged, combined = _native.scope_merged_usages( - [_dependency_row(record=record) for record in index.declarations], - [_usage_row(usage=usage) for usage in index.usages], - [_usage_row(usage=usage) for usage in collected], - ) - usages: list[UsageRecord] = [collected[position] for position in distinct] - all_usages: list[UsageRecord] = [*index.usages, *usages] - return replace( - index, - declarations=tuple( - _with_dependencies(record=record, sources=sources, usages=usages) - for record, sources in zip(index.declarations, merged, strict=True) - ), - usages=tuple(all_usages[position] for position in combined), - completeness=replace(index.completeness, runtime_usage=True, promotion_impact=True), - ) - - -def _dependency_row( - *, record: DeclarationRecord -) -> tuple[NativeQualifiedRow, list[NativeQualifiedRow] | None]: - dependencies: tuple[DeclarationIdentity, ...] | None = ( - None if record.macro is None else record.macro.dependencies - ) - return ( - qualified_row(identity=record.identity), - None if dependencies is None else [qualified_row(identity=item) for item in dependencies], - ) - - -def _with_dependencies( - *, - record: DeclarationRecord, - sources: list[tuple[bool, int]] | None, - usages: list[UsageRecord], -) -> DeclarationRecord: - if record.macro is None or sources is None: - return record - own: tuple[DeclarationIdentity, ...] = record.macro.dependencies - return replace( - record, - macro=replace( - record.macro, - dependencies=tuple( - own[position] if is_own else usages[position].declaration - for is_own, position in sources - ), - ), - ) - - -def _usage_row(*, usage: UsageRecord) -> NativeUsageKeyRow: - return ( - qualified_row(identity=usage.consumer), - qualified_row(identity=usage.declaration), - usage.kind.value, - None if usage.through is None else qualified_row(identity=usage.through), - usage.enum_member, - ) - - -def build_placement_validated_index( - *, index: ScopeIndex, enforce_placement: bool = True -) -> ScopeIndex: - """Return the index with deterministic unused and exact-placement diagnostics.""" - - kept: list[ScopeDiagnostic] = [ - item for item in index.diagnostics if item.code not in _PLACEMENT_CODES - ] - if not index.completeness.runtime_usage: - return replace( - index, - diagnostics=tuple(kept), - completeness=replace(index.completeness, placement=False), - ) - severity: DiagnosticSeverity = ( - DiagnosticSeverity.ERROR if enforce_placement else DiagnosticSeverity.WARNING - ) - for position, code, message in _native.scope_placement_diagnostics(_rows(index=index)): - declaration: DeclarationRecord = index.declarations[position] - kept.append( - ScopeDiagnostic( - code=ScopeDiagnosticCode(code), - message=message, - path=declaration.path, - line=declaration.line, - column=declaration.column, - declaration=declaration.identity, - severity=severity, - ) - ) - return replace( - index, - diagnostics=tuple(sorted(kept, key=_diagnostic_key)), - completeness=replace(index.completeness, placement=True), - ) - - -def declaration_directory(*, kind: DeclarationKind, scope: ScopeKind, path: str | None) -> str: - """Return the project-relative folder where a declaration with this placement lives.""" - - return _native.scope_declaration_directory(kind.value, scope.value, path) - - -def relocate_for_resource_move( - *, index: ScopeIndex, resource: ResourceIdentity, destination: str -) -> tuple[DeclarationRecord, ...] | None: - """Return declarations whose placement changes when a resource moves, or None if unsettled.""" - - if not index.completeness.runtime_usage: - return None - relocations: list[tuple[int, str, str, NativeRootRow, str | None]] | None = ( - _native.scope_relocations( - _rows(index=index), (resource.kind.value, resource.name), destination - ) - ) - if relocations is None: - return None - return tuple( - replace( - index.declarations[position], - path=path, - scope=ScopeKind(scope), - ownership_root=OwnershipRoot( - root_path, - OwnershipRootKind(root_kind), - None if resource_kind is None else ResourceKind(resource_kind), - ), - owning_path=owning_path, - ) - for position, path, scope, (root_path, root_kind, resource_kind), owning_path in ( - relocations - ) - ) - - -def _rows(*, index: ScopeIndex) -> NativePlacementRows: - return ( - [ - ( - (record.identity.kind.value, record.identity.name), - record.path, - root_row(root=record.ownership_root), - ) - for record in index.resources - ], - [ - ( - qualified_row(identity=record.identity), - record.path, - record.line, - record.column, - record.scope.value, - root_row(root=record.ownership_root), - record.owning_path, - ) - for record in index.declarations - ], - [ - ( - qualified_row(identity=usage.consumer), - qualified_row(identity=usage.declaration), - None if usage.through is None else qualified_row(identity=usage.through), - ) - for usage in index.usages - ], - ) - - -def qualified_row(*, identity: ResourceIdentity | DeclarationIdentity) -> NativeQualifiedRow: - if isinstance(identity, ResourceIdentity) or identity.owner is None: - return identity.kind.value, identity.name, None - return ( - identity.kind.value, - identity.name, - (identity.owner.kind.value, identity.owner.name), - ) - - -def root_row(*, root: OwnershipRoot) -> NativeRootRow: - return ( - root.path, - root.kind.value, - None if root.resource_kind is None else root.resource_kind.value, - ) - - -def _diagnostic_key(item: ScopeDiagnostic) -> tuple[str, int, int, str, str]: - return (item.path or "", item.line or 0, item.column or 0, item.code.value, item.message) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_rows.py b/src/sqlbuild/compiler/scopes/_helpers/report_rows.py deleted file mode 100644 index e4e9aa297b..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/report_rows.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Report models and scope indexes as the rows native serialization and projection read.""" - -from __future__ import annotations - -from enum import Enum - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row -from sqlbuild.compiler.scopes._helpers.reports import filters_row, values_row -from sqlbuild.compiler.scopes.exceptions import ScopeError -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationReport, - MovePreview, - ScopeBrowseResult, - ScopeCompleteness, - ScopeDiagnostic, - ScopeFolder, - ScopeIndex, - ScopeListResult, - ScopeReport, - ScopeResourceReport, - ScopeSection, -) -from sqlbuild.compiler.scopes.types import ( - JsonValue, - NativeScopeDeclarationRow, - NativeScopeDiagnosticRow, - NativeScopeExplanationRow, - NativeScopeFolderRow, - NativeScopeMoveRow, - NativeScopeSectionRow, -) - - -def serialize_result(*, result: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: - """Deterministic schema-version-two ASCII JSON of one scope query result.""" - - try: - if isinstance(result, ScopeReport): - return _native.serialize_scope_result_row("report", _report_row(report=result)) - if isinstance(result, ScopeBrowseResult): - return _native.serialize_scope_result_row( - "browse", - ( - result.folder, - [_folder_row(folder=folder) for folder in result.folders], - _diagnostic_rows(diagnostics=result.diagnostics), - result.complete, - ), - ) - return _native.serialize_scope_result_row( - "list", - ( - result.folder, - _declaration_rows(reports=result.declarations), - _section_row(section=result.section), - _diagnostic_rows(diagnostics=result.diagnostics), - ), - ) - except ValueError as error: - raise ScopeError(str(error)) from error - - -def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: - """Deterministic value-free metadata of one canonical index.""" - - completeness: ScopeCompleteness = index.completeness - return _native.scope_metadata_projection_rows( - ( - [root_row(root=root) for root in index.ownership_roots], - [ - ( - (record.identity.kind.value, record.identity.name), - record.path, - root_row(root=record.ownership_root), - ) - for record in index.resources - ], - [ - ( - qualified_row(identity=record.identity), - record.path, - record.line, - record.column, - record.scope.value, - root_row(root=record.ownership_root), - record.owning_path, - ) - for record in index.declarations - ], - [values_row(record=record) for record in index.declarations], - [ - ( - qualified_row(identity=usage.consumer), - qualified_row(identity=usage.declaration), - usage.kind.value, - None if usage.through is None else qualified_row(identity=usage.through), - usage.enum_member, - ) - for usage in index.usages - ], - [ - ( - (grant.resource.kind.value, grant.resource.name), - qualified_row(identity=grant.declaration), - qualified_row(identity=grant.through), - grant.kind.value, - ) - for grant in index.grants - ], - [ - ( - (fact.resource.kind.value, fact.resource.name), - qualified_row(identity=fact.declaration), - fact.reason.value, - None if fact.through is None else qualified_row(identity=fact.through), - ) - for fact in index.visibility - ], - [ - ( - (fact.resource.kind.value, fact.resource.name), - qualified_row(identity=fact.declaration), - fact.reason.value, - None, - ) - for fact in index.inaccessible - ], - _diagnostic_rows(diagnostics=index.diagnostics), - ( - completeness.discovery, - completeness.static_visibility, - completeness.runtime_usage, - completeness.relationships, - completeness.placement, - completeness.promotion_impact, - ), - ) - ) - - -def _folder_row(*, folder: ScopeFolder) -> NativeScopeFolderRow: - return ( - folder.path, - folder.name, - folder.descendant_count, - folder.child_count, - folder.used_count, - [(str(kind), count) for kind, count in folder.kind_counts], - ) - - -def _text(value: object) -> str: - return str(value.value) if isinstance(value, Enum) else str(value) - - -def _diagnostic_rows(*, diagnostics: tuple[ScopeDiagnostic, ...]) -> list[NativeScopeDiagnosticRow]: - return [ - ( - _text(diagnostic.code), - diagnostic.message, - _text(diagnostic.severity), - diagnostic.path, - diagnostic.line, - diagnostic.column, - None - if diagnostic.declaration is None - else qualified_row(identity=diagnostic.declaration), - ( - None - if diagnostic.resource is None - else (diagnostic.resource.kind.value, diagnostic.resource.name) - ), - ) - for diagnostic in diagnostics - ] - - -def _declaration_row(report: DeclarationReport) -> NativeScopeDeclarationRow: - return ( - ( - report.identity, - report.kind, - report.name, - report.owner, - (report.definition.path, report.definition.line, report.definition.column), - report.scope, - report.owning_path, - ), - ( - ( - None - if report.visibility is None - else (report.visibility.reason, report.visibility.through) - ), - report.inaccessible_reason, - [(str(key), value) for key, value in report.metadata], - ), - ( - list(report.consumers), - list(report.dependencies), - list(report.grants), - report.required_scope, - report.required_path, - list(report.promotion_impact), - ), - ) - - -def _declaration_rows(*, reports: tuple[DeclarationReport, ...]) -> list[NativeScopeDeclarationRow]: - return [_declaration_row(report) for report in reports] - - -def _section_row(*, section: ScopeSection) -> NativeScopeSectionRow: - return ( - section.name, - section.total, - section.returned, - section.collapsed, - section.collapsed_count, - section.truncated, - section.complete, - section.next_cursor, - section.cursor, - section.page_size, - ) - - -def _explanation_row(*, explanation: DeclarationExplanation) -> NativeScopeExplanationRow: - return ( - None if explanation.declaration is None else _declaration_row(explanation.declaration), - explanation.complete, - ) - - -def _move_row(*, move: MovePreview) -> NativeScopeMoveRow: - return ( - move.resource, - move.destination, - move.new_ownership_root, - _declaration_rows(reports=move.retained), - _declaration_rows(reports=move.gained), - _declaration_rows(reports=move.lost), - list(move.invalidated_usages), - _declaration_rows(reports=move.private_retained), - _declaration_rows(reports=move.relationship_retained), - move.complete, - ) - - -def _report_row(*, report: ScopeReport) -> object: - resource: ScopeResourceReport = report.resource - return ( - ( - resource.target, - resource.identity, - resource.path, - resource.prospective, - resource.directory, - resource.duplicate_count, - ), - [(entry.kind, entry.path, entry.declaration_count) for entry in report.scope_chain], - _declaration_rows(reports=report.available), - _declaration_rows(reports=report.used), - _declaration_rows(reports=report.relationship_scope), - _declaration_rows(reports=report.nearby_unavailable), - filters_row(filters=report.filters), - [_section_row(section=section) for section in report.sections], - None if report.explanation is None else _explanation_row(explanation=report.explanation), - None if report.move_preview is None else _move_row(move=report.move_preview), - _diagnostic_rows(diagnostics=report.diagnostics), - report.complete, - ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/reports.py b/src/sqlbuild/compiler/scopes/_helpers/reports.py deleted file mode 100644 index a1ffb3a3ce..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/reports.py +++ /dev/null @@ -1,371 +0,0 @@ -"""Scope reports answered by the native scope report engine, projected as report models.""" - -from __future__ import annotations - -from pathlib import PurePath -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes._helpers.placement import qualified_row -from sqlbuild.compiler.scopes.exceptions import ( - InvalidQualifiedIdentityError, - InvalidScopePathError, -) -from sqlbuild.compiler.scopes.models import ( - ConstantMetadata, - DeclarationExplanation, - DeclarationIdentity, - DeclarationRecord, - DeclarationReport, - EnumMetadata, - MacroMetadata, - MovePreview, - ResourceIdentity, - ScopeBrowseResult, - ScopeChainEntry, - ScopeCompleteness, - ScopeDiagnostic, - ScopeFolder, - ScopeListResult, - ScopeLookup, - ScopeReport, - ScopeReportFilters, - ScopeResourceReport, - ScopeSection, - SourceLocation, - VisibilityProvenance, -) -from sqlbuild.compiler.scopes.types import ( - NativeScopeBrowseRow, - NativeScopeDeclarationRow, - NativeScopeExplanationRow, - NativeScopeFailureRow, - NativeScopeFiltersRow, - NativeScopeListRow, - NativeScopeMoveRow, - NativeScopeRaisedRow, - NativeScopeReportRow, - NativeScopeSectionRow, - NativeScopeTargetRow, - NativeScopeValuesRow, - ScopeDiagnosticCode, - ScopeQueryTarget, -) - -_FAILURES: dict[str, type[Exception]] = { - "identity": InvalidQualifiedIdentityError, - "path": InvalidScopePathError, - "index": IndexError, -} - - -def build_scope_report( - *, - lookup: ScopeLookup, - target: ScopeQueryTarget | None, - at: str | PurePath | None, - directory: bool, - filters: ScopeReportFilters | None, -) -> ScopeReport: - """Query one existing target or prospective path; invalid index paths raise.""" - - selected: ScopeReportFilters = filters or ScopeReportFilters() - row, failure = _reports(lookup).report( - ( - None if target is None else target_row(target=target), - None if at is None else str(at), - directory, - filters_row(filters=selected), - ) - ) - return _report(lookup=lookup, row=_checked(row=row, failure=failure), filters=selected) - - -def explain_declaration( - *, - lookup: ScopeLookup, - declaration: str | DeclarationIdentity, - target: str | PurePath | ResourceIdentity | None, - at: str | PurePath | None, - directory: bool, -) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: - """Explain one qualified declaration in an optional resource or path context.""" - - row, failure = _reports(lookup).explain( - ( - ( - (qualified_row(identity=declaration), "") - if isinstance(declaration, DeclarationIdentity) - else (None, str(declaration)) - ), - None if target is None else target_row(target=target), - None if at is None else str(at), - directory, - ) - ) - explanation, diagnostics = _checked(row=row, failure=failure) - return _explanation(row=explanation), _diagnostics(lookup=lookup, rows=diagnostics) - - -def browse_folders( - *, - lookup: ScopeLookup, - folder: str, - target: str | PurePath | ResourceIdentity | None, - target_is_prospective: bool, -) -> ScopeBrowseResult: - """The direct child declaration-definition folders of `folder`.""" - - folder_text, folders, diagnostics, complete = _reports(lookup).browse( - ( - folder, - None if target is None else target_row(target=target), - target_is_prospective, - None, - ) - ) - return _browse(lookup=lookup, row=(folder_text, folders, diagnostics, complete)) - - -def list_folder( - *, - lookup: ScopeLookup, - folder: str, - filters: ScopeReportFilters | None, - target: str | PurePath | ResourceIdentity | None, - target_is_prospective: bool, -) -> ScopeListResult: - """Declarations recursively beneath one definition folder.""" - - row, failure = _reports(lookup).list( - ( - folder, - None if target is None else target_row(target=target), - target_is_prospective, - None if filters is None else filters_row(filters=filters), - ) - ) - return _list(lookup=lookup, row=_checked(row=row, failure=failure)) - - -def build_move_preview( - *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath -) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: - """Visibility deltas of a resource move, without mutating the lookup.""" - - row, failure = _reports(lookup).move_preview((target_row(target=resource), str(destination))) - preview, diagnostics = _checked(row=row, failure=failure) - return ( - None if preview is None else _move(row=preview), - _diagnostics(lookup=lookup, rows=diagnostics), - ) - - -def target_row(*, target: ScopeQueryTarget) -> NativeScopeTargetRow: - """A query target as the native engine reads it.""" - - if isinstance(target, (ResourceIdentity, DeclarationIdentity)): - return qualified_row(identity=target), "", False - return None, str(target), isinstance(target, PurePath) - - -def filters_row(*, filters: ScopeReportFilters) -> NativeScopeFiltersRow: - """Report filters as the native engine reads them.""" - - return ( - list(filters.sections), - filters.include_nearby, - filters.defined_under, - [str(kind) for kind in filters.kinds], - filters.glob, - filters.used_only, - filters.dependency_depth, - filters.sort, - filters.cursor, - filters.page_size, - filters.nearby_depth, - filters.globals, - ) - - -def values_row(*, record: DeclarationRecord) -> NativeScopeValuesRow: - """One declaration's value metadata as the native engine reads it.""" - - macro: MacroMetadata | None = record.macro - enum: EnumMetadata | None = record.enum - constant: ConstantMetadata | None = record.constant - return ( - None - if macro is None - else ( - list(macro.parameters), - [qualified_row(identity=item) for item in macro.dependencies], - macro.source_digest, - ), - None if enum is None else ([member.name for member in enum.members], enum.scalar_type), - None - if constant is None - else ( - constant.logical_type, - constant.collection_kind, - constant.item_count, - constant.nullable, - constant.render_as, - ), - ) - - -def _reports(lookup: ScopeLookup) -> _native.NativeScopeReports: - completeness: ScopeCompleteness = lookup.index.completeness - return _native.build_native_scope_reports( - cast(_native.NativeScopeLookup, lookup.native), - ( - [values_row(record=record) for record in lookup.index.declarations], - ( - completeness.discovery, - completeness.static_visibility, - completeness.runtime_usage, - completeness.relationships, - completeness.placement, - completeness.promotion_impact, - ), - ), - ) - - -def _checked[Row](*, row: Row | None, failure: NativeScopeFailureRow) -> Row: - if failure is not None: - kind, message = failure - raise _FAILURES[kind](message) - return cast("Row", row) - - -def _diagnostics( - *, lookup: ScopeLookup, rows: list[NativeScopeRaisedRow] -) -> tuple[ScopeDiagnostic, ...]: - diagnostics: list[ScopeDiagnostic] = [] - for position, raised in rows: - if position is not None: - diagnostics.append(lookup.index.diagnostics[position]) - continue - code, message, path = cast("tuple[str, str, str | None]", raised) - diagnostics.append(ScopeDiagnostic(ScopeDiagnosticCode(code), message, path=path)) - return tuple(diagnostics) - - -def _declaration(row: NativeScopeDeclarationRow) -> DeclarationReport: - ( - (identity, kind, name, owner, (path, line, column), scope, owning_path), - (visibility, inaccessible_reason, metadata), - (consumers, dependencies, grants, required_scope, required_path, promotion_impact), - ) = row - return DeclarationReport( - identity=identity, - kind=kind, - name=name, - owner=owner, - definition=SourceLocation(path, line, column), - scope=scope, - owning_path=owning_path, - visibility=None if visibility is None else VisibilityProvenance(*visibility), - inaccessible_reason=inaccessible_reason, - metadata=tuple(metadata), - consumers=tuple(consumers), - dependencies=tuple(dependencies), - grants=tuple(grants), - required_scope=required_scope, - required_path=required_path, - promotion_impact=tuple(promotion_impact), - ) - - -def _declarations(rows: list[NativeScopeDeclarationRow]) -> tuple[DeclarationReport, ...]: - return tuple(_declaration(row) for row in rows) - - -def _section(row: NativeScopeSectionRow) -> ScopeSection: - return ScopeSection(*row) - - -def _explanation(*, row: NativeScopeExplanationRow) -> DeclarationExplanation: - declaration, complete = row - return DeclarationExplanation( - None if declaration is None else _declaration(declaration), complete - ) - - -def _move(*, row: NativeScopeMoveRow) -> MovePreview: - resource, destination, root, retained, gained, lost, invalidated, private, related, done = row - return MovePreview( - resource=resource, - destination=destination, - new_ownership_root=root, - retained=_declarations(retained), - gained=_declarations(gained), - lost=_declarations(lost), - invalidated_usages=tuple(invalidated), - private_retained=_declarations(private), - relationship_retained=_declarations(related), - complete=done, - ) - - -def _report( - *, - lookup: ScopeLookup, - row: NativeScopeReportRow[NativeScopeRaisedRow], - filters: ScopeReportFilters, -) -> ScopeReport: - ( - resource, - chain, - available, - used, - relationship_scope, - nearby, - _filters, - sections, - explanation, - move, - diagnostics, - complete, - ) = row - return ScopeReport( - resource=ScopeResourceReport(*resource), - scope_chain=tuple(ScopeChainEntry(*entry) for entry in chain), - available=_declarations(available), - used=_declarations(used), - relationship_scope=_declarations(relationship_scope), - nearby_unavailable=_declarations(nearby), - filters=filters, - sections=tuple(_section(section) for section in sections), - explanation=None if explanation is None else _explanation(row=explanation), - move_preview=None if move is None else _move(row=move), - diagnostics=_diagnostics(lookup=lookup, rows=diagnostics), - complete=complete, - ) - - -def _browse( - *, lookup: ScopeLookup, row: NativeScopeBrowseRow[NativeScopeRaisedRow] -) -> ScopeBrowseResult: - folder, folders, diagnostics, complete = row - return ScopeBrowseResult( - folder, - tuple( - ScopeFolder(path, name, descendants, children, used, tuple(kinds)) - for path, name, descendants, children, used, kinds in folders - ), - _diagnostics(lookup=lookup, rows=diagnostics), - complete, - ) - - -def _list(*, lookup: ScopeLookup, row: NativeScopeListRow[NativeScopeRaisedRow]) -> ScopeListResult: - folder, declarations, section, diagnostics = row - return ScopeListResult( - folder, - _declarations(declarations), - _section(section), - _diagnostics(lookup=lookup, rows=diagnostics), - ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/visibility.py b/src/sqlbuild/compiler/scopes/_helpers/visibility.py deleted file mode 100644 index 384014ca31..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/visibility.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Declaration visibility over a scope lookup, resolved by the native scope lookup.""" - -from __future__ import annotations - -from pathlib import PurePath -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row -from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, PATH_SEPARATOR -from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError, InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - InaccessibleRecord, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeLookup, - ScopeTargetQuery, - VisibilityRecord, - VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - InaccessibleReason, - NativeResolvedRows, - NativeResourceRecordRow, - OwnershipRootKind, - ResourceKind, - VisibilityReason, -) - -_DECLARATION_STEM: str = "__declaration__.sql" - - -def query_target( - *, lookup: ScopeLookup, target: ResourceIdentity | DeclarationIdentity | str | PurePath -) -> ScopeTargetQuery: - """The resources or declarations `target` names; invalid text raises the scope error.""" - - identity: bool = isinstance(target, (ResourceIdentity, DeclarationIdentity)) - raw: str = "" if identity else str(target) - declarations, positions, identity_error, path_error = _lookup(lookup).target( - qualified_row(identity=cast("ResourceIdentity | DeclarationIdentity", target)) - if identity - else None, - raw, - ) - if identity_error is not None: - raise InvalidQualifiedIdentityError(identity_error) - if path_error is not None: - raise InvalidScopePathError(path_error) - value: str | ResourceIdentity | DeclarationIdentity = ( - cast("ResourceIdentity | DeclarationIdentity", target) if identity else raw - ) - if declarations: - return ScopeTargetQuery( - value, - declaration_matches=tuple(lookup.index.declarations[item] for item in positions), - ) - return ScopeTargetQuery(value, tuple(lookup.index.resources[item] for item in positions)) - - -def resolve_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> VisibilityResolution: - """Every visible fact and inaccessible declaration, with reasons, of each matched resource.""" - - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - visible: list[VisibilityRecord] = [] - inaccessible: list[InaccessibleRecord] = [] - for resource in query.matches: - resource_visible, resource_inaccessible = _resolve(lookup=lookup, resource=resource) - visible.extend(resource_visible) - inaccessible.extend( - InaccessibleRecord( - resource.identity, - lookup.index.declarations[position].identity, - InaccessibleReason(reason), - ) - for position, reason in resource_inaccessible - ) - return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) - - -def resolve_declaration_visibility( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> DeclarationVisibility: - """Resolve visible facts and inaccessible identities without per-pair inaccessible reasons.""" - - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - visible: list[VisibilityRecord] = [] - inaccessible: list[DeclarationIdentity] = [] - for resource in query.matches: - resource_visible, resource_inaccessible = _resolve(lookup=lookup, resource=resource) - visible.extend(resource_visible) - inaccessible.extend( - lookup.index.declarations[position].identity for position, _ in resource_inaccessible - ) - return DeclarationVisibility(query, tuple(visible), tuple(inaccessible)) - - -def resolve_path_visibility( - *, lookup: ScopeLookup, path: str | PurePath -) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: - """Classify declarations for an authored path, including declaration definition files.""" - - normalized_path: str = normalize_path(path=path) - resource: ResourceRecord = ResourceRecord( - identity=ResourceIdentity(ResourceKind.MODEL, f""), - path=normalized_path, - ownership_root=OwnershipRoot( - path=CURRENT_PATH_COMPONENT, - kind=OwnershipRootKind.RESOURCE, - resource_kind=ResourceKind.MODEL, - ), - ) - visible, inaccessible, error = _resolved_rows(lookup=lookup, resource=resource) - if error is not None: - raise InvalidScopePathError(error) - declarations: tuple[DeclarationRecord, ...] = lookup.index.declarations - return ( - tuple(declarations[position] for position, _, _ in visible), - tuple(declarations[position] for position, _ in inaccessible), - ) - - -def declaration_lexical_path(*, record: DeclarationRecord) -> str: - """Return the authored path whose folder controls what a declaration file can use.""" - - return f"{record.owning_path or CURRENT_PATH_COMPONENT}{PATH_SEPARATOR}{_DECLARATION_STEM}" - - -def _resolve( - *, lookup: ScopeLookup, resource: ResourceRecord -) -> tuple[list[VisibilityRecord], list[tuple[int, str]]]: - visible, inaccessible, error = _resolved_rows(lookup=lookup, resource=resource) - if error is not None: - raise InvalidScopePathError(error) - identities: tuple[DeclarationIdentity, ...] = lookup.visibility_index.identities - return ( - [ - VisibilityRecord( - resource.identity, - identities[position], - VisibilityReason(reason), - None if grant is None else lookup.index.grants[grant].through, - ) - for position, reason, grant in visible - ], - inaccessible, - ) - - -def _resolved_rows(*, lookup: ScopeLookup, resource: ResourceRecord) -> NativeResolvedRows: - return _lookup(lookup).resolve(_resource_row(resource=resource)) - - -def _resource_row(*, resource: ResourceRecord) -> NativeResourceRecordRow: - return ( - (resource.identity.kind.value, resource.identity.name), - resource.path, - root_row(root=resource.ownership_root), - ) - - -def _lookup(lookup: ScopeLookup) -> _native.NativeScopeLookup: - return cast(_native.NativeScopeLookup, lookup.native) diff --git a/src/sqlbuild/compiler/scopes/classes/__init__.py b/src/sqlbuild/compiler/scopes/classes/__init__.py deleted file mode 100644 index 69ee566cdc..0000000000 --- a/src/sqlbuild/compiler/scopes/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler scope runtime classes.""" diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py deleted file mode 100644 index d2917488f4..0000000000 --- a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py +++ /dev/null @@ -1,214 +0,0 @@ -"""A natively built scope index, materialised as the Python scope models it replaces.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile -from sqlbuild.compiler.scopes._helpers.lookup import build_lookup -from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows -from sqlbuild.compiler.scopes.exceptions import UnindexableScopeError -from sqlbuild.compiler.scopes.models import ( - DeclarationRecord, - GrantRecord, - NativeDeclarationValues, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeCompleteness, - ScopeDiagnostic, - ScopeIndex, - ScopeLookup, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - GrantKind, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - UsageKind, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -type _IndexRows = tuple[ - list[str], - list[tuple[str, str, str, str | None, str | None]], - list[int], - list[int], - list[tuple[int, int]], - list[tuple[str, str, str | None, int | None, int | None, int | None, int | None]], - bool, -] -type _GrantRows = list[tuple[str, str, int, str | int, str]] -_RESOURCE_ROOTS: dict[ResourceKind, str] = { - ResourceKind.MODEL: "models", - ResourceKind.TEST: "tests/unit", - ResourceKind.SCENARIO: "tests/scenarios", - ResourceKind.FUNCTION: "functions/sql", - ResourceKind.SOURCE: "sources", -} -_FIXED_ROOTS: dict[str, OwnershipRoot] = { - "seed": OwnershipRoot("seeds", OwnershipRootKind.GLOBAL, ResourceKind.SEED), - "python_function": OwnershipRoot( - "functions/python", OwnershipRootKind.GLOBAL, ResourceKind.FUNCTION - ), -} -_STATIC_COMPLETENESS: ScopeCompleteness = ScopeCompleteness( - runtime_usage=False, relationships=False, placement=False, promotion_impact=False -) - - -class NativeScopeIndex: - """One native scope index and the Python records materialised from it.""" - - def __init__(self, *, native: _native.NativeScopeIndex, rows: NativeScopeRows) -> None: - records: _IndexRows = native.records() - paths, declaration_rows, resource_order, declaration_order, usages, diagnostics, scoped = ( - records - ) - self._native: _native.NativeScopeIndex = native - self._resources: list[ResourceRecord] = [ - ResourceRecord( - identity=identity, - path=path, - ownership_root=_resource_root(identity=identity, root=root), - ) - for identity, path, root in zip( - rows.resource_identities, paths, rows.resource_roots, strict=True - ) - ] - self._declarations: list[DeclarationRecord] = [ - _declaration_record(values=values, row=row) - for values, row in zip(rows.declaration_values, declaration_rows, strict=True) - ] - self._usages: list[UsageRecord] = [ - UsageRecord( - consumer=self._declarations[consumer].identity, - declaration=rows.declaration_values[consumer].dependencies[dependency], - kind=UsageKind.DECLARATION_DEPENDENCY, - ) - for consumer, dependency in usages - ] - self._grants: list[GrantRecord] = [] - self.has_scoped_relationship_declarations: bool = scoped - self.index: ScopeIndex = ScopeIndex( - ownership_roots=tuple( - OwnershipRoot(path=path, resource_kind=kind) - for kind, path in sorted(_RESOURCE_ROOTS.items(), key=lambda item: item[1]) - ), - resources=tuple(self._resources[position] for position in resource_order), - declarations=tuple(self._declarations[position] for position in declaration_order), - usages=tuple(self._usages), - diagnostics=tuple( - ScopeDiagnostic( - ScopeDiagnosticCode(code), - message, - path=path, - line=line, - column=column, - declaration=( - None if declaration is None else self._declarations[declaration].identity - ), - resource=None if resource is None else self._resources[resource].identity, - ) - for code, message, path, line, column, declaration, resource in diagnostics - ), - completeness=_STATIC_COMPLETENESS, - ) - - def grant_relationships( - self, - *, - discovered_inputs: DiscoveredProjectInputs, - syntax: SqlLexicalSyntax, - scan_store: tuple[_native.SqlTestScanStore | None, str], - ) -> tuple[tuple[Path, str], ...]: - """Grant every test's and scenario's relationships; return each extraction fault.""" - - store, algorithm = scan_store - rows, faults = self._native.grant_relationships( - [_test_file_row(test_file) for test_file in discovered_inputs.test_files], - [ - (scenario.name, str(scenario.relative_path), scenario.sql_body) - for scenario in discovered_inputs.scenario_files - ], - { - "syntax": syntax.native_mapping, - "store": store, - "scan_key": (algorithm, syntax.cache_key), - }, - ) - if rows is None: - raise UnindexableScopeError - identities: dict[tuple[str, str], ResourceIdentity] = {} - self._grants = [ - GrantRecord( - resource=identities.setdefault( - (kind, name), ResourceIdentity(ResourceKind(kind), name) - ), - declaration=self._declarations[declaration].identity, - through=( - ResourceIdentity(ResourceKind.MODEL, through) - if isinstance(through, str) - else self._declarations[through].identity - ), - kind=GrantKind(grant_kind), - ) - for kind, name, declaration, through, grant_kind in rows - ] - return tuple((Path(path), message) for path, message in faults) - - def index_with_relationships(self) -> ScopeIndex: - """Return the index with its grants attached and relationships complete.""" - - return replace( - self.index, - grants=tuple(self._grants), - completeness=replace(self.index.completeness, relationships=True), - ) - - def lookup(self, *, index: ScopeIndex) -> ScopeLookup: - """The native scope lookup of `index_with_relationships()`.""" - - return build_lookup(index=index) - - -def _test_file_row( - test_file: DiscoveredSqlTestFile, -) -> tuple[str, list[tuple[str, str, str]]]: - stem: str = test_file.relative_path.stem - return ( - str(test_file.relative_path), - [(block.name or stem, block.sql_body, block.mode.value) for block in test_file.blocks], - ) - - -def _resource_root(*, identity: ResourceIdentity, root: str) -> OwnershipRoot: - fixed: OwnershipRoot | None = _FIXED_ROOTS.get(root) - if fixed is not None: - return fixed - return OwnershipRoot(path=_RESOURCE_ROOTS[identity.kind], resource_kind=identity.kind) - - -def _declaration_record( - *, values: NativeDeclarationValues, row: tuple[str, str, str, str | None, str | None] -) -> DeclarationRecord: - path, root_path, root_kind, root_resource_kind, owning_path = row - return DeclarationRecord( - identity=values.identity, - path=path, - line=values.line, - column=1, - scope=values.scope, - ownership_root=OwnershipRoot( - root_path, - OwnershipRootKind(root_kind), - None if root_resource_kind is None else ResourceKind(root_resource_kind), - ), - owning_path=owning_path, - macro=values.macro, - enum=values.enum, - constant=values.constant, - ) diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py b/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py deleted file mode 100644 index c488587ab5..0000000000 --- a/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py +++ /dev/null @@ -1,402 +0,0 @@ -"""The builder's walk of discovered inputs as the plain rows the native scope index reads.""" - -from __future__ import annotations - -import hashlib -import inspect -from collections.abc import Mapping -from pathlib import Path -from types import CodeType - -from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredAuditFile, - DiscoveredHookFunction, - DiscoveredMacroFile, - DiscoveredModelSchemaFile, - DiscoveredProjectInputs, - DiscoveredSqlHookFile, - EnumDeclaration, -) -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.models import ( - ConstantMetadata, - DeclarationIdentity, - EnumMemberMetadata, - EnumMetadata, - MacroMetadata, - NativeDeclarationValues, - ResourceIdentity, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - NativeDeclarationRow, - NativeIdentityRow, - NativeResourceRow, - ResourceKind, - ScopeKind, -) -from sqlbuild.sql_values.types import SqlValueKind - -_KIND_ROOT: str = "kind" -_SEED_ROOT: str = "seed" -_PYTHON_FUNCTION_ROOT: str = "python_function" -_PRIVATE_ROOT: str = "models" -_MACRO_FALLBACK: str = "macros" -_ENUM_FALLBACK: str = "enums" -_CONSTANT_FALLBACK: str = "constants" -_COLLECTION_KINDS: frozenset[SqlValueKind] = frozenset( - {SqlValueKind.LIST, SqlValueKind.SET, SqlValueKind.OBJECT} -) - - -class NativeScopeRows: - """Resource and declaration rows in the Python builder's walk order, without records.""" - - def __init__( - self, - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro], - static_macros: tuple[StaticMacroExport, ...] = (), - ) -> None: - self.resources: list[NativeResourceRow] = [] - self.resource_identities: list[ResourceIdentity] = [] - self.resource_roots: list[str] = [] - self.declarations: list[NativeDeclarationRow] = [] - self.declaration_values: list[NativeDeclarationValues] = [] - self._add_models(discovered_inputs=discovered_inputs) - self._add_resources(discovered_inputs=discovered_inputs) - self._add_declaration_files(discovered_inputs=discovered_inputs) - self._add_named_declarations(discovered_inputs=discovered_inputs) - self._add_macros(discovered_inputs=discovered_inputs, loaded_macros=loaded_macros) - self._add_static_macros(discovered_inputs=discovered_inputs, static_macros=static_macros) - - def _add_resource(self, *, kind: ResourceKind, name: str, path: Path, root: str) -> None: - self.resources.append((kind.value, name, str(path), root)) - self.resource_identities.append(ResourceIdentity(kind, name)) - self.resource_roots.append(root) - - def _add_declaration( - self, - *, - values: NativeDeclarationValues, - path: Path | str, - ownership_root: Path | str | None, - root_fallback: Path | str, - owning_path: Path | str | None, - ) -> None: - identity: DeclarationIdentity = values.identity - self.declarations.append( - ( - identity.kind.value, - identity.name, - _identity_row(identity.owner), - str(path), - values.line, - values.scope.value, - None if ownership_root is None else str(ownership_root), - str(root_fallback), - None if owning_path is None else str(owning_path), - [ - (item.kind.value, item.name, _identity_row(item.owner)) - for item in values.dependencies - ], - ) - ) - self.declaration_values.append(values) - - def _add_models(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: - for model_file in discovered_inputs.model_files: - model_name: str = _name_or_stem( - value=model_file.header_values.get("name"), path=model_file.relative_path - ) - self._add_resource( - kind=ResourceKind.MODEL, - name=model_name, - path=model_file.relative_path, - root=_KIND_ROOT, - ) - owner: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - for enum in model_file.enum_declarations: - self._add_private(declaration=enum, owner=owner, path=model_file.relative_path) - for constant in model_file.constant_declarations: - self._add_private(declaration=constant, owner=owner, path=model_file.relative_path) - - def _add_private( - self, - *, - declaration: EnumDeclaration | ConstantDeclaration, - owner: ResourceIdentity, - path: Path, - ) -> None: - values: NativeDeclarationValues = ( - NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner), - line=1, - scope=ScopeKind.PRIVATE, - enum=_enum_metadata(declaration), - ) - if isinstance(declaration, EnumDeclaration) - else NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner), - line=1, - scope=ScopeKind.PRIVATE, - constant=_constant_metadata(declaration), - ) - ) - self._add_declaration( - values=values, - path=path, - ownership_root=_PRIVATE_ROOT, - root_fallback=_PRIVATE_ROOT, - owning_path=path.parent, - ) - - def _add_resources(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: - for test_file in discovered_inputs.test_files: - for block in test_file.blocks: - self._add_resource( - kind=ResourceKind.TEST, - name=block.name or test_file.relative_path.stem, - path=test_file.relative_path, - root=_KIND_ROOT, - ) - for scenario_file in discovered_inputs.scenario_files: - self._add_resource( - kind=ResourceKind.SCENARIO, - name=scenario_file.name, - path=scenario_file.relative_path, - root=_KIND_ROOT, - ) - for function_file in discovered_inputs.sql_function_files: - self._add_resource( - kind=ResourceKind.FUNCTION, - name=_name_or_stem( - value=function_file.header_values.get("name"), - path=function_file.relative_path, - ), - path=function_file.relative_path, - root=_KIND_ROOT, - ) - for python_function_file in discovered_inputs.python_function_files: - self._add_resource( - kind=ResourceKind.FUNCTION, - name=python_function_file.file_path.stem, - path=python_function_file.relative_path, - root=_PYTHON_FUNCTION_ROOT, - ) - for seed_file in discovered_inputs.seed_files: - self._add_resource( - kind=ResourceKind.SEED, - name=seed_file.file_path.stem, - path=seed_file.relative_path, - root=_SEED_ROOT, - ) - for source_file in discovered_inputs.source_files: - for source in source_file.source_entries: - self._add_resource( - kind=ResourceKind.SOURCE, - name=source.name, - path=source_file.relative_path, - root=_KIND_ROOT, - ) - - def _add_declaration_files(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: - for enum_file in discovered_inputs.enum_files: - for enum in enum_file.declarations: - self._add_declaration( - values=NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.ENUM, enum.name), - line=1, - scope=enum_file.scope_kind, - enum=_enum_metadata(enum), - ), - path=enum_file.relative_path, - ownership_root=enum_file.ownership_root, - root_fallback=_ENUM_FALLBACK, - owning_path=enum_file.owning_path, - ) - for constant_file in discovered_inputs.constant_files: - for constant in constant_file.declarations: - self._add_declaration( - values=NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.CONSTANT, constant.name), - line=1, - scope=constant_file.scope_kind, - constant=_constant_metadata(constant), - ), - path=constant_file.relative_path, - ownership_root=constant_file.ownership_root, - root_fallback=_CONSTANT_FALLBACK, - owning_path=constant_file.owning_path, - ) - - def _add_named_declarations(self, *, discovered_inputs: DiscoveredProjectInputs) -> None: - for audit_file in discovered_inputs.audit_files: - if audit_file.declaration_kind is DeclarationKind.AUDIT: - self._add_named( - item=audit_file, - kind=DeclarationKind.AUDIT, - name=audit_file.relative_path.stem, - path=audit_file.relative_path, - ) - continue - for block in audit_file.blocks: - self._add_named( - item=audit_file, - kind=DeclarationKind.SINGULAR_AUDIT, - name=block.name or audit_file.relative_path.stem, - path=audit_file.relative_path, - ) - for schema_file in discovered_inputs.model_schema_files: - for declaration in schema_file.declarations: - self._add_named( - item=schema_file, - kind=DeclarationKind.SCHEMA, - name=declaration.name, - path=schema_file.relative_path, - ) - for hook_file in discovered_inputs.sql_hook_files: - self._add_named( - item=hook_file, - kind=DeclarationKind.SQL_HOOK, - name=hook_file.name, - path=hook_file.relative_path, - ) - for hook_function in discovered_inputs.hook_functions: - self._add_named( - item=hook_function, - kind=DeclarationKind.PYTHON_HOOK, - name=hook_function.name, - path=hook_function.relative_path, - ) - - def _add_named( - self, - *, - item: DiscoveredAuditFile - | DiscoveredModelSchemaFile - | DiscoveredSqlHookFile - | DiscoveredHookFunction, - kind: DeclarationKind, - name: str, - path: Path, - ) -> None: - self._add_declaration( - values=NativeDeclarationValues( - identity=DeclarationIdentity(kind, name), line=1, scope=item.scope_kind - ), - path=path, - ownership_root=item.ownership_root, - root_fallback=item.declaration_root or item.relative_path.parent, - owning_path=item.owning_path, - ) - - def _add_macros( - self, - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro], - ) -> None: - macro_files: dict[str, DiscoveredMacroFile] = { - normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files - } - for loaded in loaded_macros.values(): - discovered: DiscoveredMacroFile | None = macro_files.get( - normalize_path(path=loaded.relative_path) - ) - self._add_declaration( - values=NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.MACRO, loaded.name), - line=_macro_line(loaded), - scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, - macro=_macro_metadata(loaded), - dependencies=loaded.dependencies, - ), - path=loaded.relative_path, - ownership_root=discovered.ownership_root if discovered is not None else None, - root_fallback=_MACRO_FALLBACK, - owning_path=discovered.owning_path if discovered is not None else None, - ) - - def _add_static_macros( - self, - *, - discovered_inputs: DiscoveredProjectInputs, - static_macros: tuple[StaticMacroExport, ...], - ) -> None: - macro_files: dict[str, DiscoveredMacroFile] = { - normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files - } - for export in static_macros: - discovered: DiscoveredMacroFile | None = macro_files.get( - normalize_path(path=export.relative_path) - ) - self._add_declaration( - values=NativeDeclarationValues( - identity=DeclarationIdentity(DeclarationKind.MACRO, export.name), - line=export.line, - scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, - macro=MacroMetadata( - parameters=export.parameters, - dependencies=export.dependencies, - source_digest=export.source_digest, - ), - dependencies=export.dependencies, - ), - path=export.relative_path, - ownership_root=discovered.ownership_root if discovered is not None else None, - root_fallback=_MACRO_FALLBACK, - owning_path=discovered.owning_path if discovered is not None else None, - ) - - -def _identity_row(identity: ResourceIdentity | None) -> NativeIdentityRow | None: - return None if identity is None else (identity.kind.value, identity.name) - - -def _name_or_stem(*, value: object, path: Path) -> str: - return value if isinstance(value, str) and value else path.stem - - -def _enum_metadata(declaration: EnumDeclaration) -> EnumMetadata: - """Return the member names and scalar type of one enum.""" - - return EnumMetadata( - members=tuple(EnumMemberMetadata(item.name) for item in declaration.members), - scalar_type=declaration.scalar_type, - ) - - -def _constant_metadata(declaration: ConstantDeclaration) -> ConstantMetadata: - """Return the logical type and collection shape of one constant.""" - - is_collection: bool = declaration.value.kind in _COLLECTION_KINDS - payload: object = declaration.value.value - item_count: int | None = len(payload) if is_collection and isinstance(payload, tuple) else None - return ConstantMetadata( - logical_type=declaration.logical_type.display_name, - collection_kind=declaration.value.kind.value if is_collection else None, - item_count=item_count, - nullable=declaration.value.kind is SqlValueKind.NULL, - render_as=declaration.render_as.value if declaration.render_as is not None else None, - ) - - -def _macro_metadata(loaded: LoadedMacro) -> MacroMetadata: - """Return the parameters, dependencies and source digest of one loaded macro.""" - - return MacroMetadata( - parameters=tuple(inspect.signature(loaded.function).parameters), - dependencies=loaded.dependencies, - source_digest=hashlib.sha256(loaded.raw_source.encode()).hexdigest(), - ) - - -def _macro_line(loaded: LoadedMacro) -> int: - """Return the first line of a loaded macro's function, or 1 without code.""" - - code: object = getattr(loaded.function, "__code__", None) - return code.co_firstlineno if isinstance(code, CodeType) else 1 diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index c00c1a6d13..1e90b63f1d 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -12,6 +12,7 @@ PATH_SEPARATOR: str = "/" WINDOWS_PATH_SEPARATOR: str = "\\" CURRENT_PATH_COMPONENT: str = "." +DECLARATION_LEXICAL_STEM: str = "__declaration__.sql" PARENT_PATH_COMPONENT: str = ".." WINDOWS_DRIVE_SEPARATOR: str = ":" WINDOWS_DRIVE_PREFIX_LENGTH: int = 2 diff --git a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py b/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py index 3e6a13994e..7856678e28 100644 --- a/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py +++ b/src/sqlbuild/compiler/scopes/main/_declaration_lexical_path.py @@ -2,11 +2,17 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers import visibility +from sqlbuild.compiler.scopes.constants import ( + CURRENT_PATH_COMPONENT, + DECLARATION_LEXICAL_STEM, + PATH_SEPARATOR, +) from sqlbuild.compiler.scopes.models import DeclarationRecord def declaration_lexical_path(*, record: DeclarationRecord) -> str: """Return the authored path whose folder controls what a declaration file can use.""" - return visibility.declaration_lexical_path(record=record) + return ( + f"{record.owning_path or CURRENT_PATH_COMPONENT}{PATH_SEPARATOR}{DECLARATION_LEXICAL_STEM}" + ) diff --git a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py index f5cb4fd0af..08cd685100 100644 --- a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py @@ -2,10 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.placement import ( - build_placement_validated_index, - merged_usage_index, -) +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord @@ -17,6 +14,4 @@ def get_placement_validated_scope_index( ) -> ScopeIndex: """Return complete usage and placement diagnostics, merging any collected compile usages.""" - if compile_usages is not None: - index = merged_usage_index(index=index, collected=compile_usages) - return build_placement_validated_index(index=index, enforce_placement=enforce_placement) + return _native.scope_placement_validated_index(index, enforce_placement, compile_usages) diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py index d0ad64a8ba..2679b5344f 100644 --- a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py @@ -7,9 +7,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex -from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows -from sqlbuild.compiler.scopes.exceptions import UnindexableScopeError def open_native_scope_index( @@ -17,17 +14,7 @@ def open_native_scope_index( discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro], static_macros: tuple[StaticMacroExport, ...] = (), -) -> NativeScopeIndex: +) -> _native.NativeScopeIndex: """Return the natively built index; discovery's normalized paths always index.""" - rows: NativeScopeRows = NativeScopeRows( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - static_macros=static_macros, - ) - native: _native.NativeScopeIndex | None = _native.build_native_scope_index( - rows.resources, rows.declarations - ) - if native is None: - raise UnindexableScopeError - return NativeScopeIndex(native=native, rows=rows) + return _native.open_native_scope_index(discovered_inputs, loaded_macros, static_macros) diff --git a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py new file mode 100644 index 0000000000..d1ab0243c3 --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py @@ -0,0 +1,21 @@ +"""Resolve what one scope target names.""" + +from __future__ import annotations + +from pathlib import PurePath + +import sqlbuild._native as _native +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + ResourceIdentity, + ScopeLookup, + ScopeTargetQuery, +) + + +def query_scope_target( + *, lookup: ScopeLookup, target: ResourceIdentity | DeclarationIdentity | str | PurePath +) -> ScopeTargetQuery: + """The resources or declarations `target` names; invalid text raises the scope error.""" + + return _native.scope_query_target(lookup, target) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py index d4fe55c279..427b4814f6 100644 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py +++ b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.visibility import resolve_declaration_visibility +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import DeclarationVisibility, ResourceIdentity, ScopeLookup @@ -13,4 +13,4 @@ def resolve_scope_declaration_visibility( ) -> DeclarationVisibility: """Return visible facts and inaccessible declaration identities for one target.""" - return resolve_declaration_visibility(lookup=lookup, target=target) + return _native.scope_resolve_visibility(lookup, target, False) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py index bc1bd0a551..acb0dbf959 100644 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py +++ b/src/sqlbuild/compiler/scopes/main/_resolve_scope_path_visibility.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.visibility import resolve_path_visibility +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeLookup @@ -13,4 +13,4 @@ def resolve_scope_path_visibility( ) -> tuple[tuple[DeclarationRecord, ...], tuple[DeclarationRecord, ...]]: """Return visible and inaccessible declarations for an authored path.""" - return resolve_path_visibility(lookup=lookup, path=path) + return _native.scope_resolve_path_visibility(lookup, path) diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py index ba3f801909..098231c4dc 100644 --- a/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py +++ b/src/sqlbuild/compiler/scopes/main/_resolve_scope_visibility.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.visibility import resolve_visibility +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeLookup, VisibilityResolution @@ -13,4 +13,4 @@ def resolve_scope_visibility( ) -> VisibilityResolution: """Return visible and inaccessible facts, or an explicit unknown target result.""" - return resolve_visibility(lookup=lookup, target=target) + return _native.scope_resolve_visibility(lookup, target, True) diff --git a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py index 379c9520d2..c91b4f0f26 100644 --- a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py +++ b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.reports import browse_folders +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeBrowseResult, ScopeLookup @@ -15,9 +15,4 @@ def browse_scope_folders( ) -> ScopeBrowseResult: """Browse direct declaration-definition child folders only.""" - return browse_folders( - lookup=lookup, - folder=folder, - target=target, - target_is_prospective=target_is_prospective, - ) + return _native.scope_browse(lookup, (folder, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py b/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py index a7d704e593..d96e802679 100644 --- a/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py +++ b/src/sqlbuild/compiler/scopes/main/build_scope_lookup.py @@ -2,11 +2,11 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.lookup import build_lookup +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup def build_scope_lookup(*, index: ScopeIndex) -> ScopeLookup: """Canonicalize scope facts and build deeply immutable lookup mappings.""" - return build_lookup(index=index) + return _native.build_scope_lookup(index) diff --git a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py index 5484f70937..f2f339fa40 100644 --- a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py +++ b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.reports import explain_declaration +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, @@ -24,6 +24,4 @@ def explain_scope_declaration( ) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: """Explain a qualified declaration in an optional resource/path context.""" - return explain_declaration( - lookup=lookup, declaration=declaration, target=target, at=at, directory=directory - ) + return _native.scope_explain(lookup, (declaration, target, at, directory)) diff --git a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py index f6653741ed..e46c688f92 100644 --- a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py +++ b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.reports import list_folder +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ( ResourceIdentity, ScopeListResult, @@ -21,10 +21,4 @@ def list_scope_declarations( ) -> ScopeListResult: """List declarations recursively beneath one browsed folder.""" - return list_folder( - lookup=lookup, - folder=folder, - filters=filters, - target=target, - target_is_prospective=target_is_prospective, - ) + return _native.scope_list(lookup, (folder, filters, target, target_is_prospective)) diff --git a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py index 6debf20736..6b4c3a999c 100644 --- a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py +++ b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.reports import build_move_preview +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ( MovePreview, ResourceIdentity, @@ -16,4 +16,4 @@ def preview_scope_move( ) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: """Preview declaration visibility changes without mutating compiler facts.""" - return build_move_preview(lookup=lookup, resource=resource, destination=destination) + return _native.scope_move_preview(lookup, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/query_scope_report.py b/src/sqlbuild/compiler/scopes/main/query_scope_report.py index 78a620647f..59ccc704a8 100644 --- a/src/sqlbuild/compiler/scopes/main/query_scope_report.py +++ b/src/sqlbuild/compiler/scopes/main/query_scope_report.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.reports import build_scope_report +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, ResourceIdentity, @@ -24,10 +24,4 @@ def query_scope_report( ) -> ScopeReport: """Query immutable compiler-owned scope facts without filesystem access.""" - return build_scope_report( - lookup=lookup, - target=target, - at=at, - directory=directory, - filters=filters, - ) + return _native.scope_report(lookup, (target, at, directory, filters)) diff --git a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py index 9e4e436ddf..a962bc5632 100644 --- a/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py +++ b/src/sqlbuild/compiler/scopes/main/relocate_declarations_for_move.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.placement import relocate_for_resource_move +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ScopeIndex @@ -11,4 +11,4 @@ def relocate_declarations_for_move( ) -> tuple[DeclarationRecord, ...] | None: """Return declarations that must move, at their new paths, when a resource moves.""" - return relocate_for_resource_move(index=index, resource=resource, destination=destination) + return _native.scope_relocated_declarations(index, resource, destination) diff --git a/src/sqlbuild/compiler/scopes/main/scope_metadata.py b/src/sqlbuild/compiler/scopes/main/scope_metadata.py index 6c65b50209..c6af02e388 100644 --- a/src/sqlbuild/compiler/scopes/main/scope_metadata.py +++ b/src/sqlbuild/compiler/scopes/main/scope_metadata.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.report_rows import build_projection +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.scopes.types import JsonValue @@ -10,4 +10,4 @@ def scope_metadata_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: """Return deterministic, value-free scope metadata for external consumers.""" - return build_projection(index=index) + return _native.scope_metadata_projection(index) diff --git a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py index 08ab2669dc..978ac7a1d5 100644 --- a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py +++ b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py @@ -1,10 +1,10 @@ """Public canonical scope report serialization operation.""" -from sqlbuild.compiler.scopes._helpers.report_rows import serialize_result +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ScopeBrowseResult, ScopeListResult, ScopeReport def serialize_scope_report(*, report: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: """Return deterministic schema-version-one JSON for any scope query result.""" - return serialize_result(result=report) + return _native.serialize_scope_report(report) diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 64cbab6ccd..ca713943b6 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -128,19 +128,6 @@ class GrantRecord: kind: GrantKind = GrantKind.EXPECTED_MODEL -@dataclass(frozen=True) -class NativeDeclarationValues: - """The Python-side parts of one declaration record the native scope index does not hold.""" - - identity: DeclarationIdentity - line: int - scope: ScopeKind - macro: MacroMetadata | None = None - enum: EnumMetadata | None = None - constant: ConstantMetadata | None = None - dependencies: tuple[DeclarationIdentity, ...] = () - - @dataclass(frozen=True) class VisibilityRecord: """A resolved positive visibility fact with stable provenance.""" diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py index 5e18d6c16a..c1286dd702 100644 --- a/src/sqlbuild/compiler/scopes/types.py +++ b/src/sqlbuild/compiler/scopes/types.py @@ -8,109 +8,10 @@ if TYPE_CHECKING: from pathlib import PurePath - from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - ) + from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] -type NativeIdentityRow = tuple[str, str] -type NativeDeclarationIdentityRow = tuple[str, str, NativeIdentityRow | None] -type NativeResourceRow = tuple[str, str, str, str] -type NativeDeclarationRow = tuple[ - str, - str, - NativeIdentityRow | None, - str, - int, - str, - str | None, - str, - str | None, - list[NativeDeclarationIdentityRow], -] -type NativeQualifiedRow = NativeDeclarationIdentityRow -type NativeUsageKeyRow = tuple[ - NativeQualifiedRow, NativeQualifiedRow, str, NativeQualifiedRow | None, str | None -] -type NativeRootRow = tuple[str, str, str | None] -type NativeResourceRecordRow = tuple[NativeIdentityRow, str, NativeRootRow] -type ScopeDeclarationPairs[Value] = tuple[tuple[DeclarationRecord, Value], ...] -type NativeResolvedRows = tuple[ - list[tuple[int, str, int | None]], list[tuple[int, str]], str | None -] type ScopeQueryTarget = str | PurePath | ResourceIdentity | DeclarationIdentity -type NativeScopeTargetRow = tuple[NativeQualifiedRow | None, str, bool] -type NativeScopeFiltersRow = tuple[ - list[str], bool, str | None, list[str], str | None, bool, int, str, str | None, int, int, str -] -type NativeScopeValuesRow = tuple[ - tuple[list[str], list[NativeQualifiedRow], str] | None, - tuple[list[str], str] | None, - tuple[str, str | None, int | None, bool, str | None] | None, -] -type NativeScopeCompletenessRow = tuple[bool, bool, bool, bool, bool, bool] -type NativeScopeDiagnosticRow = tuple[ - str, - str, - str, - str | None, - int | None, - int | None, - NativeQualifiedRow | None, - NativeIdentityRow | None, -] -type NativeScopeRaisedRow = tuple[int | None, tuple[str, str, str | None] | None] -type NativeScopeDeclarationRow = tuple[ - tuple[str, str, str, str | None, tuple[str, int, int], str, str | None], - tuple[tuple[str, str | None] | None, str | None, list[tuple[str, object]]], - tuple[list[str], list[str], list[str], str | None, str | None, list[str]], -] -type NativeScopeSectionRow = tuple[ - str, int, int, bool, int, bool, bool, str | None, str | None, int -] -type NativeScopeResourceReportRow = tuple[str, str | None, str | None, bool, bool, int] -type NativeScopeExplanationRow = tuple[NativeScopeDeclarationRow | None, bool] -type NativeScopeMoveRow = tuple[ - str, - str, - str | None, - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - list[str], - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - bool, -] -type NativeScopeFolderRow = tuple[str, str, int, int, int, list[tuple[str, int]]] -type NativeScopeReportRow[Diagnostic] = tuple[ - NativeScopeResourceReportRow, - list[tuple[str, str, int]], - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - list[NativeScopeDeclarationRow], - NativeScopeFiltersRow, - list[NativeScopeSectionRow], - NativeScopeExplanationRow | None, - NativeScopeMoveRow | None, - list[Diagnostic], - bool, -] -type NativeScopeBrowseRow[Diagnostic] = tuple[ - str, list[NativeScopeFolderRow], list[Diagnostic], bool -] -type NativeScopeListRow[Diagnostic] = tuple[ - str, list[NativeScopeDeclarationRow], NativeScopeSectionRow, list[Diagnostic] -] -type NativeScopeFailureRow = tuple[str, str] | None -type NativePlacementRows = tuple[ - list[tuple[NativeIdentityRow, str, NativeRootRow]], - list[tuple[NativeQualifiedRow, str, int, int, str, NativeRootRow, str | None]], - list[tuple[NativeQualifiedRow, NativeQualifiedRow, NativeQualifiedRow | None]], -] class ResourceKind(StrEnum): diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 05f16f4124..77ce4fd6b9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -8,11 +8,11 @@ from pathlib import Path from typing import TextIO +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.integrations.dbt._helpers.pipeline.plan_output import dbt_failure_detail from sqlbuild.integrations.dbt.classes.dbt_compile_reference_resolver import ( DbtCompileReferenceResolver, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index a19f4d1c98..fca7d74599 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -32,8 +32,8 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer -import sqlbuild.cli.commands.main.project._compile as compile_command_module import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse +import sqlbuild.cli.compile_reuse.main._write_reusable_compile as write_reusable_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages @@ -3355,7 +3355,7 @@ def write_before_reuse_store( ) -> None: """Let another writer change target/ once, after a compile wrote it and before it stores.""" - original: Callable[..., None] = compile_command_module.write_reusable_compile + original: Callable[..., None] = write_reusable_module.write_reusable_compile pending: list[Callable[[Path], None]] = [write] def write_then_store(**kwargs: Any) -> None: @@ -3364,7 +3364,7 @@ def write_then_store(**kwargs: Any) -> None: pending.clear() original(**kwargs) - monkeypatch.setattr(compile_command_module, "write_reusable_compile", write_then_store) + monkeypatch.setattr(write_reusable_module, "write_reusable_compile", write_then_store) def rewrite_compiled_model_unchanged(project_dir: Path) -> None: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 475159e14b..7500ee8e9d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -67,7 +67,7 @@ "polyglot_sql", "pydantic", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.cli.commands.main.project._compile", + "sqlbuild.cli.commands._helpers.compile.output", "sqlbuild.cli.compile.models", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 260e23899d..8ead2728e1 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -12,6 +12,7 @@ import pytest import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -27,7 +28,6 @@ from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType from sqlbuild.spec.contracts.models import ( SchemaColumn, diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index 4068a39894..a028d5803e 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -10,6 +10,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import build_project_graph from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record @@ -19,7 +20,6 @@ build_native_column_lineage, ) from sqlbuild.compiler.lineage.models import ProjectColumnLineage -from sqlbuild.compiler.pipeline.main.graph import build_project_graph _PROJECT_TOML: str = ( 'name = "orders_lineage"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 425be5f070..af90e57ef7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import build_project_graph from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target @@ -16,7 +17,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.pipeline.main import compile as compile_pipeline -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index d13eb68ab8..ac470932cd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( NativeCatalogCase, ValidationCacheCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index dd1bb0e179..c4f925d99d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -6,12 +6,12 @@ import pytest +from sqlbuild._native import build_compiled_project from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SnowflakeOutputInferenceCase, SnowflakeSemanticReleaseCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index a69974eb43..d5fc551a24 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -8,6 +8,7 @@ import pytest import sqlbuild._native as native +from sqlbuild._native import build_compiled_project from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( FunctionArgumentTypeCase, SemanticTriageCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index ea88190766..3ef08e7615 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import build_compiled_project from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, @@ -17,7 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 53c6ea0830..38cdc4e4bc 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -234,9 +234,9 @@ def python_resource_calls(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: calls: Counter[str] = Counter() _ = [ monkeypatch.setattr( - project_module, + project_module._native, name, - _counted(name=name, function=getattr(project_module, name), calls=calls), + _counted(name=name, function=getattr(project_module._native, name), calls=calls), ) for name in _PYTHON_RESOURCE_FUNCTIONS ] diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 356542fb46..451d1d21c5 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -32,7 +32,6 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( @@ -323,7 +322,7 @@ def native_scope_attempts( def counted( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] - ) -> NativeScopeIndex: + ) -> _native.NativeScopeIndex: attempts.append(None) return open_native_scope_index( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index c3bc9138a8..6bdde6b886 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -15,6 +15,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -25,7 +26,6 @@ from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 859b4d7464..e1a86455e8 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -13,6 +13,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import build_project_graph from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( @@ -20,7 +21,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssembly, NativeSqlTestAssemblyRequest, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index d2cf82f76f..1a73fdec19 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -2,6 +2,7 @@ import gc import json +import sys from collections.abc import Callable from pathlib import Path from typing import Any, cast @@ -9,16 +10,17 @@ import pytest from sqlbuild.cli.commands._helpers.compile import lineage as compile_lineage -from sqlbuild.cli.commands._helpers.compile import pipeline as compile_pipeline -from sqlbuild.cli.commands._helpers.compile import status as compile_status +from sqlbuild.cli.commands._helpers.compile import target_writer +from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters from sqlbuild.cli.commands.classes import prepared_compile_artifacts -from sqlbuild.cli.commands.main.project import _compile as compile_command from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.discovery.main import discover as discovery_entry from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.presentation.classes import transient_status_reporter from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, @@ -61,7 +63,7 @@ def test_given_local_project_when_running_compile_then_it_does_not_connect( ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -130,10 +132,10 @@ def complete(self, message: str, *, blank_line_after: bool = False) -> None: def close(self) -> None: return None - monkeypatch.setattr(compile_status.sys, "stdout", FakeStdout()) - monkeypatch.setattr(compile_status, "TransientStatusReporter", FakeStatusReporter) + monkeypatch.setattr(sys, "stdout", FakeStdout()) + monkeypatch.setattr(transient_status_reporter, "TransientStatusReporter", FakeStatusReporter) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -175,7 +177,7 @@ def test_given_dag_flag_when_running_compile_then_writes_dag_artifact( ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -232,7 +234,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d ) -> None: project_dir: Path = prepare_python_compile_project(tmp_path) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -356,7 +358,7 @@ def test_given_contract_errors_when_running_compile_then_reports_diagnostics( encoding="utf-8", ) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -411,7 +413,7 @@ def test_given_contract_errors_when_running_compile_json_then_serializes_diagnos encoding="utf-8", ) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -479,7 +481,7 @@ def test_given_explicit_column_contract_mode_when_compiling_column_audit_then_au encoding="utf-8", ) monkeypatch.setattr( - compile_pipeline, + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) @@ -594,7 +596,7 @@ def test_given_rule_error_when_compiling_json_then_withheld_test_planning_errors ) planning_calls: list[Path] = [] static_planning: Callable[..., tuple[CompilerDiagnostic, ...]] = ( - compile_pipeline.static_sql_test_planning_diagnostics + target_writer.static_sql_test_planning_diagnostics ) def counted_static_planning(**kwargs: Any) -> tuple[CompilerDiagnostic, ...]: @@ -602,10 +604,10 @@ def counted_static_planning(**kwargs: Any) -> tuple[CompilerDiagnostic, ...]: return static_planning(**kwargs) monkeypatch.setattr( - compile_pipeline, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() ) monkeypatch.setattr( - compile_pipeline, "static_sql_test_planning_diagnostics", counted_static_planning + target_writer, "static_sql_test_planning_diagnostics", counted_static_planning ) monkeypatch.setattr( prepared_compile_artifacts, @@ -653,16 +655,16 @@ def test_given_collection_state_when_compiling_then_cycles_are_paused_and_state_ ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) enabled_during: list[bool] = [] - analyze: Callable[..., Any] = compile_pipeline.analyze_compile_project + discover: Callable[..., Any] = discovery_entry.discover_project_inputs - def observed_analyze(**kwargs: Any) -> Any: + def observed_discover(**kwargs: Any) -> Any: enabled_during.append(gc.isenabled()) - return analyze(**kwargs) + return discover(**kwargs) monkeypatch.setattr( - compile_pipeline, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() + runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() ) - monkeypatch.setattr(compile_command, "analyze_compile_project", observed_analyze) + monkeypatch.setattr(discovery_entry, "discover_project_inputs", observed_discover) set_collection: dict[bool, Callable[[], None]] = {True: gc.enable, False: gc.disable} was_enabled: bool = gc.isenabled() set_collection[test_case.collection_enabled_before]() diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index 02ba9206a4..6ccf45f542 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -15,9 +15,8 @@ from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process _FULL_COMPILE_MODULES: tuple[str, ...] = ( - "sqlbuild.cli.commands.main.project._compile", - "sqlbuild.cli.commands._helpers.compile.pipeline", - "sqlbuild.compiler.pipeline.main.compiled_project", + "sqlbuild.cli.commands._helpers.compile.output", + "sqlbuild.compiler.compile.main._assemble_project", ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 86b10cb315..5060952341 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -368,6 +368,42 @@ def test_given_inaccessible_scoped_macro_import_when_loading_then_rejects_before macro_files={"macros/orders.py": "class Formatter:\n pass\n"}, expected_error_fragment="declaration 'Formatter' must be underscore-private", ), + LoadProjectMacrosErrorTestCase( + description="reports an earlier module's declaration before a later syntax error", + macro_files={ + "macros/a_orders.py": "DEFAULT_LIMIT = 10\n", + "macros/b_customers.py": "def customers(:\n", + }, + expected_error_fragment=( + "^Macro module 'macros/a_orders.py' declaration 'DEFAULT_LIMIT' must be " + "underscore-private$" + ), + ), + LoadProjectMacrosErrorTestCase( + description="reports a later module's syntax error after earlier modules pass", + macro_files={ + "macros/a_orders.py": "def orders() -> str:\n return 'orders'\n", + "macros/b_customers.py": "def customers(:\n", + }, + expected_error_fragment="^Failed to parse macros from '.*b_customers.py': ", + ), + LoadProjectMacrosErrorTestCase( + description="names the shallowest module-level use of an imported macro", + macro_files={ + "macros/shared.py": ( + "def first() -> str:\n return 'first'\n\n" + "def second() -> str:\n return 'second'\n" + ), + "macros/consumer.py": ( + "from macros.shared import first, second\n\n" + "_PAIRS = [[first], second]\n\n" + "def consumer() -> str:\n return 'consumer'\n" + ), + }, + expected_error_fragment=( + "uses imported macro 'second' outside an ordinary function body" + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py index 8cb0f0d63b..ab3eaf9d83 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py @@ -4,10 +4,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.analysis.validation import ( - validate_hook_sql_syntax, - validate_sql_syntax, -) +import sqlbuild._native as _native from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ValidateHookSqlSyntaxTestCase, @@ -73,11 +70,10 @@ def test_given_valid_sql_when_validating_syntax_then_does_not_raise( test_case: ValidateSqlSyntaxTestCase, ) -> None: - validate_sql_syntax( - query_sql=test_case.query_sql, - model_name=_MODEL_NAME, - file_path=_FILE_PATH, - dialect=test_case.dialect, + _native.validate_sql_syntax( + test_case.query_sql, + (_MODEL_NAME, _FILE_PATH), + (None, test_case.dialect), ) assert test_case.expected_valid is True @@ -111,11 +107,10 @@ def test_given_invalid_sql_when_validating_syntax_then_raises_compile_error( test_case: ValidateSqlSyntaxTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_sql_syntax( - query_sql=test_case.query_sql, - model_name=_MODEL_NAME, - file_path=_FILE_PATH, - dialect=test_case.dialect, + _native.validate_sql_syntax( + test_case.query_sql, + (_MODEL_NAME, _FILE_PATH), + (None, test_case.dialect), ) assert test_case.expected_valid is False @@ -142,12 +137,11 @@ def test_given_invalid_sql_when_validating_syntax_then_raises_compile_error( def test_given_polyglot_supported_hook_sql_when_validating_then_does_not_raise( test_case: ValidateHookSqlSyntaxTestCase, ) -> None: - validate_hook_sql_syntax( - value=test_case.hook_sql, - hook_name="pre_hooks", - model_name=_MODEL_NAME, - file_path=_FILE_PATH, - dialect=test_case.dialect, + _native.validate_hook_sql_syntax( + test_case.hook_sql, + "pre_hooks", + (_MODEL_NAME, _FILE_PATH), + (None, test_case.dialect), ) assert test_case.expected_valid is True @@ -170,12 +164,11 @@ def test_given_invalid_hook_sql_when_validating_then_raises_compile_error( test_case: ValidateHookSqlSyntaxTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_hook_sql_syntax( - value=test_case.hook_sql, - hook_name="pre_hooks", - model_name=_MODEL_NAME, - file_path=_FILE_PATH, - dialect=test_case.dialect, + _native.validate_hook_sql_syntax( + test_case.hook_sql, + "pre_hooks", + (_MODEL_NAME, _FILE_PATH), + (None, test_case.dialect), ) assert test_case.expected_valid is False diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py index 99712b3c65..315de7de6a 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -9,7 +9,7 @@ from pathlib import PurePath from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes._helpers.visibility import query_target +from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( resolve_scope_declaration_visibility, ) @@ -207,7 +207,7 @@ def _reference_resolution( ) -> VisibilityResolution: """Classify every declaration against every matched resource, as before indexing.""" - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) + query: ScopeTargetQuery = query_scope_target(lookup=lookup, target=target) visible: list[VisibilityRecord] = [] inaccessible: list[InaccessibleRecord] = [] for resource in query.matches: diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index eab92a2b97..2e7c130848 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -23,9 +23,9 @@ EnumDeclaration, EnumMember, ) -from sqlbuild.compiler.scopes._helpers.visibility import query_target from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index +from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup @@ -291,7 +291,7 @@ def test_given_unknown_qualified_target_when_querying_then_distinguishes_unknown ) -> None: lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex()) - query: ScopeTargetQuery = query_target(lookup=lookup, target="model:missing") + query: ScopeTargetQuery = query_scope_target(lookup=lookup, target="model:missing") resolution: VisibilityResolution = resolve_scope_visibility( lookup=lookup, target="model:missing" ) @@ -317,7 +317,7 @@ def test_given_declaration_qualified_target_when_querying_then_returns_declarati ) lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex(declarations=(declaration,))) - query: ScopeTargetQuery = query_target( + query: ScopeTargetQuery = query_scope_target( lookup=lookup, target="constant:minimum_value", ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 9024e609d5..d67cdd7500 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -4,11 +4,11 @@ import pytest +from sqlbuild._native import build_project_graph from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.main.run_rules import run_rules