diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..24d196b11 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + verify: + name: Verify + runs-on: ubuntu-24.04 + timeout-minutes: 20 + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install the pinned Rust toolchain + run: rustup toolchain install + # Caches the dependency builds, polyglot above all, keyed by Cargo.lock and the toolchain. + - uses: Swatinem/rust-cache@v2 + with: + shared-key: verify + - name: Install Python + run: uv python install 3.12 + - name: Install development tools + run: uv sync --locked + - name: Run static checks + run: make check-ci + - name: Run tests + run: make test diff --git a/.gitignore b/.gitignore index 1ed8e8fb8..181c3b5e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,25 +1,31 @@ -# Generated by Cargo -# will have compiled files and executables -debug -target - -# These are backup files generated by rustfmt +# Rust +/target/ **/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information *.pdb - -# Generated by cargo mutants -# Contains mutation testing data **/mutants.out*/ -# Python +# Python tooling __pycache__/ *.py[cod] +.venv/ +.ruff_cache/ + +# Fensu +.fensu/cache/ +.fensu/memory/ + +# Local data +*.duckdb +*.duckdb.wal +.tmp* + +# Editors and OS files +.DS_Store +.idea/ +.vscode/ -# RustRover -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# Local agent configuration and generated skills +.claude/* +/.agents/skills/ +/.claude/skills/ +/.opencode/skills/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..81d6116fb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,91 @@ +# Agent Instructions + +## Product Rules + +- SQLQuack refuses to guess. Never emit a DuckDB node that no verified rule produced, and never pass + a source construct through because it prints identically in DuckDB. Identical syntax is not + evidence of identical meaning. +- Every refusal names its construct, its source position and its category: missing rule, not + expressible in DuckDB, or parse fidelity. Add categories rather than overloading one. +- Do not trust or enable polyglot's `transpile` feature or any of its dialect conversions. Use + polyglot for parsing, name resolution and printing only. +- The source dialect is a parameter of every entry point. Do not hard-code Snowflake below the + command line. +- Entries in the spelling-only field list and normalisations (`sqlquack-core` comparison constants) + weaken the re-parse checks. Each needs a reason and a test showing two spellings differ only + there. Do not add one to make a failing check pass. + +## Evidence Integrity + +- Recorded warehouse results are the only arbiter of correctness. Agents never record, re-record, + edit or delete recorded results, and never contact a warehouse. Report pending cases to the + maintainer instead. +- Recorded results are append-only and named by the hash of their case. A changed case is a new + case. +- Everything that could be weakened to make a failing case pass (comparator tolerances, required + edge-case inputs, known divergences) lives in one reviewed policy file. There are no per-case or + per-rule overrides elsewhere. +- When a case fails, either fix the rule or make it refuse. Never widen a rule to raise the + conversion rate. + +## Testing + +- Every behaviour change ships with tests. Prefer tests that run the real pipeline and the real + `sqlquack` binary over tests of isolated helpers. +- Tests follow the fensu Rust test layout: a `tests.rs` harness of `#[path]` modules, a sibling + `test_types.rs` with test-case structs, a local `test_cases` array per test, one loop, and + assertion messages carrying `test_case.description`. +- Run focused tests with `cargo test -p `. `make test` runs the workspace suite. +- Before pushing, run `make check-ci`. + +## Public Repository Hygiene + +- Treat every tracked file, fixture, filename, commit, branch, pull request, review comment and CI + log as publicly visible, including history. +- Use only neutral synthetic examples such as orders, customers, products, inventory, fulfilment + and support tickets. Never copy organisation-, client-, provider-, environment- or + production-specific names, SQL, data, query output, identifiers or counts into the repository, + even when the source was read-only. +- Measurements on private projects stay private. A bug they expose becomes a rewritten neutral + reproduction. +- Load and follow the `public-repository-hygiene` skill before editing or publishing. Run its + scanner over the working tree and introduced history before every push. Never bypass the + configured Git hooks; a missing or failing scanner blocks publication. +- Do not copy the private restricted-vocabulary list or scanner into this repository. + +## Command-Line User Experience + +- Potentially slow phases print a concise start message and an explicit completion or failure + message on stderr. +- Machine-readable output (`--json`) goes to stdout alone; progress goes to stderr. +- Reports never print SQL text or file names unless the user asks for them. + +## Native Builds + +- Polyglot is one large crate. Keep crates small so a rule change does not rebuild everything, and + do not add dependencies that rebuild polyglot. +- Pin polyglot to the same revision as SQLBuild and bump it deliberately, with a census before and + after. + +## Subagent Verification + +- The primary agent owns the verification plan. Implementation subagents run only tests covering + their change plus focused static checks; review subagents are read-only. +- Follow-up reviews verify only previously reported findings and their boundaries. + +## Delivery Workflow + +- Consolidate related work for one release into one branch and one pull request. +- Run focused tests and `make check-ci` before pushing; CI is the broad gate. +- Pull request titles follow Conventional Commits. Descriptions have `## Why`, `## Changes` and + `## Verification` sections. +- Monitor CI after every push and address failures before considering delivery complete. + +## Review Discipline + +- Treat review findings as hypotheses to validate against supported behaviour and realistic + execution paths. +- Prioritise correctness: a conversion that changes meaning is the worst possible defect, and a + refusal is always an acceptable fix. +- Keep fixes within scope. Escalate anything that expands supported behaviour, recorded evidence or + the policy file to the maintainer. diff --git a/Cargo.lock b/Cargo.lock index a67682946..1ffaea41b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,24 +2,389 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "polyglot-sql" +version = "0.13.9" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +dependencies = [ + "polyglot-sql-ast-derive", + "polyglot-sql-function-catalogs", + "regex", + "serde", + "serde_json", + "stacker", + "thiserror", + "unicode-segmentation", +] + +[[package]] +name = "polyglot-sql-ast-derive" +version = "0.13.9" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "polyglot-sql-function-catalogs" +version = "0.13.9" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" + [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "same-file" version = "1.0.6" @@ -30,41 +395,190 @@ dependencies = [ ] [[package]] -name = "sqlquack" -version = "0.1.0" +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] [[package]] -name = "sqlquack-python" -version = "0.1.0" +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ - "sqlquack", + "serde_derive", ] [[package]] -name = "sqlquack-structure-checker" -version = "0.1.0" +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", - "syn", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sqlquack" +version = "0.2.0" +dependencies = [ + "clap", + "rayon", + "serde", + "serde_json", + "sqlquack-compare", + "sqlquack-core", + "sqlquack-duckdb", + "sqlquack-rules", + "sqlquack-snowflake", + "tempfile", "walkdir", ] +[[package]] +name = "sqlquack-compare" +version = "0.2.0" + +[[package]] +name = "sqlquack-core" +version = "0.2.0" +dependencies = [ + "polyglot-sql", + "serde", + "serde_json", +] + +[[package]] +name = "sqlquack-duckdb" +version = "0.2.0" + +[[package]] +name = "sqlquack-rules" +version = "0.2.0" +dependencies = [ + "sqlquack-core", +] + +[[package]] +name = "sqlquack-snowflake" +version = "0.2.0" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "walkdir" @@ -99,3 +613,9 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 0fd698b6c..ed7f15ae4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,60 @@ [workspace] members = [ + "crates/sqlquack-core", + "crates/sqlquack-snowflake", + "crates/sqlquack-rules", + "crates/sqlquack-duckdb", + "crates/sqlquack-compare", "crates/sqlquack", - "crates/sqlquack-python", - "crates/sqlquack-structure-checker", ] resolver = "2" [workspace.package] -edition = "2021" -license = "MIT" -repository = "https://github.com/klonge/sqlquack" +version = "0.2.0" +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/chio-labs/sqlquack" +homepage = "https://github.com/chio-labs/sqlquack" readme = "README.md" +publish = false +rust-version = "1.88" + +[workspace.dependencies] +clap = { version = "4.5", features = ["derive"] } +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ + "dialect-snowflake", + "dialect-duckdb", + "dialect-bigquery", + "dialect-databricks", + "generate", + "semantic", + "stacker", +] } +rayon = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sqlquack-compare = { path = "crates/sqlquack-compare" } +sqlquack-core = { path = "crates/sqlquack-core" } +sqlquack-duckdb = { path = "crates/sqlquack-duckdb" } +sqlquack-rules = { path = "crates/sqlquack-rules" } +sqlquack-snowflake = { path = "crates/sqlquack-snowflake" } +tempfile = "3" +walkdir = "2" + +[workspace.lints.rust] +unsafe_code = "forbid" +unreachable_pub = "deny" +unused_must_use = "deny" + +[workspace.lints.clippy] +await_holding_lock = "deny" + +[profile.dev] +debug = "line-tables-only" + +[profile.dev.package."*"] +opt-level = 1 +debug = false + +[profile.release] +strip = "symbols" diff --git a/Makefile b/Makefile index 526944a19..fc81ec9d9 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,29 @@ -.PHONY: format check-format check-structure check-rust test-rust check verify +SHELL := /bin/bash + +.PHONY: format rust-lint python-check fensu check-ci test census format: - cargo fmt + cargo fmt --all + uv run ruff format . -check-format: - cargo fmt --check +rust-lint: + cargo fmt --all --check + cargo clippy --workspace --all-targets --locked -- -D warnings -check-structure: - cargo run -p sqlquack-structure-checker --quiet +python-check: + uv run ruff format --check . + uv run ruff check . + uv run ty check -check-rust: - cargo check --all +fensu: + uv run fensu check -test-rust: - cargo test --all +# The static gate: CI and the pre-push verify hook run exactly this. +check-ci: rust-lint python-check fensu -check: check-format check-structure check-rust +test: + cargo test --workspace --locked -verify: check test-rust +# Example: make census CENSUS_ARGS="path/to/sql --json" +census: + cargo run --release --locked -p sqlquack -- census $(CENSUS_ARGS) diff --git a/README.md b/README.md index c0dbb58d6..be4afb2dc 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,45 @@ # SQLQuack -SQLQuack is an early-stage SQLGlot-inspired Rust engine for warehouse SQL analysis and fail-closed DuckDB transpilation. +SQLQuack converts Snowflake SQL into DuckDB SQL, and refuses to guess. -SQLQuack is not a broad `N x N` dialect transpiler. It is intended for callers that need structured facts about expanded warehouse SQL: diagnostics, output columns, references, lineage, and conservative type/nullability information. +Most SQL transpilers print *something* for every input. Often that output parses and runs on DuckDB but +means something different: `CONCAT` handles NULLs differently, regular expressions anchor +differently, decimal division keeps a different scale. SQLQuack takes the opposite approach. Every +conversion either comes from a rule verified against recorded Snowflake results, or it is refused +with a precise reason: the construct, where it is, and why. -DuckDB output is fail-closed. SQLQuack should only emit DuckDB SQL when the source semantics are expected to be preserved; unsupported or ambiguous constructs should produce diagnostics instead of best-effort rewrites. +**Status: early.** No conversion rules exist yet, so SQLQuack currently refuses every construct. What +works today: -Status: early development. APIs are unstable. +- parsing Snowflake SQL with [polyglot](https://github.com/chio-labs/polyglot) and classifying every + function, operator, type, cast, join and clause in it; +- a structured refusal for each construct: *missing rule*, *not expressible in DuckDB*, or + *parse fidelity*; +- two re-parse checks: printing the Snowflake tree back to Snowflake, and checking that DuckDB SQL + parses back to the tree it was printed from; +- `sqlquack census`, which reports which constructs a body of SQL uses and how much of it would + convert as rules are added. + +## Census + +```console +$ cargo run --release -p sqlquack -- census path/to/sql +$ cargo run --release -p sqlquack -- census path/to/sql --json +``` + +Each `.sql` file is one query, or one model compiled by +[SQLBuild](https://github.com/chio-labs/sqlbuild); SQLBuild's reference calls are recognised. The +report contains counts and construct names only, never SQL text or file names. + +## How it works + +See [docs/architecture.md](docs/architecture.md). + +## Development + +`make check-ci` runs every static gate (rustfmt, clippy, fensu, ruff and ty) and `make test` runs +the tests. See [AGENTS.md](AGENTS.md) for the rules every change follows. + +## License + +Apache-2.0. diff --git a/crates/sqlquack-compare/Cargo.toml b/crates/sqlquack-compare/Cargo.toml new file mode 100644 index 000000000..2414b6ffc --- /dev/null +++ b/crates/sqlquack-compare/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sqlquack-compare" +version.workspace = true +description = "The strict result comparator for SQLQuack's recorded evidence." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/crates/sqlquack-compare/src/lib.rs b/crates/sqlquack-compare/src/lib.rs new file mode 100644 index 000000000..59580133a --- /dev/null +++ b/crates/sqlquack-compare/src/lib.rs @@ -0,0 +1,3 @@ +//! The strict comparator for recorded warehouse results; a stub until recorded evidence exists. + +#![forbid(unsafe_code)] diff --git a/crates/sqlquack-core/Cargo.toml b/crates/sqlquack-core/Cargo.toml new file mode 100644 index 000000000..0a8177d31 --- /dev/null +++ b/crates/sqlquack-core/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqlquack-core" +version.workspace = true +description = "Parsing, construct classification, refusals and re-parse checks for SQLQuack." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +polyglot-sql.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-core/src/classification/_helpers/clauses.rs b/crates/sqlquack-core/src/classification/_helpers/clauses.rs new file mode 100644 index 000000000..1a7664d5b --- /dev/null +++ b/crates/sqlquack-core/src/classification/_helpers/clauses.rs @@ -0,0 +1,192 @@ +//! Clauses, join kinds and set-operation forms, from exhaustively destructured query nodes. + +use polyglot_sql::Expression; +use polyglot_sql::expressions::{Except, Intersect, Join, Select, Union}; + +use crate::classification::_helpers::names::snake_case; +use crate::classification::models::{Construct, ConstructFamily}; + +pub(crate) fn clause_constructs(node: &Expression) -> Vec { + let names = match node { + Expression::Select(select) => select_clauses(select), + Expression::Union(union) => union_clauses(union), + Expression::Intersect(intersect) => intersect_clauses(intersect), + Expression::Except(except) => except_clauses(except), + _ => Vec::new(), + }; + let mut constructs: Vec = names + .into_iter() + .map(|name| Construct::new(ConstructFamily::Clause, name)) + .collect(); + if let Expression::Select(select) = node { + constructs.extend(select.joins.iter().flat_map(join_constructs)); + } + constructs +} + +fn present(flags: &[(bool, &str)]) -> Vec { + flags + .iter() + .filter(|(present, _)| *present) + .map(|(_, name)| (*name).to_owned()) + .collect() +} + +fn select_clauses(select: &Select) -> Vec { + let Select { + expressions: _, + from: _, + joins: _, + lateral_views, + prewhere, + where_clause, + group_by, + having, + qualify, + order_by, + distribute_by, + cluster_by, + sort_by, + limit, + offset, + limit_by, + fetch, + distinct, + distinct_on, + top, + with, + sample, + settings, + format, + windows, + hint, + connect, + into, + locks, + for_xml, + for_json, + leading_comments: _, + post_select_comments: _, + kind, + operation_modifiers, + qualify_after_window: _, + option, + exclude, + } = select; + present(&[ + (with.is_some(), "with"), + (*distinct, "distinct"), + (distinct_on.is_some(), "distinct_on"), + (top.is_some(), "top"), + (!lateral_views.is_empty(), "lateral_view"), + (prewhere.is_some(), "prewhere"), + (where_clause.is_some(), "where"), + (group_by.is_some(), "group_by"), + (having.is_some(), "having"), + (windows.is_some(), "window"), + (qualify.is_some(), "qualify"), + (connect.is_some(), "connect_by"), + (order_by.is_some(), "order_by"), + (distribute_by.is_some(), "distribute_by"), + (cluster_by.is_some(), "cluster_by"), + (sort_by.is_some(), "sort_by"), + (limit.is_some(), "limit"), + (limit_by.is_some(), "limit_by"), + (offset.is_some(), "offset"), + (fetch.is_some(), "fetch"), + (sample.is_some(), "sample"), + (settings.is_some(), "settings"), + (format.is_some(), "format"), + (hint.is_some(), "hint"), + (into.is_some(), "into"), + (!locks.is_empty(), "lock"), + (!for_xml.is_empty(), "for_xml"), + (!for_json.is_empty(), "for_json"), + (kind.is_some(), "select_kind"), + (!operation_modifiers.is_empty(), "operation_modifier"), + (option.is_some(), "option"), + (exclude.is_some(), "exclude"), + ]) +} + +macro_rules! set_operation_clauses { + ($function:ident, $type:ident, $prefix:literal) => { + fn $function(operation: &$type) -> Vec { + let $type { + left: _, + right: _, + all, + distinct, + with, + order_by, + limit, + offset, + distribute_by, + sort_by, + cluster_by, + by_name, + side, + kind, + corresponding, + strict, + on_columns, + } = operation; + present(&[ + (*all, concat!($prefix, "_all")), + (*distinct, concat!($prefix, "_distinct")), + (*by_name, concat!($prefix, "_by_name")), + (side.is_some(), concat!($prefix, "_side")), + (kind.is_some(), concat!($prefix, "_kind")), + (*corresponding, concat!($prefix, "_corresponding")), + (*strict, concat!($prefix, "_strict")), + (!on_columns.is_empty(), concat!($prefix, "_on_columns")), + (with.is_some(), "with"), + (order_by.is_some(), "order_by"), + (limit.is_some(), "limit"), + (offset.is_some(), "offset"), + (distribute_by.is_some(), "distribute_by"), + (sort_by.is_some(), "sort_by"), + (cluster_by.is_some(), "cluster_by"), + ]) + } + }; +} + +set_operation_clauses!(union_clauses, Union, "union"); +set_operation_clauses!(intersect_clauses, Intersect, "intersect"); +set_operation_clauses!(except_clauses, Except, "except"); + +fn join_constructs(join: &Join) -> Vec { + let Join { + this: _, + on: _, + using, + kind, + use_inner_keyword: _, + use_outer_keyword: _, + deferred_condition: _, + join_hint, + match_condition, + pivots, + comments: _, + nesting_group: _, + directed, + } = join; + let mut constructs = vec![Construct::new( + ConstructFamily::Join, + snake_case(&format!("{kind:?}")), + )]; + let names = present(&[ + (!using.is_empty(), "join_using"), + (join_hint.is_some(), "join_hint"), + (match_condition.is_some(), "match_condition"), + (!pivots.is_empty(), "join_pivot"), + (*directed, "join_directed"), + ]); + constructs.extend( + names + .into_iter() + .map(|name| Construct::new(ConstructFamily::Clause, name)), + ); + constructs +} diff --git a/crates/sqlquack-core/src/classification/_helpers/identity.rs b/crates/sqlquack-core/src/classification/_helpers/identity.rs new file mode 100644 index 000000000..23a481f74 --- /dev/null +++ b/crates/sqlquack-core/src/classification/_helpers/identity.rs @@ -0,0 +1,44 @@ +//! The constructs a single node stands for, apart from its clauses. + +use polyglot_sql::expressions::{Cast, DataType}; +use polyglot_sql::{DialectType, Expression}; + +use crate::classification::_helpers::names::base_type_name; +use crate::classification::constants::{PROJECT_FUNCTION_CONSTRUCT, PROJECT_FUNCTION_PLACEHOLDER}; +use crate::classification::models::{Construct, ConstructFamily}; + +pub(crate) fn node_constructs(node: &Expression, dialect: DialectType) -> Vec { + match node { + Expression::Function(function) => vec![function_construct(&function.name)], + Expression::AggregateFunction(function) => vec![function_construct(&function.name)], + Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { + cast_constructs(node.variant_name(), cast, dialect) + } + other => vec![Construct::new(ConstructFamily::Node, other.variant_name())], + } +} + +fn function_construct(name: &str) -> Construct { + if name.eq_ignore_ascii_case(PROJECT_FUNCTION_PLACEHOLDER) { + return Construct::new(ConstructFamily::Function, PROJECT_FUNCTION_CONSTRUCT); + } + Construct::new(ConstructFamily::Function, name.to_ascii_uppercase()) +} + +fn cast_constructs(variant: &str, cast: &Cast, dialect: DialectType) -> Vec { + let mut constructs = vec![ + Construct::new(ConstructFamily::Node, variant), + Construct::new(ConstructFamily::Cast, type_name(&cast.to, dialect)), + ]; + if cast.double_colon_syntax { + constructs.push(Construct::new(ConstructFamily::Syntax, "double_colon_cast")); + } + constructs +} + +fn type_name(data_type: &DataType, dialect: DialectType) -> String { + match polyglot_sql::generate_data_type(data_type, dialect) { + Ok(rendered) => base_type_name(&rendered), + Err(_) => "".to_owned(), + } +} diff --git a/crates/sqlquack-core/src/classification/_helpers/locations.rs b/crates/sqlquack-core/src/classification/_helpers/locations.rs new file mode 100644 index 000000000..b52c13c4c --- /dev/null +++ b/crates/sqlquack-core/src/classification/_helpers/locations.rs @@ -0,0 +1,96 @@ +//! Source positions for nodes, borrowed from neighbours when polyglot kept none. + +use polyglot_sql::Expression; +use polyglot_sql::tokens::Span; + +use crate::parsing::models::SourceLocation; + +pub(crate) struct LineIndex { + line_starts: Vec, + length: usize, +} + +impl LineIndex { + pub(crate) fn new(source: &str) -> Self { + let mut line_starts = vec![0]; + let mut length = 0; + for (offset, character) in source.chars().enumerate() { + if character == '\n' { + line_starts.push(offset + 1); + } + length = offset + 1; + } + Self { + line_starts, + length, + } + } + + pub(crate) fn location(&self, span: Span) -> SourceLocation { + if span.start > self.length { + return SourceLocation { + start: span.start, + end: span.end, + line: span.line, + column: span.column, + }; + } + let line_index = self + .line_starts + .partition_point(|line_start| *line_start <= span.start) + .saturating_sub(1); + SourceLocation { + start: span.start, + end: span.end, + line: line_index + 1, + column: span.start - self.line_starts[line_index] + 1, + } + } +} + +pub(crate) fn own_span(node: &Expression) -> Option { + match node { + Expression::Identifier(identifier) => identifier.span, + Expression::Column(column) => column.span.or(column.name.span), + Expression::Table(table) => table.span.or(table.name.span), + Expression::Star(star) => star.span, + Expression::Function(function) => function.span, + Expression::Count(count) => count.span, + _ => None, + } +} + +pub(crate) fn resolve_locations( + nodes: &[&Expression], + parents: &[Option], + index: &LineIndex, +) -> Vec> { + let count = nodes.len(); + let mut subtree_end: Vec = (1..=count).collect(); + for position in (0..count).rev() { + if let Some(parent) = parents[position] { + subtree_end[parent] = subtree_end[parent].max(subtree_end[position]); + } + } + let spans: Vec> = nodes.iter().map(|node| own_span(node)).collect(); + let mut next_span: Vec> = vec![None; count + 1]; + for position in (0..count).rev() { + next_span[position] = if spans[position].is_some() { + Some(position) + } else { + next_span[position + 1] + }; + } + let mut locations: Vec> = Vec::with_capacity(count); + for position in 0..count { + let inside = next_span[position] + .filter(|found| *found < subtree_end[position]) + .and_then(|found| spans[found]); + let location = match inside { + Some(span) => Some(index.location(span)), + None => parents[position].and_then(|parent| locations[parent]), + }; + locations.push(location); + } + locations +} diff --git a/crates/sqlquack-core/src/classification/_helpers/mod.rs b/crates/sqlquack-core/src/classification/_helpers/mod.rs new file mode 100644 index 000000000..01f941e51 --- /dev/null +++ b/crates/sqlquack-core/src/classification/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod clauses; +pub(crate) mod identity; +pub(crate) mod locations; +pub(crate) mod names; diff --git a/crates/sqlquack-core/src/classification/_helpers/names.rs b/crates/sqlquack-core/src/classification/_helpers/names.rs new file mode 100644 index 000000000..30851c8e2 --- /dev/null +++ b/crates/sqlquack-core/src/classification/_helpers/names.rs @@ -0,0 +1,25 @@ +//! Name spellings for construct keys. + +pub(crate) fn snake_case(camel: &str) -> String { + let mut snake = String::with_capacity(camel.len() + 4); + for (index, character) in camel.chars().enumerate() { + if character.is_ascii_uppercase() { + if index > 0 { + snake.push('_'); + } + snake.push(character.to_ascii_lowercase()); + } else { + snake.push(character); + } + } + snake +} + +pub(crate) fn base_type_name(rendered: &str) -> String { + let base = rendered + .split(['(', '<', ' ']) + .next() + .unwrap_or(rendered) + .trim(); + base.to_ascii_uppercase() +} diff --git a/crates/sqlquack-core/src/classification/constants.rs b/crates/sqlquack-core/src/classification/constants.rs new file mode 100644 index 000000000..e049534f6 --- /dev/null +++ b/crates/sqlquack-core/src/classification/constants.rs @@ -0,0 +1,7 @@ +//! Reserved names shared with callers that rewrite project-specific calls before parsing. + +/// The function name callers substitute for project user-defined functions. +pub const PROJECT_FUNCTION_PLACEHOLDER: &str = "__sqlquack_udf"; + +/// The construct name of a project user-defined function call. +pub const PROJECT_FUNCTION_CONSTRUCT: &str = ""; diff --git a/crates/sqlquack-core/src/classification/main/classify.rs b/crates/sqlquack-core/src/classification/main/classify.rs new file mode 100644 index 000000000..29f1b16b1 --- /dev/null +++ b/crates/sqlquack-core/src/classification/main/classify.rs @@ -0,0 +1,40 @@ +//! `classify`: every construct a statement uses, with its position. + +use polyglot_sql::Expression; +use polyglot_sql::traversal::ExpressionWalk; + +use crate::classification::_helpers::clauses::clause_constructs; +use crate::classification::_helpers::identity::node_constructs; +use crate::classification::_helpers::locations::{LineIndex, resolve_locations}; +use crate::classification::models::Occurrence; +use crate::parsing::models::SourceDialect; + +/// Every construct in `statement`, in source order, positioned against the `source` it came from. +pub fn classify(statement: &Expression, source: &str, dialect: SourceDialect) -> Vec { + let mut nodes: Vec<&Expression> = Vec::new(); + let mut parents: Vec> = Vec::new(); + let mut pending: Vec<(&Expression, Option)> = vec![(statement, None)]; + while let Some((node, parent)) = pending.pop() { + let position = nodes.len(); + nodes.push(node); + parents.push(parent); + pending.extend( + node.children() + .into_iter() + .rev() + .map(|child| (child, Some(position))), + ); + } + let locations = resolve_locations(&nodes, &parents, &LineIndex::new(source)); + let mut occurrences: Vec = Vec::new(); + for (node, location) in nodes.iter().zip(locations) { + let constructs = node_constructs(node, dialect.polyglot()) + .into_iter() + .chain(clause_constructs(node)); + occurrences.extend(constructs.map(|construct| Occurrence { + construct, + location, + })); + } + occurrences +} diff --git a/crates/sqlquack-core/src/classification/main/mod.rs b/crates/sqlquack-core/src/classification/main/mod.rs new file mode 100644 index 000000000..5d34d6f08 --- /dev/null +++ b/crates/sqlquack-core/src/classification/main/mod.rs @@ -0,0 +1 @@ +pub mod classify; diff --git a/crates/sqlquack-core/src/classification/mod.rs b/crates/sqlquack-core/src/classification/mod.rs new file mode 100644 index 000000000..4969ea88d --- /dev/null +++ b/crates/sqlquack-core/src/classification/mod.rs @@ -0,0 +1,9 @@ +//! Classification of every function, operator, type, cast, join and clause in a statement. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-core/src/classification/models.rs b/crates/sqlquack-core/src/classification/models.rs new file mode 100644 index 000000000..51641d027 --- /dev/null +++ b/crates/sqlquack-core/src/classification/models.rs @@ -0,0 +1,84 @@ +//! Constructs and their occurrences. + +use std::fmt::{self, Display}; + +use serde::{Serialize, Serializer}; + +use crate::parsing::models::SourceLocation; + +/// The kind of thing a construct names. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ConstructFamily { + /// An AST node kind, such as `select`, `eq` or `coalesce`. + Node, + /// A function the parser kept as a generic call, by upper-case name. + Function, + /// The base target type of a cast. + Cast, + /// A join kind. + Join, + /// A query clause, such as `qualify` or `group_by`. + Clause, + /// A spelling with its own meaning, such as `::` casts. + Syntax, +} + +impl ConstructFamily { + /// The prefix used in construct keys. + pub fn prefix(self) -> &'static str { + match self { + Self::Node => "node", + Self::Function => "fn", + Self::Cast => "cast", + Self::Join => "join", + Self::Clause => "clause", + Self::Syntax => "syntax", + } + } +} + +/// One source construct that needs a conversion rule, keyed as `:`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Construct { + /// What the construct names. + pub family: ConstructFamily, + /// The node kind, function name, type, join kind, clause or syntax. + pub name: String, +} + +impl Construct { + /// A construct of `family` called `name`. + pub fn new(family: ConstructFamily, name: impl Into) -> Self { + Self { + family, + name: name.into(), + } + } + + /// The `:` key used in reports. + pub fn key(&self) -> String { + self.to_string() + } +} + +impl Display for Construct { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.family.prefix(), self.name) + } +} + +impl Serialize for Construct { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +/// One use of a construct in a statement. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct Occurrence { + /// The construct used. + pub construct: Construct, + /// Where it is, when polyglot kept a position for the node or one of its neighbours. + pub location: Option, +} diff --git a/crates/sqlquack-core/src/classification/tests.rs b/crates/sqlquack-core/src/classification/tests.rs new file mode 100644 index 000000000..a0c06499d --- /dev/null +++ b/crates/sqlquack-core/src/classification/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_classify.rs"] +mod test_classify; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-core/src/classification/tests/helpers.rs b/crates/sqlquack-core/src/classification/tests/helpers.rs new file mode 100644 index 000000000..3965dc77e --- /dev/null +++ b/crates/sqlquack-core/src/classification/tests/helpers.rs @@ -0,0 +1,57 @@ +use std::collections::BTreeSet; + +use polyglot_sql::Expression; +use polyglot_sql::traversal::ExpressionWalk; + +use crate::classification::main::classify::classify; +use crate::classification::models::{ConstructFamily, Occurrence}; +use crate::parsing::main::parse_source::parse_source; +use crate::parsing::models::SourceDialect; + +pub(crate) fn parse_one(sql: &str) -> Expression { + let mut statements = parse_source(sql, SourceDialect::Snowflake).expect("test SQL parses"); + assert_eq!(statements.len(), 1, "test SQL holds one statement"); + statements.remove(0) +} + +fn occurrences(sql: &str) -> Vec { + classify(&parse_one(sql), sql, SourceDialect::Snowflake) +} + +pub(crate) fn missing_keys(sql: &str, expected: &[&str]) -> Vec { + let found: BTreeSet = occurrences(sql) + .into_iter() + .map(|occurrence| occurrence.construct.key()) + .collect(); + expected + .iter() + .filter(|key| !found.contains(**key)) + .map(|key| (*key).to_owned()) + .collect() +} + +pub(crate) fn unclassified_nodes(sql: &str) -> usize { + let identities = occurrences(sql) + .iter() + .filter(|occurrence| { + [ConstructFamily::Node, ConstructFamily::Function] + .contains(&occurrence.construct.family) + }) + .count(); + parse_one(sql).dfs().count().abs_diff(identities) +} + +pub(crate) fn unlocated_constructs(sql: &str) -> usize { + occurrences(sql) + .iter() + .filter(|occurrence| occurrence.location.is_none()) + .count() +} + +pub(crate) fn line_and_column(sql: &str, key: &str) -> Option<[usize; 2]> { + occurrences(sql) + .into_iter() + .find(|occurrence| occurrence.construct.key() == key) + .and_then(|occurrence| occurrence.location) + .map(|location| [location.line, location.column]) +} diff --git a/crates/sqlquack-core/src/classification/tests/test_classify.rs b/crates/sqlquack-core/src/classification/tests/test_classify.rs new file mode 100644 index 000000000..8d1dd8408 --- /dev/null +++ b/crates/sqlquack-core/src/classification/tests/test_classify.rs @@ -0,0 +1,126 @@ +use crate::classification::tests::helpers::{ + line_and_column, missing_keys, unclassified_nodes, unlocated_constructs, +}; +use crate::classification::tests::test_types::{ + CompletenessTestCase, ConstructTestCase, LocationTestCase, +}; + +#[test] +fn given_snowflake_query_when_classifying_then_reports_each_construct_family() { + let test_cases = [ + ConstructTestCase { + description: "generic functions are keyed by upper-case name", + sql: "SELECT zeroifnull(o.amount) AS amount FROM orders o", + expected_keys: &["fn:ZEROIFNULL", "node:alias", "node:column", "node:select"], + }, + ConstructTestCase { + description: "casts record the node, polyglot's base type and the `::` spelling", + sql: "SELECT o.amount::NUMBER(10, 2) AS amount FROM orders o", + expected_keys: &["node:cast", "cast:DECIMAL", "syntax:double_colon_cast"], + }, + ConstructTestCase { + description: "function-style casts record the node and the base type", + sql: "SELECT CAST(o.ordered_at AS DATE) FROM orders o", + expected_keys: &["node:cast", "cast:DATE"], + }, + ConstructTestCase { + description: "join kinds and query clauses are constructs", + sql: "SELECT c.customer_id FROM customers c LEFT JOIN orders o \ + ON o.customer_id = c.customer_id WHERE o.order_id IS NULL \ + QUALIFY ROW_NUMBER() OVER (PARTITION BY c.customer_id ORDER BY c.created_at) = 1", + expected_keys: &[ + "join:left", + "clause:where", + "clause:qualify", + "node:eq", + "node:is_null", + "node:row_number", + "node:window_function", + ], + }, + ConstructTestCase { + description: "set operations record their ALL form", + sql: "SELECT product_id FROM products UNION ALL SELECT product_id FROM inventory", + expected_keys: &["node:union", "clause:union_all"], + }, + ConstructTestCase { + description: "project functions collapse into one construct", + sql: "SELECT __sqlquack_udf(o.status) FROM orders o", + expected_keys: &["fn:"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + missing_keys(test_case.sql, test_case.expected_keys), + Vec::::new(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_any_query_when_classifying_then_every_node_has_a_located_construct() { + let test_cases = [ + CompletenessTestCase { + description: "a CTE with aggregates, joins, conditionals and ordering", + sql: "WITH recent AS (SELECT customer_id, SUM(amount) AS total FROM orders \ + WHERE ordered_at >= DATEADD('day', -30, CURRENT_DATE()) GROUP BY customer_id) \ + SELECT c.name, COALESCE(r.total, 0) AS total, IFF(r.total > 100, 'high', 'low') \ + FROM customers c LEFT JOIN recent r ON r.customer_id = c.customer_id \ + ORDER BY total DESC NULLS LAST LIMIT 10", + expected_unclassified_nodes: 0, + expected_unlocated_constructs: 0, + }, + CompletenessTestCase { + description: "semi-structured paths and a lateral flatten", + sql: "SELECT o.payload:customer.name::VARCHAR AS name, f.value \ + FROM orders o, LATERAL FLATTEN(input => o.payload:items) f", + expected_unclassified_nodes: 0, + expected_unlocated_constructs: 0, + }, + ]; + + for test_case in test_cases { + assert_eq!( + unclassified_nodes(test_case.sql), + test_case.expected_unclassified_nodes, + "{}", + test_case.description + ); + assert_eq!( + unlocated_constructs(test_case.sql), + test_case.expected_unlocated_constructs, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_multi_line_query_when_classifying_then_locations_point_into_the_source() { + let test_cases = [ + LocationTestCase { + description: "a column keeps its own line and column", + sql: "SELECT\n o.order_id\nFROM orders o", + key: "node:column", + expected_line_and_column: Some([2, 3]), + }, + LocationTestCase { + description: "a call without a position of its own takes its first argument's", + sql: "SELECT\n o.order_id,\n ZEROIFNULL(o.amount) AS amount\nFROM orders o", + key: "fn:ZEROIFNULL", + expected_line_and_column: Some([3, 14]), + }, + ]; + + for test_case in test_cases { + assert_eq!( + line_and_column(test_case.sql, test_case.key), + test_case.expected_line_and_column, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/classification/tests/test_types.rs b/crates/sqlquack-core/src/classification/tests/test_types.rs new file mode 100644 index 000000000..10582c84b --- /dev/null +++ b/crates/sqlquack-core/src/classification/tests/test_types.rs @@ -0,0 +1,19 @@ +pub(crate) struct ConstructTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_keys: &'static [&'static str], +} + +pub(crate) struct CompletenessTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_unclassified_nodes: usize, + pub(crate) expected_unlocated_constructs: usize, +} + +pub(crate) struct LocationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) key: &'static str, + pub(crate) expected_line_and_column: Option<[usize; 2]>, +} diff --git a/crates/sqlquack-core/src/comparison/_helpers/difference.rs b/crates/sqlquack-core/src/comparison/_helpers/difference.rs new file mode 100644 index 000000000..cc86511b8 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/_helpers/difference.rs @@ -0,0 +1,60 @@ +//! The first differing path between two normalised trees. + +use serde_json::Value; + +use crate::comparison::constants::DIFFERENCE_EXCERPT_CHARACTERS; +use crate::comparison::models::TreeDifference; + +const ABSENT: &str = ""; + +pub(crate) fn first_difference(expected: &Value, actual: &Value) -> Option { + let mut pending: Vec<(String, Option<&Value>, Option<&Value>)> = + vec![(String::new(), Some(expected), Some(actual))]; + while let Some((path, expected, actual)) = pending.pop() { + match (expected, actual) { + (Some(Value::Object(left)), Some(Value::Object(right))) => { + let mut keys: Vec<&String> = left.keys().chain(right.keys()).collect(); + keys.sort(); + keys.dedup(); + for key in keys.into_iter().rev() { + pending.push((format!("{path}.{key}"), left.get(key), right.get(key))); + } + } + (Some(Value::Array(left)), Some(Value::Array(right))) => { + for index in (0..left.len().max(right.len())).rev() { + pending.push(( + format!("{path}[{index}]"), + left.get(index), + right.get(index), + )); + } + } + (left, right) if left == right => {} + (left, right) => { + return Some(TreeDifference { + path: if path.is_empty() { + ".".to_owned() + } else { + path + }, + expected: excerpt(left), + actual: excerpt(right), + }); + } + } + } + None +} + +fn excerpt(value: Option<&Value>) -> String { + let Some(value) = value else { + return ABSENT.to_owned(); + }; + let text = value.to_string(); + if text.chars().count() <= DIFFERENCE_EXCERPT_CHARACTERS { + return text; + } + let mut shortened: String = text.chars().take(DIFFERENCE_EXCERPT_CHARACTERS).collect(); + shortened.push('…'); + shortened +} diff --git a/crates/sqlquack-core/src/comparison/_helpers/mod.rs b/crates/sqlquack-core/src/comparison/_helpers/mod.rs new file mode 100644 index 000000000..3d72e2aff --- /dev/null +++ b/crates/sqlquack-core/src/comparison/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod difference; +pub(crate) mod normalise; diff --git a/crates/sqlquack-core/src/comparison/_helpers/normalise.rs b/crates/sqlquack-core/src/comparison/_helpers/normalise.rs new file mode 100644 index 000000000..92e5706d5 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/_helpers/normalise.rs @@ -0,0 +1,51 @@ +//! Rewrites a serialised tree to the spelling-free form both sides are compared in. + +use polyglot_sql::Expression; +use serde_json::{Map, Value}; + +use crate::comparison::constants::SPELLING_ONLY_FIELDS; + +const ANNOTATED: &str = "annotated"; +const LITERAL_TYPE: &str = "literal_type"; +const DOLLAR_STRING: &str = "dollar_string"; +const STRING: &str = "string"; + +pub(crate) fn normalise(value: Value) -> Value { + match value { + Value::Array(items) => Value::Array(items.into_iter().map(normalise).collect()), + Value::Object(object) => normalise_object(object), + other => other, + } +} + +fn normalise_object(object: Map) -> Value { + if object.len() == 1 + && let Some(Value::Object(annotated)) = object.get(ANNOTATED) + && let Some(inner) = annotated.get("this") + { + return normalise(inner.clone()); + } + let mut normalised = Map::new(); + let function_like = + object.contains_key("args") && !matches!(object.get("quoted"), Some(Value::Bool(true))); + for (key, value) in object { + if SPELLING_ONLY_FIELDS.iter().any(|field| field.matches(&key)) { + continue; + } + let value = match (key.as_str(), value) { + ("name", Value::String(name)) if function_like => { + Value::String(name.to_ascii_uppercase()) + } + (LITERAL_TYPE, Value::String(kind)) if kind == DOLLAR_STRING => { + Value::String(STRING.to_owned()) + } + (_, value) => normalise(value), + }; + normalised.insert(key, value); + } + Value::Object(normalised) +} + +pub(crate) fn normalised_tree(statement: &Expression) -> Result { + serde_json::to_value(statement).map(normalise) +} diff --git a/crates/sqlquack-core/src/comparison/constants.rs b/crates/sqlquack-core/src/comparison/constants.rs new file mode 100644 index 000000000..8f3c459f1 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/constants.rs @@ -0,0 +1,76 @@ +//! The declared spelling-only fields and normalisations, each with a reason and a test. + +use crate::comparison::models::{FieldMatch, SpellingNormalisation, SpellingOnlyField}; + +/// Fields removed from both trees before comparison. +pub const SPELLING_ONLY_FIELDS: &[SpellingOnlyField] = &[ + SpellingOnlyField { + name: "span", + matching: FieldMatch::Exact, + reason: "source positions move with whitespace and layout", + }, + SpellingOnlyField { + name: "inferred_type", + matching: FieldMatch::Exact, + reason: "type annotation fills it after parsing; trees are compared as parsed", + }, + SpellingOnlyField { + name: "comments", + matching: FieldMatch::Contains, + reason: "comments carry no meaning", + }, + SpellingOnlyField { + name: "double_colon_syntax", + matching: FieldMatch::Exact, + reason: "`x::T` and `CAST(x AS T)` are the same cast once parsed", + }, + SpellingOnlyField { + name: "alias_explicit_as", + matching: FieldMatch::Exact, + reason: "`x AS y` and `x y` name the same alias", + }, + SpellingOnlyField { + name: "alias_keyword", + matching: FieldMatch::Exact, + reason: "the letter case of the AS keyword", + }, + SpellingOnlyField { + name: "use_inner_keyword", + matching: FieldMatch::Exact, + reason: "`JOIN` and `INNER JOIN` are the same join", + }, + SpellingOnlyField { + name: "use_outer_keyword", + matching: FieldMatch::Exact, + reason: "`LEFT JOIN` and `LEFT OUTER JOIN` are the same join", + }, + SpellingOnlyField { + name: "original_name", + matching: FieldMatch::Exact, + reason: "the node kind fixes the function; this keeps only the alias it was written with", + }, + SpellingOnlyField { + name: "integer_spelling", + matching: FieldMatch::Exact, + reason: "`INTEGER` and `INT` are the same type", + }, +]; + +/// Values rewritten to one spelling in both trees before comparison. +pub const SPELLING_NORMALISATIONS: &[SpellingNormalisation] = &[ + SpellingNormalisation { + description: "unquoted function names (`name` beside `args`) are upper-cased", + reason: "unquoted function names are case-insensitive, and printers upper-case them", + }, + SpellingNormalisation { + description: "dollar-quoted string literals become ordinary string literals", + reason: "`$$a$$` and `'a'` are the same string value", + }, + SpellingNormalisation { + description: "comment wrappers (`annotated`) are replaced by the node they wrap", + reason: "comments carry no meaning", + }, +]; + +/// The longest JSON excerpt a difference reports for each side. +pub const DIFFERENCE_EXCERPT_CHARACTERS: usize = 160; diff --git a/crates/sqlquack-core/src/comparison/main/compare_trees.rs b/crates/sqlquack-core/src/comparison/main/compare_trees.rs new file mode 100644 index 000000000..64df58b44 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/main/compare_trees.rs @@ -0,0 +1,26 @@ +//! `compare_trees`: whether two statements mean the same, up to spelling. + +use polyglot_sql::Expression; + +use crate::comparison::_helpers::difference::first_difference; +use crate::comparison::_helpers::normalise::normalised_tree; +use crate::comparison::models::TreeDifference; + +/// The first difference after normalisation, or `None`; an unserialisable tree never matches. +pub fn compare_trees(expected: &Expression, actual: &Expression) -> Option { + match (normalised_tree(expected), normalised_tree(actual)) { + (Ok(expected), Ok(actual)) => first_difference(&expected, &actual), + (expected, actual) => Some(TreeDifference { + path: ".".to_owned(), + expected: describe(expected.err()), + actual: describe(actual.err()), + }), + } +} + +fn describe(error: Option) -> String { + match error { + Some(error) => format!(""), + None => "".to_owned(), + } +} diff --git a/crates/sqlquack-core/src/comparison/main/mod.rs b/crates/sqlquack-core/src/comparison/main/mod.rs new file mode 100644 index 000000000..8b82bfb3a --- /dev/null +++ b/crates/sqlquack-core/src/comparison/main/mod.rs @@ -0,0 +1 @@ +pub mod compare_trees; diff --git a/crates/sqlquack-core/src/comparison/mod.rs b/crates/sqlquack-core/src/comparison/mod.rs new file mode 100644 index 000000000..fe835ee2a --- /dev/null +++ b/crates/sqlquack-core/src/comparison/mod.rs @@ -0,0 +1,9 @@ +//! Tree comparison as JSON, ignoring a declared list of spelling-only fields. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-core/src/comparison/models.rs b/crates/sqlquack-core/src/comparison/models.rs new file mode 100644 index 000000000..2b9a4703f --- /dev/null +++ b/crates/sqlquack-core/src/comparison/models.rs @@ -0,0 +1,65 @@ +//! Comparison declarations and results. + +use std::fmt::{self, Display}; + +use serde::Serialize; + +/// How a declared field name matches JSON object keys. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FieldMatch { + /// Keys equal to the name. + Exact, + /// Keys containing the name. + Contains, +} + +/// A field whose value records only how SQL was spelled, never what it means. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SpellingOnlyField { + /// The JSON key, or the part of it, that is ignored. + pub name: &'static str, + /// How `name` matches keys. + pub matching: FieldMatch, + /// Why the field cannot change meaning. + pub reason: &'static str, +} + +impl SpellingOnlyField { + /// Whether `key` is this field. + pub fn matches(&self, key: &str) -> bool { + match self.matching { + FieldMatch::Exact => key == self.name, + FieldMatch::Contains => key.contains(self.name), + } + } +} + +/// A value rewritten to one spelling before comparison. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SpellingNormalisation { + /// What is rewritten. + pub description: &'static str, + /// Why both spellings mean the same. + pub reason: &'static str, +} + +/// The first place two trees differ. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct TreeDifference { + /// JSON path of the first differing value, such as `.select.expressions[2]`. + pub path: String, + /// The expected value there, as compact JSON, shortened. + pub expected: String, + /// The actual value there, as compact JSON, shortened. + pub actual: String, +} + +impl Display for TreeDifference { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "trees differ at {}: expected {}, found {}", + self.path, self.expected, self.actual + ) + } +} diff --git a/crates/sqlquack-core/src/comparison/tests.rs b/crates/sqlquack-core/src/comparison/tests.rs new file mode 100644 index 000000000..0c447c34f --- /dev/null +++ b/crates/sqlquack-core/src/comparison/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_compare_trees.rs"] +mod test_compare_trees; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-core/src/comparison/tests/helpers.rs b/crates/sqlquack-core/src/comparison/tests/helpers.rs new file mode 100644 index 000000000..123eb0c18 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/tests/helpers.rs @@ -0,0 +1,60 @@ +use polyglot_sql::Expression; +use polyglot_sql::expressions::DataType; +use serde_json::Value; + +use crate::comparison::constants::SPELLING_ONLY_FIELDS; +use crate::comparison::main::compare_trees::compare_trees; +use crate::parsing::main::parse_source::parse_source; +use crate::parsing::models::SourceDialect; + +pub(crate) fn parse_one(sql: &str) -> Expression { + let mut statements = parse_source(sql, SourceDialect::Snowflake).expect("test SQL parses"); + assert_eq!(statements.len(), 1, "test SQL holds one statement"); + statements.remove(0) +} + +fn raw(tree: &Expression) -> Value { + serde_json::to_value(tree).expect("trees serialise") +} + +fn contains_key(value: &Value, wanted: &str) -> bool { + let in_object = value.as_object().is_some_and(|object| { + object + .iter() + .any(|(key, inner)| key.contains(wanted) || contains_key(inner, wanted)) + }); + let in_array = value + .as_array() + .is_some_and(|items| items.iter().any(|item| contains_key(item, wanted))); + in_object || in_array +} + +pub(crate) fn raw_trees_equal(left: &Expression, right: &Expression) -> bool { + raw(left) == raw(right) +} + +pub(crate) fn field_present(left: &Expression, right: &Expression, field: &str) -> bool { + contains_key(&raw(left), field) || contains_key(&raw(right), field) +} + +pub(crate) fn difference_path(left: &Expression, right: &Expression) -> Option { + compare_trees(left, right).map(|difference| difference.path) +} + +pub(crate) fn sql_difference_path(left: &str, right: &str) -> Option { + difference_path(&parse_one(left), &parse_one(right)) +} + +pub(crate) fn untested_fields(tested: &[&str]) -> Vec<&'static str> { + SPELLING_ONLY_FIELDS + .iter() + .map(|field| field.name) + .filter(|name| !tested.contains(name)) + .collect() +} + +pub(crate) fn typed_column(name: &str) -> Expression { + let mut column = Expression::column(name); + column.set_inferred_type(DataType::Boolean); + column +} diff --git a/crates/sqlquack-core/src/comparison/tests/test_compare_trees.rs b/crates/sqlquack-core/src/comparison/tests/test_compare_trees.rs new file mode 100644 index 000000000..a7659a830 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/tests/test_compare_trees.rs @@ -0,0 +1,238 @@ +use polyglot_sql::Expression; + +use crate::comparison::tests::helpers::{ + difference_path, field_present, parse_one, raw_trees_equal, sql_difference_path, typed_column, + untested_fields, +}; +use crate::comparison::tests::test_types::{ + DeclarationTestCase, DifferenceTestCase, SpellingTestCase, TreeTestCase, +}; + +#[test] +fn given_spellings_differing_only_in_a_declared_field_when_comparing_then_trees_match() { + let test_cases = [ + SpellingTestCase { + description: "whitespace moves source positions", + field: "span", + left: "SELECT order_id FROM orders", + right: "SELECT order_id FROM orders", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "a comment", + field: "comments", + left: "SELECT order_id /* the key */ FROM orders", + right: "SELECT order_id FROM orders", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "`::` and CAST", + field: "double_colon_syntax", + left: "SELECT amount::INT FROM orders", + right: "SELECT CAST(amount AS INT) FROM orders", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "an alias with and without AS", + field: "alias_explicit_as", + left: "SELECT amount AS total FROM orders", + right: "SELECT amount total FROM orders", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "the letter case of AS", + field: "alias_keyword", + left: "SELECT * FROM (SELECT 1 AS n) as numbers", + right: "SELECT * FROM (SELECT 1 AS n) AS numbers", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "JOIN and INNER JOIN", + field: "use_inner_keyword", + left: "SELECT * FROM orders o INNER JOIN customers c ON o.customer_id = c.customer_id", + right: "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.customer_id", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "LEFT JOIN and LEFT OUTER JOIN", + field: "use_outer_keyword", + left: "SELECT * FROM orders o LEFT OUTER JOIN customers c \ + ON o.customer_id = c.customer_id", + right: "SELECT * FROM orders o LEFT JOIN customers c ON o.customer_id = c.customer_id", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "the name a typed function node was written with", + field: "original_name", + left: "SELECT count(sku) FROM products", + right: "SELECT COUNT(sku) FROM products", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "INTEGER and INT", + field: "integer_spelling", + left: "SELECT quantity::INTEGER FROM inventory", + right: "SELECT quantity::INT FROM inventory", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "the letter case of a generic function name", + field: "name", + left: "SELECT my_lookup(sku) FROM products", + right: "SELECT MY_LOOKUP(sku) FROM products", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + SpellingTestCase { + description: "a dollar-quoted string and a quoted string", + field: "literal_type", + left: "SELECT $$fragile$$ FROM products", + right: "SELECT 'fragile' FROM products", + expected_raw_trees_equal: false, + expected_field_present: true, + expected_difference_path: None, + }, + ]; + + for test_case in test_cases { + let left = parse_one(test_case.left); + let right = parse_one(test_case.right); + assert_eq!( + raw_trees_equal(&left, &right), + test_case.expected_raw_trees_equal, + "{}", + test_case.description + ); + assert_eq!( + field_present(&left, &right, test_case.field), + test_case.expected_field_present, + "{}", + test_case.description + ); + assert_eq!( + difference_path(&left, &right), + test_case.expected_difference_path, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_declared_spelling_only_fields_when_listing_tests_then_each_field_has_one() { + let test_cases = [DeclarationTestCase { + description: "every declared field has a spelling test above or below", + tested_fields: &[ + "span", + "inferred_type", + "comments", + "double_colon_syntax", + "alias_explicit_as", + "alias_keyword", + "use_inner_keyword", + "use_outer_keyword", + "original_name", + "integer_spelling", + ], + expected_untested_fields: Vec::new(), + }]; + + for test_case in test_cases { + assert_eq!( + untested_fields(test_case.tested_fields), + test_case.expected_untested_fields, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_type_annotation_when_comparing_then_inferred_type_is_ignored() { + let test_cases = [TreeTestCase { + description: "a column with and without an inferred type", + left: Expression::column("order_id"), + right: typed_column("order_id"), + expected_raw_trees_equal: false, + expected_difference_path: None, + }]; + + for test_case in test_cases { + assert_eq!( + raw_trees_equal(&test_case.left, &test_case.right), + test_case.expected_raw_trees_equal, + "{}", + test_case.description + ); + assert_eq!( + difference_path(&test_case.left, &test_case.right), + test_case.expected_difference_path, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_trees_with_different_meaning_when_comparing_then_reports_first_differing_path() { + let test_cases = [ + DifferenceTestCase { + description: "a different column", + left: "SELECT order_id FROM orders", + right: "SELECT customer_id FROM orders", + expected_difference_path: Some(".select.expressions[0].column.name.name".to_owned()), + }, + DifferenceTestCase { + description: "a different literal", + left: "SELECT 'shipped' FROM orders", + right: "SELECT 'pending' FROM orders", + expected_difference_path: Some(".select.expressions[0].literal.value".to_owned()), + }, + DifferenceTestCase { + description: "a quoted identifier keeps its case", + left: "SELECT \"Status\" FROM orders", + right: "SELECT \"STATUS\" FROM orders", + expected_difference_path: Some(".select.expressions[0].column.name.name".to_owned()), + }, + DifferenceTestCase { + description: "a quoted function name keeps its case", + left: "SELECT \"Lookup\"(sku) FROM products", + right: "SELECT \"LOOKUP\"(sku) FROM products", + expected_difference_path: Some(".select.expressions[0].function.name".to_owned()), + }, + DifferenceTestCase { + description: "an extra projection", + left: "SELECT order_id FROM orders", + right: "SELECT order_id, status FROM orders", + expected_difference_path: Some(".select.expressions[1]".to_owned()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + sql_difference_path(test_case.left, test_case.right), + test_case.expected_difference_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/comparison/tests/test_types.rs b/crates/sqlquack-core/src/comparison/tests/test_types.rs new file mode 100644 index 000000000..351530f04 --- /dev/null +++ b/crates/sqlquack-core/src/comparison/tests/test_types.rs @@ -0,0 +1,32 @@ +use polyglot_sql::Expression; + +pub(crate) struct SpellingTestCase { + pub(crate) description: &'static str, + pub(crate) field: &'static str, + pub(crate) left: &'static str, + pub(crate) right: &'static str, + pub(crate) expected_raw_trees_equal: bool, + pub(crate) expected_field_present: bool, + pub(crate) expected_difference_path: Option, +} + +pub(crate) struct DeclarationTestCase { + pub(crate) description: &'static str, + pub(crate) tested_fields: &'static [&'static str], + pub(crate) expected_untested_fields: Vec<&'static str>, +} + +pub(crate) struct TreeTestCase { + pub(crate) description: &'static str, + pub(crate) left: Expression, + pub(crate) right: Expression, + pub(crate) expected_raw_trees_equal: bool, + pub(crate) expected_difference_path: Option, +} + +pub(crate) struct DifferenceTestCase { + pub(crate) description: &'static str, + pub(crate) left: &'static str, + pub(crate) right: &'static str, + pub(crate) expected_difference_path: Option, +} diff --git a/crates/sqlquack-core/src/lib.rs b/crates/sqlquack-core/src/lib.rs new file mode 100644 index 000000000..15c33248b --- /dev/null +++ b/crates/sqlquack-core/src/lib.rs @@ -0,0 +1,9 @@ +//! Parsing, construct classification, refusals and re-parse checks for SQLQuack. + +#![forbid(unsafe_code)] + +pub mod classification; +pub mod comparison; +pub mod parsing; +pub mod refusals; +pub mod reparse; diff --git a/crates/sqlquack-core/src/parsing/_helpers/guarded.rs b/crates/sqlquack-core/src/parsing/_helpers/guarded.rs new file mode 100644 index 000000000..9e78644c5 --- /dev/null +++ b/crates/sqlquack-core/src/parsing/_helpers/guarded.rs @@ -0,0 +1,59 @@ +//! One guarded polyglot parse for any dialect. + +use polyglot_sql::{ComplexityGuardOptions, DialectType, Error, Expression, ParseOptions}; + +use crate::parsing::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::parsing::errors::ParseFailure; +use crate::parsing::models::SourceLocation; + +pub(crate) fn parse_guarded( + sql: &str, + dialect: DialectType, +) -> Result, ParseFailure> { + let options = ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + }), + }; + polyglot_sql::parse_with_options(sql, dialect, &options).map_err(|error| failure(&error)) +} + +fn failure(error: &Error) -> ParseFailure { + let location = match error { + Error::Tokenize { + line, + column, + start, + end, + .. + } + | Error::Parse { + line, + column, + start, + end, + .. + } + | Error::Syntax { + line, + column, + start, + end, + .. + } => Some(SourceLocation { + start: *start, + end: *end, + line: *line, + column: *column, + }), + _ => None, + }; + let message = match error { + Error::Tokenize { message, .. } + | Error::Parse { message, .. } + | Error::Syntax { message, .. } => message.clone(), + other => other.to_string(), + }; + ParseFailure { message, location } +} diff --git a/crates/sqlquack-core/src/parsing/_helpers/mod.rs b/crates/sqlquack-core/src/parsing/_helpers/mod.rs new file mode 100644 index 000000000..51e838fd4 --- /dev/null +++ b/crates/sqlquack-core/src/parsing/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod guarded; diff --git a/crates/sqlquack-core/src/parsing/constants.rs b/crates/sqlquack-core/src/parsing/constants.rs new file mode 100644 index 000000000..3fe01a591 --- /dev/null +++ b/crates/sqlquack-core/src/parsing/constants.rs @@ -0,0 +1,4 @@ +//! Parser limits. + +/// Nested function calls allowed per statement, above polyglot's default for generated SQL. +pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; diff --git a/crates/sqlquack-core/src/parsing/errors.rs b/crates/sqlquack-core/src/parsing/errors.rs new file mode 100644 index 000000000..ac33fabb5 --- /dev/null +++ b/crates/sqlquack-core/src/parsing/errors.rs @@ -0,0 +1,31 @@ +//! Parse failures. + +use std::fmt::{self, Display}; + +use serde::Serialize; + +use crate::parsing::models::SourceLocation; + +/// SQL that polyglot could not parse, with the parser's message and position. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ParseFailure { + /// The parser's message. + pub message: String, + /// Where the parser stopped, when it reported a position. + pub location: Option, +} + +impl Display for ParseFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.location { + Some(location) => write!( + formatter, + "line {}, column {}: {}", + location.line, location.column, self.message + ), + None => formatter.write_str(&self.message), + } + } +} + +impl std::error::Error for ParseFailure {} diff --git a/crates/sqlquack-core/src/parsing/main/mod.rs b/crates/sqlquack-core/src/parsing/main/mod.rs new file mode 100644 index 000000000..96fcc11da --- /dev/null +++ b/crates/sqlquack-core/src/parsing/main/mod.rs @@ -0,0 +1,2 @@ +pub mod parse_source; +pub mod parse_target; diff --git a/crates/sqlquack-core/src/parsing/main/parse_source.rs b/crates/sqlquack-core/src/parsing/main/parse_source.rs new file mode 100644 index 000000000..1ff7572dc --- /dev/null +++ b/crates/sqlquack-core/src/parsing/main/parse_source.rs @@ -0,0 +1,12 @@ +//! `parse_source`: source SQL to polyglot statements. + +use polyglot_sql::Expression; + +use crate::parsing::_helpers::guarded::parse_guarded; +use crate::parsing::errors::ParseFailure; +use crate::parsing::models::SourceDialect; + +/// Parses source SQL, which may hold several statements, in its own dialect. +pub fn parse_source(sql: &str, dialect: SourceDialect) -> Result, ParseFailure> { + parse_guarded(sql, dialect.polyglot()) +} diff --git a/crates/sqlquack-core/src/parsing/main/parse_target.rs b/crates/sqlquack-core/src/parsing/main/parse_target.rs new file mode 100644 index 000000000..86a2c74ee --- /dev/null +++ b/crates/sqlquack-core/src/parsing/main/parse_target.rs @@ -0,0 +1,11 @@ +//! `parse_target`: DuckDB SQL to polyglot statements. + +use polyglot_sql::{DialectType, Expression}; + +use crate::parsing::_helpers::guarded::parse_guarded; +use crate::parsing::errors::ParseFailure; + +/// Parses DuckDB SQL with the same limits as source SQL. +pub fn parse_target(sql: &str) -> Result, ParseFailure> { + parse_guarded(sql, DialectType::DuckDB) +} diff --git a/crates/sqlquack-core/src/parsing/mod.rs b/crates/sqlquack-core/src/parsing/mod.rs new file mode 100644 index 000000000..9e061c2cc --- /dev/null +++ b/crates/sqlquack-core/src/parsing/mod.rs @@ -0,0 +1,7 @@ +//! Guarded parsing of source and target SQL with polyglot. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlquack-core/src/parsing/models.rs b/crates/sqlquack-core/src/parsing/models.rs new file mode 100644 index 000000000..f49a2371b --- /dev/null +++ b/crates/sqlquack-core/src/parsing/models.rs @@ -0,0 +1,48 @@ +//! Dialects and source positions of parsed SQL. + +use polyglot_sql::DialectType; +use serde::Serialize; +/// A dialect SQLQuack converts from. Every entry point takes it as a parameter. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SourceDialect { + /// Snowflake SQL. + Snowflake, +} + +impl SourceDialect { + /// The parser and printer dialect for this source. + pub fn polyglot(self) -> DialectType { + match self { + Self::Snowflake => DialectType::Snowflake, + } + } + + /// The lower-case name used on the command line and in reports. + pub fn name(self) -> &'static str { + match self { + Self::Snowflake => "snowflake", + } + } + + /// The source dialect with this command-line name. + pub fn from_name(name: &str) -> Option { + match name.to_ascii_lowercase().as_str() { + "snowflake" => Some(Self::Snowflake), + _ => None, + } + } +} + +/// A position in source SQL, in Unicode characters. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize)] +pub struct SourceLocation { + /// Character offset of the first character. + pub start: usize, + /// Character offset one past the last character. + pub end: usize, + /// One-based line. + pub line: usize, + /// One-based column. + pub column: usize, +} diff --git a/crates/sqlquack-core/src/refusals/main/mod.rs b/crates/sqlquack-core/src/refusals/main/mod.rs new file mode 100644 index 000000000..779100177 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/main/mod.rs @@ -0,0 +1,3 @@ +pub mod refuse_constructs; +pub mod refuse_infidelity; +pub mod refuse_unparsed; diff --git a/crates/sqlquack-core/src/refusals/main/refuse_constructs.rs b/crates/sqlquack-core/src/refusals/main/refuse_constructs.rs new file mode 100644 index 000000000..3b3758b19 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/main/refuse_constructs.rs @@ -0,0 +1,27 @@ +//! `refuse_constructs`: one refusal for every construct no rule converts. + +use crate::classification::models::Occurrence; +use crate::refusals::models::{Disposition, Refusal, RefusalCategory}; +use crate::refusals::types::ConstructPolicy; + +/// Refuses every occurrence the policy cannot convert, in order. +pub fn refuse_constructs(occurrences: &[Occurrence], policy: &dyn ConstructPolicy) -> Vec { + occurrences + .iter() + .map(|occurrence| { + let (category, reason) = match policy.disposition(&occurrence.construct) { + Disposition::Missing => ( + RefusalCategory::MissingRule, + format!("no verified rule converts {}", occurrence.construct), + ), + Disposition::NotExpressible { reason } => (RefusalCategory::NotExpressible, reason), + }; + Refusal { + construct: Some(occurrence.construct.clone()), + location: occurrence.location, + category, + reason, + } + }) + .collect() +} diff --git a/crates/sqlquack-core/src/refusals/main/refuse_infidelity.rs b/crates/sqlquack-core/src/refusals/main/refuse_infidelity.rs new file mode 100644 index 000000000..dfb325794 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/main/refuse_infidelity.rs @@ -0,0 +1,32 @@ +//! `refuse_infidelity`: the refusal for a statement whose print-back changes its tree. + +use crate::refusals::models::{Refusal, RefusalCategory}; +use crate::reparse::models::FidelityOutcome; + +/// Refuses a statement unless printing it back in its own dialect reproduced the same tree. +pub fn refuse_infidelity(outcome: &FidelityOutcome) -> Option { + let reason = match outcome { + FidelityOutcome::Identical => return None, + FidelityOutcome::Canonicalised { difference } => format!( + "printing back canonicalises the tree at {}, and no recorded proof covers that \ + canonicalisation", + difference.path + ), + FidelityOutcome::Unstable { difference } => format!( + "printing back does not reach a stable tree; it changes at {}", + difference.path + ), + FidelityOutcome::PrintFailed { message } => { + format!("the parsed tree does not print back: {message}") + } + FidelityOutcome::ReparseFailed { message } => { + format!("the printed SQL does not parse again: {message}") + } + }; + Some(Refusal { + construct: None, + location: None, + category: RefusalCategory::ParseFidelity, + reason, + }) +} diff --git a/crates/sqlquack-core/src/refusals/main/refuse_unparsed.rs b/crates/sqlquack-core/src/refusals/main/refuse_unparsed.rs new file mode 100644 index 000000000..5434a1e66 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/main/refuse_unparsed.rs @@ -0,0 +1,14 @@ +//! `refuse_unparsed`: the refusal for SQL that does not parse. + +use crate::parsing::errors::ParseFailure; +use crate::refusals::models::{Refusal, RefusalCategory}; + +/// Refuses a whole input that polyglot could not parse. +pub fn refuse_unparsed(failure: &ParseFailure) -> Refusal { + Refusal { + construct: None, + location: failure.location, + category: RefusalCategory::ParseFidelity, + reason: format!("the SQL does not parse: {}", failure.message), + } +} diff --git a/crates/sqlquack-core/src/refusals/mod.rs b/crates/sqlquack-core/src/refusals/mod.rs new file mode 100644 index 000000000..9172ff4bb --- /dev/null +++ b/crates/sqlquack-core/src/refusals/mod.rs @@ -0,0 +1,8 @@ +//! Structured refusals: what SQLQuack will not convert, where, and why. + +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-core/src/refusals/models.rs b/crates/sqlquack-core/src/refusals/models.rs new file mode 100644 index 000000000..508d67a53 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/models.rs @@ -0,0 +1,62 @@ +//! Refusals and the decisions behind them. + +use std::fmt::{self, Display}; + +use serde::Serialize; + +use crate::classification::models::Construct; +use crate::parsing::models::SourceLocation; + +/// Why a construct or statement is refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RefusalCategory { + /// No verified rule converts the construct yet. + MissingRule, + /// DuckDB cannot express the construct's source meaning. + NotExpressible, + /// The SQL did not parse, or printing it back in its own dialect changed the tree. + ParseFidelity, +} + +impl RefusalCategory { + /// The category as it appears in reports. + pub fn label(self) -> &'static str { + match self { + Self::MissingRule => "missing rule", + Self::NotExpressible => "not expressible in DuckDB", + Self::ParseFidelity => "parse fidelity", + } + } +} + +impl Display for RefusalCategory { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.label()) + } +} + +/// One refusal, precise enough to act on. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct Refusal { + /// The refused construct; absent when the whole statement is refused. + pub construct: Option, + /// Where the refused construct or failure is. + pub location: Option, + /// Why it is refused. + pub category: RefusalCategory, + /// The specific reason, such as the differing tree path. + pub reason: String, +} + +/// What the rule catalogue says about one construct. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Disposition { + /// No rule covers the construct. + Missing, + /// The construct is declared impossible to express in DuckDB, for the given reason. + NotExpressible { + /// Why DuckDB cannot express it. + reason: String, + }, +} diff --git a/crates/sqlquack-core/src/refusals/tests.rs b/crates/sqlquack-core/src/refusals/tests.rs new file mode 100644 index 000000000..f8af9c2aa --- /dev/null +++ b/crates/sqlquack-core/src/refusals/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_refusals.rs"] +mod test_refusals; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-core/src/refusals/tests/helpers.rs b/crates/sqlquack-core/src/refusals/tests/helpers.rs new file mode 100644 index 000000000..2ecceda67 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/tests/helpers.rs @@ -0,0 +1,65 @@ +use crate::classification::main::classify::classify; +use crate::classification::models::Occurrence; +use crate::comparison::models::TreeDifference; +use crate::parsing::main::parse_source::parse_source; +use crate::parsing::models::SourceDialect; +use crate::refusals::main::refuse_constructs::refuse_constructs; +use crate::refusals::main::refuse_infidelity::refuse_infidelity; +use crate::refusals::main::refuse_unparsed::refuse_unparsed; +use crate::refusals::models::{Refusal, RefusalCategory}; +use crate::refusals::tests::test_types::DeclaringPolicy; +use crate::reparse::models::FidelityOutcome; + +fn occurrences(sql: &str) -> Vec { + parse_source(sql, SourceDialect::Snowflake) + .expect("test SQL parses") + .iter() + .flat_map(|statement| classify(statement, sql, SourceDialect::Snowflake)) + .collect() +} + +fn refusal_matches(refusal: &Refusal, occurrence: &Occurrence) -> bool { + refusal.construct.as_ref() == Some(&occurrence.construct) + && refusal.location == occurrence.location + && refusal.reason.contains(&occurrence.construct.key()) +} + +pub(crate) fn unmatched_refusals(sql: &str, inexpressible: &'static [&'static str]) -> usize { + let found = occurrences(sql); + let refusals = refuse_constructs(&found, &DeclaringPolicy { inexpressible }); + let matched = refusals + .iter() + .zip(&found) + .filter(|(refusal, occurrence)| refusal_matches(refusal, occurrence)) + .count(); + refusals.len().max(found.len()) - matched +} + +pub(crate) fn not_expressible_reasons( + sql: &str, + inexpressible: &'static [&'static str], +) -> Vec { + refuse_constructs(&occurrences(sql), &DeclaringPolicy { inexpressible }) + .into_iter() + .filter(|refusal| refusal.category == RefusalCategory::NotExpressible) + .map(|refusal| refusal.reason) + .collect() +} + +pub(crate) fn unparsed_refusal(sql: &str) -> Refusal { + refuse_unparsed( + &parse_source(sql, SourceDialect::Snowflake).expect_err("test SQL does not parse"), + ) +} + +pub(crate) fn infidelity_category(outcome: &FidelityOutcome) -> Option { + refuse_infidelity(outcome).map(|refusal| refusal.category) +} + +pub(crate) fn difference() -> TreeDifference { + TreeDifference { + path: ".select.limit".to_owned(), + expected: "1".to_owned(), + actual: "2".to_owned(), + } +} diff --git a/crates/sqlquack-core/src/refusals/tests/test_refusals.rs b/crates/sqlquack-core/src/refusals/tests/test_refusals.rs new file mode 100644 index 000000000..971c369b2 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/tests/test_refusals.rs @@ -0,0 +1,119 @@ +use crate::refusals::models::RefusalCategory; +use crate::refusals::tests::helpers::{ + difference, infidelity_category, not_expressible_reasons, unmatched_refusals, unparsed_refusal, +}; +use crate::refusals::tests::test_types::{ + ConstructRefusalTestCase, InfidelityTestCase, UnparsedTestCase, +}; +use crate::reparse::models::FidelityOutcome; + +#[test] +fn given_catalogue_when_refusing_constructs_then_every_occurrence_gets_its_own_refusal() { + let test_cases = [ + ConstructRefusalTestCase { + description: "an empty catalogue refuses everything as a missing rule", + sql: "SELECT c.customer_id, COUNT(o.order_id) FROM customers c \ + JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id", + inexpressible: &[], + expected_unmatched_refusals: 0, + expected_not_expressible_reasons: Vec::new(), + }, + ConstructRefusalTestCase { + description: "a declared construct is refused with the declared reason", + sql: "SELECT RANDOM() FROM products", + inexpressible: &["node:random"], + expected_unmatched_refusals: 0, + expected_not_expressible_reasons: vec![ + "node:random is declared inexpressible for this test".to_owned(), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + unmatched_refusals(test_case.sql, test_case.inexpressible), + test_case.expected_unmatched_refusals, + "{}", + test_case.description + ); + assert_eq!( + not_expressible_reasons(test_case.sql, test_case.inexpressible), + test_case.expected_not_expressible_reasons, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unparseable_sql_when_refusing_then_reports_parse_fidelity_with_location() { + let test_cases = [UnparsedTestCase { + description: "a SELECT with no projection", + sql: "SELECT FROM WHERE", + expected_category: RefusalCategory::ParseFidelity, + expected_located: true, + }]; + + for test_case in test_cases { + let refusal = unparsed_refusal(test_case.sql); + assert_eq!( + refusal.category, test_case.expected_category, + "{}", + test_case.description + ); + assert_eq!( + refusal.location.is_some(), + test_case.expected_located, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_fidelity_outcome_when_refusing_then_only_identical_trees_pass() { + let test_cases = [ + InfidelityTestCase { + description: "an identical print-back", + outcome: FidelityOutcome::Identical, + expected_category: None, + }, + InfidelityTestCase { + description: "an unproven canonicalisation", + outcome: FidelityOutcome::Canonicalised { + difference: difference(), + }, + expected_category: Some(RefusalCategory::ParseFidelity), + }, + InfidelityTestCase { + description: "a tree that never settles", + outcome: FidelityOutcome::Unstable { + difference: difference(), + }, + expected_category: Some(RefusalCategory::ParseFidelity), + }, + InfidelityTestCase { + description: "a tree that does not print", + outcome: FidelityOutcome::PrintFailed { + message: "printer failed".to_owned(), + }, + expected_category: Some(RefusalCategory::ParseFidelity), + }, + InfidelityTestCase { + description: "a print that does not parse", + outcome: FidelityOutcome::ReparseFailed { + message: "parser failed".to_owned(), + }, + expected_category: Some(RefusalCategory::ParseFidelity), + }, + ]; + + for test_case in test_cases { + assert_eq!( + infidelity_category(&test_case.outcome), + test_case.expected_category, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/refusals/tests/test_types.rs b/crates/sqlquack-core/src/refusals/tests/test_types.rs new file mode 100644 index 000000000..ca8867320 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/tests/test_types.rs @@ -0,0 +1,44 @@ +use crate::classification::models::Construct; +use crate::refusals::models::{Disposition, RefusalCategory}; +use crate::refusals::types::ConstructPolicy; +use crate::reparse::models::FidelityOutcome; + +/// A catalogue declaring the listed constructs inexpressible and nothing else. +pub(crate) struct DeclaringPolicy { + pub(crate) inexpressible: &'static [&'static str], +} + +impl ConstructPolicy for DeclaringPolicy { + fn disposition(&self, construct: &Construct) -> Disposition { + let key = construct.key(); + self.inexpressible + .iter() + .find(|declared| **declared == key) + .map_or(Disposition::Missing, |declared| { + Disposition::NotExpressible { + reason: format!("{declared} is declared inexpressible for this test"), + } + }) + } +} + +pub(crate) struct ConstructRefusalTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) inexpressible: &'static [&'static str], + pub(crate) expected_unmatched_refusals: usize, + pub(crate) expected_not_expressible_reasons: Vec, +} + +pub(crate) struct UnparsedTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_category: RefusalCategory, + pub(crate) expected_located: bool, +} + +pub(crate) struct InfidelityTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: FidelityOutcome, + pub(crate) expected_category: Option, +} diff --git a/crates/sqlquack-core/src/refusals/types.rs b/crates/sqlquack-core/src/refusals/types.rs new file mode 100644 index 000000000..287dabd89 --- /dev/null +++ b/crates/sqlquack-core/src/refusals/types.rs @@ -0,0 +1,10 @@ +//! The rule catalogue seen from the refusal pipeline. + +use crate::classification::models::Construct; +use crate::refusals::models::Disposition; + +/// Decides what happens to each construct. Rule crates implement it. +pub trait ConstructPolicy { + /// The catalogue's decision for `construct`. + fn disposition(&self, construct: &Construct) -> Disposition; +} diff --git a/crates/sqlquack-core/src/reparse/_helpers/mod.rs b/crates/sqlquack-core/src/reparse/_helpers/mod.rs new file mode 100644 index 000000000..887232b1c --- /dev/null +++ b/crates/sqlquack-core/src/reparse/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod single; diff --git a/crates/sqlquack-core/src/reparse/_helpers/single.rs b/crates/sqlquack-core/src/reparse/_helpers/single.rs new file mode 100644 index 000000000..377fcda4f --- /dev/null +++ b/crates/sqlquack-core/src/reparse/_helpers/single.rs @@ -0,0 +1,18 @@ +//! Re-parsing printed SQL that must hold exactly one statement. + +use polyglot_sql::Expression; + +use crate::parsing::errors::ParseFailure; + +pub(crate) fn single_statement( + parsed: Result, ParseFailure>, +) -> Result { + let mut statements = parsed.map_err(|failure| failure.to_string())?; + if statements.len() != 1 { + return Err(format!( + "expected one statement, found {}", + statements.len() + )); + } + Ok(statements.remove(0)) +} diff --git a/crates/sqlquack-core/src/reparse/main/check_duckdb_print.rs b/crates/sqlquack-core/src/reparse/main/check_duckdb_print.rs new file mode 100644 index 000000000..1a058f3a9 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/main/check_duckdb_print.rs @@ -0,0 +1,25 @@ +//! `check_duckdb_print`: DuckDB SQL for an intended tree, only if it parses back to that tree. + +use polyglot_sql::{DialectType, Expression}; + +use crate::comparison::main::compare_trees::compare_trees; +use crate::parsing::main::parse_target::parse_target; +use crate::reparse::_helpers::single::single_statement; +use crate::reparse::models::PrinterCheckFailure; + +/// DuckDB SQL for `intended`, returned only when it parses back to the same tree. +pub fn check_duckdb_print(intended: &Expression) -> Result { + let sql = polyglot_sql::generate(intended, DialectType::DuckDB).map_err(|error| { + PrinterCheckFailure::PrintFailed { + message: error.to_string(), + } + })?; + let reparsed = match single_statement(parse_target(&sql)) { + Ok(reparsed) => reparsed, + Err(message) => return Err(PrinterCheckFailure::ReparseFailed { sql, message }), + }; + match compare_trees(intended, &reparsed) { + None => Ok(sql), + Some(difference) => Err(PrinterCheckFailure::Changed { sql, difference }), + } +} diff --git a/crates/sqlquack-core/src/reparse/main/check_source_fidelity.rs b/crates/sqlquack-core/src/reparse/main/check_source_fidelity.rs new file mode 100644 index 000000000..01415225f --- /dev/null +++ b/crates/sqlquack-core/src/reparse/main/check_source_fidelity.rs @@ -0,0 +1,41 @@ +//! `check_source_fidelity`: whether the parsed tree survives printing back to its own dialect. + +use polyglot_sql::Expression; + +use crate::comparison::main::compare_trees::compare_trees; +use crate::parsing::main::parse_source::parse_source; +use crate::parsing::models::SourceDialect; +use crate::reparse::_helpers::single::single_statement; +use crate::reparse::models::FidelityOutcome; + +/// Prints `statement` in its own dialect, re-parses it and classifies any tree change. +pub fn check_source_fidelity(statement: &Expression, dialect: SourceDialect) -> FidelityOutcome { + let first = match print_and_parse(statement, dialect) { + Ok(first) => first, + Err(outcome) => return outcome, + }; + let Some(change) = compare_trees(statement, &first) else { + return FidelityOutcome::Identical; + }; + let second = match print_and_parse(&first, dialect) { + Ok(second) => second, + Err(outcome) => return outcome, + }; + match compare_trees(&first, &second) { + None => FidelityOutcome::Canonicalised { difference: change }, + Some(difference) => FidelityOutcome::Unstable { difference }, + } +} + +fn print_and_parse( + statement: &Expression, + dialect: SourceDialect, +) -> Result { + let sql = polyglot_sql::generate(statement, dialect.polyglot()).map_err(|error| { + FidelityOutcome::PrintFailed { + message: error.to_string(), + } + })?; + single_statement(parse_source(&sql, dialect)) + .map_err(|message| FidelityOutcome::ReparseFailed { message }) +} diff --git a/crates/sqlquack-core/src/reparse/main/mod.rs b/crates/sqlquack-core/src/reparse/main/mod.rs new file mode 100644 index 000000000..8496a1222 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/main/mod.rs @@ -0,0 +1,2 @@ +pub mod check_duckdb_print; +pub mod check_source_fidelity; diff --git a/crates/sqlquack-core/src/reparse/mod.rs b/crates/sqlquack-core/src/reparse/mod.rs new file mode 100644 index 000000000..100e9fba3 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/mod.rs @@ -0,0 +1,8 @@ +//! The two re-parse checks: source print-back fidelity and DuckDB printer agreement. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-core/src/reparse/models.rs b/crates/sqlquack-core/src/reparse/models.rs new file mode 100644 index 000000000..51d3b2354 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/models.rs @@ -0,0 +1,82 @@ +//! Re-parse check results. + +use std::fmt::{self, Display}; + +use serde::Serialize; + +use crate::comparison::models::TreeDifference; + +/// What printing a parsed statement back in its own dialect did to its tree. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum FidelityOutcome { + /// The printed SQL parses to the same tree. + Identical, + /// The first print changed the tree, which then settles; accepted only with a recorded proof. + Canonicalised { + /// Where the first print changed the tree. + difference: TreeDifference, + }, + /// The tree keeps changing on every print. + Unstable { + /// Where the second print changed the tree again. + difference: TreeDifference, + }, + /// The tree could not be printed. + PrintFailed { + /// The printer's message. + message: String, + }, + /// The printed SQL did not parse back to one statement. + ReparseFailed { + /// The parser's message. + message: String, + }, +} + +/// Why DuckDB SQL printed from an intended tree cannot be trusted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(tag = "failure", rename_all = "snake_case")] +pub enum PrinterCheckFailure { + /// The intended tree could not be printed as DuckDB SQL. + PrintFailed { + /// The printer's message. + message: String, + }, + /// The printed SQL did not parse back as one DuckDB statement. + ReparseFailed { + /// The printed SQL. + sql: String, + /// The parser's message. + message: String, + }, + /// The printed SQL parses to a different tree than the one intended. + Changed { + /// The printed SQL. + sql: String, + /// Where the re-parsed tree differs from the intended one. + difference: TreeDifference, + }, +} + +impl Display for PrinterCheckFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::PrintFailed { message } => { + write!(formatter, "the DuckDB printer failed: {message}") + } + Self::ReparseFailed { sql, message } => { + write!( + formatter, + "printed DuckDB SQL `{sql}` does not parse: {message}" + ) + } + Self::Changed { sql, difference } => write!( + formatter, + "printed DuckDB SQL `{sql}` does not parse to the intended tree: {difference}" + ), + } + } +} + +impl std::error::Error for PrinterCheckFailure {} diff --git a/crates/sqlquack-core/src/reparse/tests.rs b/crates/sqlquack-core/src/reparse/tests.rs new file mode 100644 index 000000000..6e64b3ff0 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_duckdb_print.rs"] +mod test_duckdb_print; +#[path = "tests/test_source_fidelity.rs"] +mod test_source_fidelity; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-core/src/reparse/tests/helpers.rs b/crates/sqlquack-core/src/reparse/tests/helpers.rs new file mode 100644 index 000000000..9fa3f7716 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/tests/helpers.rs @@ -0,0 +1,44 @@ +use polyglot_sql::Expression; +use serde_json::Value; + +use crate::parsing::main::parse_source::parse_source; +use crate::parsing::main::parse_target::parse_target; +use crate::parsing::models::SourceDialect; +use crate::reparse::main::check_duckdb_print::check_duckdb_print; +use crate::reparse::main::check_source_fidelity::check_source_fidelity; + +fn single(mut statements: Vec) -> Expression { + assert_eq!(statements.len(), 1, "test SQL holds one statement"); + statements.remove(0) +} + +fn path_of(report: &Value) -> Option { + report["difference"]["path"].as_str().map(str::to_owned) +} + +pub(crate) fn fidelity_report(sql: &str) -> Value { + let statement = single(parse_source(sql, SourceDialect::Snowflake).expect("test SQL parses")); + serde_json::to_value(check_source_fidelity(&statement, SourceDialect::Snowflake)) + .expect("outcomes serialise") +} + +pub(crate) fn fidelity_outcome(report: &Value) -> String { + report["outcome"].as_str().unwrap_or_default().to_owned() +} + +pub(crate) fn fidelity_path(report: &Value) -> Option { + path_of(report) +} + +pub(crate) fn printer_report(sql: &str) -> Value { + let intended = single(parse_target(sql).expect("test SQL parses")); + serde_json::to_value(check_duckdb_print(&intended).err()).expect("failures serialise") +} + +pub(crate) fn printer_failure(report: &Value) -> Option<&str> { + report["failure"].as_str() +} + +pub(crate) fn printer_path(report: &Value) -> Option { + path_of(report) +} diff --git a/crates/sqlquack-core/src/reparse/tests/test_duckdb_print.rs b/crates/sqlquack-core/src/reparse/tests/test_duckdb_print.rs new file mode 100644 index 000000000..82f6d2b0b --- /dev/null +++ b/crates/sqlquack-core/src/reparse/tests/test_duckdb_print.rs @@ -0,0 +1,43 @@ +use crate::reparse::tests::helpers::{printer_failure, printer_path, printer_report}; +use crate::reparse::tests::test_types::PrinterTestCase; + +#[test] +fn given_intended_duckdb_tree_when_printing_then_refuses_any_tree_change() { + let test_cases = [ + PrinterTestCase { + description: "a grouped query with explicit NULL ordering prints faithfully", + duckdb_sql: "SELECT customer_id, COUNT(*) AS orders FROM orders GROUP BY customer_id \ + ORDER BY orders DESC NULLS LAST", + expected_failure: None, + expected_difference_path: None, + }, + PrinterTestCase { + description: "dayofweek prints as ISODOW, which numbers days differently", + duckdb_sql: "SELECT dayofweek(shipped_on) FROM orders", + expected_failure: Some("changed"), + expected_difference_path: Some(".select.expressions[0].day_of_week".to_owned()), + }, + PrinterTestCase { + description: "the regex match operator prints as a function DuckDB lacks", + duckdb_sql: "SELECT sku ~ '^A' FROM products", + expected_failure: Some("changed"), + expected_difference_path: Some(".select.expressions[0].function".to_owned()), + }, + ]; + + for test_case in test_cases { + let report = printer_report(test_case.duckdb_sql); + assert_eq!( + printer_failure(&report), + test_case.expected_failure, + "{}", + test_case.description + ); + assert_eq!( + printer_path(&report), + test_case.expected_difference_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/reparse/tests/test_source_fidelity.rs b/crates/sqlquack-core/src/reparse/tests/test_source_fidelity.rs new file mode 100644 index 000000000..812f28e40 --- /dev/null +++ b/crates/sqlquack-core/src/reparse/tests/test_source_fidelity.rs @@ -0,0 +1,37 @@ +use crate::reparse::tests::helpers::{fidelity_outcome, fidelity_path, fidelity_report}; +use crate::reparse::tests::test_types::FidelityTestCase; + +#[test] +fn given_snowflake_statement_when_printing_back_then_classifies_the_tree_change() { + let test_cases = [ + FidelityTestCase { + description: "a plain query prints back to the same tree", + sql: "SELECT o.order_id, o.amount::NUMBER(10, 2) AS amount FROM orders o \ + WHERE o.status IN ('shipped', 'delivered') ORDER BY o.order_id", + expected_outcome: "identical", + expected_difference_path: None, + }, + FidelityTestCase { + description: "TOP prints back as LIMIT, a one-step canonicalisation", + sql: "SELECT TOP 5 order_id FROM orders", + expected_outcome: "canonicalised", + expected_difference_path: Some(".select.limit".to_owned()), + }, + ]; + + for test_case in test_cases { + let report = fidelity_report(test_case.sql); + assert_eq!( + fidelity_outcome(&report), + test_case.expected_outcome, + "{}", + test_case.description + ); + assert_eq!( + fidelity_path(&report), + test_case.expected_difference_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/reparse/tests/test_types.rs b/crates/sqlquack-core/src/reparse/tests/test_types.rs new file mode 100644 index 000000000..828f2886a --- /dev/null +++ b/crates/sqlquack-core/src/reparse/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(crate) struct FidelityTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_outcome: &'static str, + pub(crate) expected_difference_path: Option, +} + +pub(crate) struct PrinterTestCase { + pub(crate) description: &'static str, + pub(crate) duckdb_sql: &'static str, + pub(crate) expected_failure: Option<&'static str>, + pub(crate) expected_difference_path: Option, +} diff --git a/crates/sqlquack-duckdb/Cargo.toml b/crates/sqlquack-duckdb/Cargo.toml new file mode 100644 index 000000000..0a6f1700e --- /dev/null +++ b/crates/sqlquack-duckdb/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sqlquack-duckdb" +version.workspace = true +description = "DuckDB building blocks shared by SQLQuack rules." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/crates/sqlquack-duckdb/src/lib.rs b/crates/sqlquack-duckdb/src/lib.rs new file mode 100644 index 000000000..12682f8bc --- /dev/null +++ b/crates/sqlquack-duckdb/src/lib.rs @@ -0,0 +1,3 @@ +//! DuckDB building blocks shared by rules; empty until the first rules land. + +#![forbid(unsafe_code)] diff --git a/crates/sqlquack-python/Cargo.toml b/crates/sqlquack-python/Cargo.toml deleted file mode 100644 index 55b41ca0b..000000000 --- a/crates/sqlquack-python/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "sqlquack-python" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -readme.workspace = true -description = "Python binding crate placeholder for SQLQuack" -publish = false - -[lib] -name = "sqlquack_python" - -[dependencies] -sqlquack = { path = "../sqlquack" } diff --git a/crates/sqlquack-python/src/lib.rs b/crates/sqlquack-python/src/lib.rs deleted file mode 100644 index 6249a7686..000000000 --- a/crates/sqlquack-python/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![forbid(unsafe_code)] - -pub use sqlquack::{ - analyze, AnalysisResult, AnalyzeOptions, Diagnostic, DiagnosticSeverity, Dialect, SqlUnit, -}; diff --git a/crates/sqlquack-rules/Cargo.toml b/crates/sqlquack-rules/Cargo.toml new file mode 100644 index 000000000..01c0e3be3 --- /dev/null +++ b/crates/sqlquack-rules/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sqlquack-rules" +version.workspace = true +description = "The SQLQuack rule catalogue: verified Snowflake-to-DuckDB rules and declared refusals." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sqlquack-core.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-rules/src/catalogue/constants.rs b/crates/sqlquack-rules/src/catalogue/constants.rs new file mode 100644 index 000000000..e540ce91e --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/constants.rs @@ -0,0 +1,6 @@ +//! Declared refusals. + +use crate::catalogue::models::NotExpressibleDeclaration; + +/// Constructs DuckDB cannot express. Each entry needs a reason and evidence before it is added. +pub const NOT_EXPRESSIBLE: &[NotExpressibleDeclaration] = &[]; diff --git a/crates/sqlquack-rules/src/catalogue/main/builtin_catalogue.rs b/crates/sqlquack-rules/src/catalogue/main/builtin_catalogue.rs new file mode 100644 index 000000000..cd5520b32 --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/main/builtin_catalogue.rs @@ -0,0 +1,11 @@ +//! `builtin_catalogue`: the catalogue SQLQuack ships. + +use crate::catalogue::constants::NOT_EXPRESSIBLE; +use crate::catalogue::models::RuleCatalogue; + +/// Every verified rule and declared refusal. +pub fn builtin_catalogue() -> RuleCatalogue { + RuleCatalogue { + not_expressible: NOT_EXPRESSIBLE.to_vec(), + } +} diff --git a/crates/sqlquack-rules/src/catalogue/main/mod.rs b/crates/sqlquack-rules/src/catalogue/main/mod.rs new file mode 100644 index 000000000..d2df9bfc0 --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/main/mod.rs @@ -0,0 +1 @@ +pub mod builtin_catalogue; diff --git a/crates/sqlquack-rules/src/catalogue/mod.rs b/crates/sqlquack-rules/src/catalogue/mod.rs new file mode 100644 index 000000000..6e1a24a8e --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/mod.rs @@ -0,0 +1,8 @@ +//! The built-in catalogue. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-rules/src/catalogue/models.rs b/crates/sqlquack-rules/src/catalogue/models.rs new file mode 100644 index 000000000..b747175c7 --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/models.rs @@ -0,0 +1,37 @@ +//! Catalogue entries. + +use sqlquack_core::classification::models::Construct; +use sqlquack_core::refusals::models::Disposition; +use sqlquack_core::refusals::types::ConstructPolicy; + +/// A construct DuckDB cannot express, and why. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NotExpressibleDeclaration { + /// The construct key, such as `fn:EXAMPLE`. + pub construct: &'static str, + /// Why DuckDB cannot express it. + pub reason: &'static str, +} + +/// The rules and declarations SQLQuack applies. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleCatalogue { + /// Constructs declared impossible to express in DuckDB. + pub not_expressible: Vec, +} + +impl ConstructPolicy for RuleCatalogue { + fn disposition(&self, construct: &Construct) -> Disposition { + let key = construct.key(); + match self + .not_expressible + .iter() + .find(|declaration| declaration.construct == key) + { + Some(declaration) => Disposition::NotExpressible { + reason: declaration.reason.to_owned(), + }, + None => Disposition::Missing, + } + } +} diff --git a/crates/sqlquack-rules/src/catalogue/tests.rs b/crates/sqlquack-rules/src/catalogue/tests.rs new file mode 100644 index 000000000..31888c621 --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_catalogue.rs"] +mod test_catalogue; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-rules/src/catalogue/tests/test_catalogue.rs b/crates/sqlquack-rules/src/catalogue/tests/test_catalogue.rs new file mode 100644 index 000000000..2e926d25d --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/tests/test_catalogue.rs @@ -0,0 +1,54 @@ +use sqlquack_core::classification::models::{Construct, ConstructFamily}; +use sqlquack_core::refusals::models::Disposition; +use sqlquack_core::refusals::types::ConstructPolicy; + +use crate::catalogue::main::builtin_catalogue::builtin_catalogue; +use crate::catalogue::models::{NotExpressibleDeclaration, RuleCatalogue}; +use crate::catalogue::tests::test_types::CatalogueTestCase; + +#[test] +fn given_catalogue_when_asked_about_construct_then_reports_its_disposition() { + let declaring = RuleCatalogue { + not_expressible: vec![NotExpressibleDeclaration { + construct: "fn:EXAMPLE", + reason: "an example declaration", + }], + }; + let test_cases = [ + CatalogueTestCase { + description: "the built-in catalogue has no rule for a node", + catalogue: builtin_catalogue(), + construct: Construct::new(ConstructFamily::Node, "select"), + expected_disposition: Disposition::Missing, + }, + CatalogueTestCase { + description: "the built-in catalogue has no rule for a function", + catalogue: builtin_catalogue(), + construct: Construct::new(ConstructFamily::Function, "CONCAT"), + expected_disposition: Disposition::Missing, + }, + CatalogueTestCase { + description: "a declared construct reports its reason", + catalogue: declaring.clone(), + construct: Construct::new(ConstructFamily::Function, "EXAMPLE"), + expected_disposition: Disposition::NotExpressible { + reason: "an example declaration".to_owned(), + }, + }, + CatalogueTestCase { + description: "an undeclared construct is still missing", + catalogue: declaring, + construct: Construct::new(ConstructFamily::Cast, "VARCHAR"), + expected_disposition: Disposition::Missing, + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.catalogue.disposition(&test_case.construct), + test_case.expected_disposition, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-rules/src/catalogue/tests/test_types.rs b/crates/sqlquack-rules/src/catalogue/tests/test_types.rs new file mode 100644 index 000000000..c2276bbb0 --- /dev/null +++ b/crates/sqlquack-rules/src/catalogue/tests/test_types.rs @@ -0,0 +1,11 @@ +use sqlquack_core::classification::models::Construct; +use sqlquack_core::refusals::models::Disposition; + +use crate::catalogue::models::RuleCatalogue; + +pub(crate) struct CatalogueTestCase { + pub(crate) description: &'static str, + pub(crate) catalogue: RuleCatalogue, + pub(crate) construct: Construct, + pub(crate) expected_disposition: Disposition, +} diff --git a/crates/sqlquack-rules/src/lib.rs b/crates/sqlquack-rules/src/lib.rs new file mode 100644 index 000000000..41cb57d7f --- /dev/null +++ b/crates/sqlquack-rules/src/lib.rs @@ -0,0 +1,5 @@ +//! The rule catalogue: verified rules and declared refusals; empty, so every construct is refused. + +#![forbid(unsafe_code)] + +pub mod catalogue; diff --git a/crates/sqlquack-snowflake/Cargo.toml b/crates/sqlquack-snowflake/Cargo.toml new file mode 100644 index 000000000..2104ceb5a --- /dev/null +++ b/crates/sqlquack-snowflake/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sqlquack-snowflake" +version.workspace = true +description = "The Snowflake typing layer: exact Snowflake result types for SQLQuack rules." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/crates/sqlquack-snowflake/src/lib.rs b/crates/sqlquack-snowflake/src/lib.rs new file mode 100644 index 000000000..0ecdec387 --- /dev/null +++ b/crates/sqlquack-snowflake/src/lib.rs @@ -0,0 +1,3 @@ +//! The Snowflake typing layer for exact result types; empty until the relational core lands. + +#![forbid(unsafe_code)] diff --git a/crates/sqlquack-structure-checker/Cargo.toml b/crates/sqlquack-structure-checker/Cargo.toml deleted file mode 100644 index 54d7c3b96..000000000 --- a/crates/sqlquack-structure-checker/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "sqlquack-structure-checker" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -readme.workspace = true -description = "Internal structure checker for SQLQuack development" -publish = false - -[dependencies] -proc-macro2 = { version = "1.0", features = ["span-locations"] } -syn = { version = "2.0", features = ["full", "visit"] } -walkdir = "2.5" - -[lib] -name = "sqlquack_structure_checker" diff --git a/crates/sqlquack-structure-checker/src/lib.rs b/crates/sqlquack-structure-checker/src/lib.rs deleted file mode 100644 index 36a920122..000000000 --- a/crates/sqlquack-structure-checker/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -#![forbid(unsafe_code)] - -pub mod models; -pub mod rules; diff --git a/crates/sqlquack-structure-checker/src/main.rs b/crates/sqlquack-structure-checker/src/main.rs deleted file mode 100644 index c0c94f2fe..000000000 --- a/crates/sqlquack-structure-checker/src/main.rs +++ /dev/null @@ -1,30 +0,0 @@ -fn main() { - let repo_root = std::env::current_dir().expect("current directory should be readable"); - let violations = sqlquack_structure_checker::rules::check_repo(&repo_root); - - for violation in &violations { - let path = violation - .path - .strip_prefix(&repo_root) - .unwrap_or(&violation.path); - match violation.line { - Some(line) => println!( - "{}:{}: {}: {}", - path.display(), - line, - violation.code, - violation.message - ), - None => println!( - "{}: {}: {}", - path.display(), - violation.code, - violation.message - ), - } - } - - if !violations.is_empty() { - std::process::exit(1); - } -} diff --git a/crates/sqlquack-structure-checker/src/models.rs b/crates/sqlquack-structure-checker/src/models.rs deleted file mode 100644 index add27f1d6..000000000 --- a/crates/sqlquack-structure-checker/src/models.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::path; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Violation { - pub code: &'static str, - pub path: path::PathBuf, - pub line: Option, - pub message: String, -} - -impl Violation { - pub(crate) fn new( - code: &'static str, - path: &path::Path, - line: Option, - message: impl Into, - ) -> Self { - Self { - code, - path: path.to_path_buf(), - line, - message: message.into(), - } - } -} diff --git a/crates/sqlquack-structure-checker/src/rules.rs b/crates/sqlquack-structure-checker/src/rules.rs deleted file mode 100644 index 59ca88ec1..000000000 --- a/crates/sqlquack-structure-checker/src/rules.rs +++ /dev/null @@ -1,1052 +0,0 @@ -use crate::models; -use std::fs; -use std::path; -use syn::visit; -use syn::visit::Visit; - -const MAX_LIB_RS_LINES: usize = 80; - -pub fn check_repo(repo_root: &path::Path) -> Vec { - let mut violations = Vec::new(); - let src_root = repo_root.join("crates/sqlquack/src"); - - if src_root.exists() { - for entry in walkdir::WalkDir::new(&src_root) { - let entry = match entry { - Ok(entry) => entry, - Err(_) => continue, - }; - let path = entry.path(); - if !path.is_file() || path.extension().and_then(|value| value.to_str()) != Some("rs") { - continue; - } - - let source = fs::read_to_string(path).unwrap_or_default(); - let syntax = syn::parse_file(&source).ok(); - - violations.extend(check_banned_names(repo_root, path)); - violations.extend(check_lib_rs_size(repo_root, path, &source)); - if let Some(file) = syntax.as_ref() { - violations.extend(check_mod_rs_shape(path, file)); - violations.extend(check_role_file_shape(path, file)); - violations.extend(check_main_entry_shape(repo_root, path, file)); - violations.extend(check_helper_file_shape(repo_root, path, file)); - violations.extend(check_reexports(repo_root, path, file)); - violations.extend(check_imports(repo_root, path, file)); - } - violations.extend(check_sqlbuild_terms(path, &source)); - } - } - - violations.extend(check_test_structure(repo_root)); - - violations.extend(check_crate_dependency_direction(repo_root)); - violations -} - -fn check_banned_names(repo_root: &path::Path, file_path: &path::Path) -> Vec { - let mut violations = Vec::new(); - let relative = relative_parts(repo_root, file_path); - - if matches!( - file_path.file_name().and_then(|value| value.to_str()), - Some("utils.rs" | "misc.rs" | "common.rs") - ) { - violations.push(models::Violation::new( - "SQS001", - file_path, - None, - "uses banned generic filename", - )); - } - for part in &relative { - if matches!(part.as_str(), "utils" | "misc" | "common") { - violations.push(models::Violation::new( - "SQS002", - file_path, - None, - format!("is under banned generic directory '{part}'"), - )); - } - } - if file_path.file_name().and_then(|value| value.to_str()) == Some("helpers.rs") { - violations.push(models::Violation::new( - "SQS003", - file_path, - None, - "use a scoped helpers/ folder instead of helpers.rs", - )); - } - - violations -} - -fn check_lib_rs_size( - repo_root: &path::Path, - file_path: &path::Path, - source: &str, -) -> Vec { - if relative_path(repo_root, file_path).as_deref() != Some("crates/sqlquack/src/lib.rs") { - return Vec::new(); - } - - let line_count = source.lines().count(); - if line_count <= MAX_LIB_RS_LINES { - return Vec::new(); - } - - vec![models::Violation::new( - "SQS004", - file_path, - None, - format!("lib.rs has {line_count} lines; keep it at or below {MAX_LIB_RS_LINES}"), - )] -} - -fn check_mod_rs_shape(file_path: &path::Path, file: &syn::File) -> Vec { - if file_path.file_name().and_then(|value| value.to_str()) != Some("mod.rs") { - return Vec::new(); - } - - file.items - .iter() - .filter(|item| !matches!(item, syn::Item::Mod(_) | syn::Item::Use(_))) - .map(|item| { - models::Violation::new( - "SQS012", - file_path, - item_line(item), - "mod.rs must contain declarations only, not implementation items", - ) - }) - .collect() -} - -fn check_role_file_shape(file_path: &path::Path, file: &syn::File) -> Vec { - match file_path.file_name().and_then(|value| value.to_str()) { - Some("models.rs") => check_models_rs_shape(file_path, file), - Some("types.rs") => check_types_rs_shape(file_path, file), - Some("constants.rs") => check_constants_rs_shape(file_path, file), - _ => Vec::new(), - } -} - -fn check_models_rs_shape(file_path: &path::Path, file: &syn::File) -> Vec { - let mut violations = Vec::new(); - for item in &file.items { - match item { - syn::Item::Fn(item_fn) => violations.push(models::Violation::new( - "SQS013", - file_path, - Some(item_fn.sig.ident.span().start().line), - "models.rs must not define free functions", - )), - syn::Item::Impl(item_impl) => { - for impl_item in &item_impl.items { - if let syn::ImplItem::Fn(method) = impl_item { - if matches!(method.vis, syn::Visibility::Public(_)) { - violations.push(models::Violation::new( - "SQS014", - file_path, - Some(method.sig.ident.span().start().line), - "models.rs must not expose public methods; use pub(crate) or narrower", - )); - } - } - } - } - _ => {} - } - } - violations -} - -fn check_types_rs_shape(file_path: &path::Path, file: &syn::File) -> Vec { - file.items - .iter() - .filter(|item| { - !matches!( - item, - syn::Item::Use(_) | syn::Item::Type(_) | syn::Item::Trait(_) - ) - }) - .map(|item| { - models::Violation::new( - "SQS015", - file_path, - item_line(item), - "types.rs must stay narrow: type aliases and traits only", - ) - }) - .collect() -} - -fn check_constants_rs_shape(file_path: &path::Path, file: &syn::File) -> Vec { - file.items - .iter() - .filter(|item| !matches!(item, syn::Item::Use(_) | syn::Item::Const(_))) - .map(|item| { - models::Violation::new( - "SQS016", - file_path, - item_line(item), - "constants.rs must contain constants only", - ) - }) - .collect() -} - -fn check_main_entry_shape( - repo_root: &path::Path, - file_path: &path::Path, - file: &syn::File, -) -> Vec { - let relative = relative_parts(repo_root, file_path); - if !relative.iter().any(|part| part == "main") - || file_path.file_name().and_then(|value| value.to_str()) == Some("mod.rs") - { - return Vec::new(); - } - - let mut visible_functions = 0; - let mut private_functions = 0; - let mut violations = Vec::new(); - - for item in &file.items { - match item { - syn::Item::Fn(item_fn) => { - if matches!(item_fn.vis, syn::Visibility::Inherited) { - private_functions += 1; - } else { - visible_functions += 1; - } - } - syn::Item::Use(_) => {} - _ => violations.push(models::Violation::new( - "SQS017", - file_path, - item_line(item), - "main/ entry files must contain orchestration functions only", - )), - } - } - - if visible_functions != 1 { - violations.push(models::Violation::new( - "SQS018", - file_path, - None, - "main/ entry files must define exactly one visible entry function", - )); - } - if private_functions > 2 { - violations.push(models::Violation::new( - "SQS019", - file_path, - None, - "main/ entry files must define at most two private helper functions", - )); - } - - violations -} - -fn check_helper_file_shape( - repo_root: &path::Path, - file_path: &path::Path, - file: &syn::File, -) -> Vec { - let relative = relative_parts(repo_root, file_path); - if !relative.iter().any(|part| part == "helpers") { - return Vec::new(); - } - - let mut violations = Vec::new(); - if file_path.file_name().and_then(|value| value.to_str()) == Some("main.rs") { - violations.push(models::Violation::new( - "SQS020", - file_path, - None, - "helpers/ must not contain main.rs", - )); - } - - for item in &file.items { - if is_public_item(item) { - violations.push(models::Violation::new( - "SQS021", - file_path, - item_line(item), - "helpers/ must not expose public API; use pub(super) or private items", - )); - } - } - - violations -} - -fn check_reexports( - repo_root: &path::Path, - file_path: &path::Path, - file: &syn::File, -) -> Vec { - let is_root_lib = - relative_path(repo_root, file_path).as_deref() == Some("crates/sqlquack/src/lib.rs"); - let mut violations = Vec::new(); - - for item in &file.items { - let syn::Item::Use(item_use) = item else { - continue; - }; - if contains_wildcard(&item_use.tree) { - violations.push(models::Violation::new( - "SQS007", - file_path, - Some(item_use.use_token.span.start().line), - "wildcard imports are not allowed", - )); - } - match item_use.vis { - syn::Visibility::Public(_) if !is_root_lib => violations.push(models::Violation::new( - "SQS022", - file_path, - Some(item_use.use_token.span.start().line), - "pub use is only allowed in lib.rs unless explicitly allowlisted", - )), - syn::Visibility::Restricted(_) => violations.push(models::Violation::new( - "SQS023", - file_path, - Some(item_use.use_token.span.start().line), - "pub(crate) use re-exports are banned; import role modules explicitly", - )), - _ => {} - } - } - - violations -} - -fn check_imports( - repo_root: &path::Path, - file_path: &path::Path, - file: &syn::File, -) -> Vec { - let Some(module_name) = top_sqlquack_module(repo_root, file_path) else { - return Vec::new(); - }; - - let mut violations = Vec::new(); - for item in &file.items { - let syn::Item::Use(item_use) = item else { - continue; - }; - for path_parts in use_tree_paths(&item_use.tree) { - violations.extend(check_forbidden_import_path( - file_path, - &module_name, - &path_parts, - item_use, - )); - violations.extend(check_cross_module_helper_import_path( - file_path, - &module_name, - &path_parts, - item_use, - )); - violations.extend(check_helper_model_import_path( - repo_root, - file_path, - &path_parts, - item_use, - )); - } - } - - violations -} - -fn check_forbidden_import_path( - file_path: &path::Path, - module_name: &str, - path_parts: &[String], - item_use: &syn::ItemUse, -) -> Vec { - if path_parts.first().map(String::as_str) != Some("crate") || path_parts.len() < 2 { - return Vec::new(); - } - let imported = path_parts[1].as_str(); - let forbidden = match module_name { - "ast" => &["parser", "analysis", "lineage", "translation"][..], - "lexer" => &["parser", "analysis", "lineage", "translation"][..], - "parser" => &["analysis", "lineage", "translation"][..], - "analysis" => &["translation"][..], - "lineage" => &["translation"][..], - "types" => &["translation"][..], - "translation" => &["parser"][..], - _ => &[], - }; - if forbidden.contains(&imported) { - return vec![models::Violation::new( - "SQS005", - file_path, - Some(item_use.use_token.span.start().line), - format!("{module_name}/ must not import downstream module crate::{imported}"), - )]; - } - Vec::new() -} - -fn check_cross_module_helper_import_path( - file_path: &path::Path, - module_name: &str, - path_parts: &[String], - item_use: &syn::ItemUse, -) -> Vec { - if path_parts.first().map(String::as_str) != Some("crate") || path_parts.len() < 3 { - return Vec::new(); - } - let imported_module = &path_parts[1]; - if path_parts[2] == "helpers" && imported_module != module_name { - return vec![models::Violation::new( - "SQS011", - file_path, - Some(item_use.use_token.span.start().line), - format!( - "{module_name}/ must not import internal helpers from crate::{imported_module}; use an approved parent/main/public surface" - ), - )]; - } - Vec::new() -} - -fn check_helper_model_import_path( - repo_root: &path::Path, - file_path: &path::Path, - path_parts: &[String], - item_use: &syn::ItemUse, -) -> Vec { - if !relative_parts(repo_root, file_path) - .iter() - .any(|part| part == "helpers") - { - return Vec::new(); - } - if path_parts.first().map(String::as_str) != Some("super") - || path_parts.get(1).map(String::as_str) != Some("models") - { - return Vec::new(); - } - if path_parts.len() > 2 || contains_wildcard(&item_use.tree) { - return vec![models::Violation::new( - "SQS009", - file_path, - Some(item_use.use_token.span.start().line), - "helpers should use `use super::models;` and qualified model names", - )]; - } - Vec::new() -} - -fn check_sqlbuild_terms(file_path: &path::Path, source: &str) -> Vec { - let terms = [ - "SqlReferenceKind", - "MODEL(", - "TEST(", - "SCENARIO(", - "AUDIT(", - "SQLBuild", - "sqlbuild", - ]; - let mut violations = Vec::new(); - for (index, line) in source.lines().enumerate() { - for term in terms { - if line.contains(term) { - violations.push(models::Violation::new( - "SQS006", - file_path, - Some(index + 1), - format!("SQLBuild-specific term '{term}' must not appear in core"), - )); - } - } - } - violations -} - -fn check_crate_dependency_direction(repo_root: &path::Path) -> Vec { - let manifest = repo_root.join("crates/sqlquack/Cargo.toml"); - let Ok(text) = fs::read_to_string(&manifest) else { - return Vec::new(); - }; - if text.contains("sqlquack-python") || text.contains("sqlquack_python") { - return vec![models::Violation::new( - "SQS010", - &manifest, - None, - "sqlquack core must not depend on sqlquack-python", - )]; - } - Vec::new() -} - -fn check_test_structure(repo_root: &path::Path) -> Vec { - let mut violations = Vec::new(); - let crates_root = repo_root.join("crates"); - if !crates_root.exists() { - return violations; - } - - for crate_entry in walkdir::WalkDir::new(&crates_root).max_depth(1) { - let Ok(crate_entry) = crate_entry else { - continue; - }; - if !crate_entry.path().is_dir() || crate_entry.path() == crates_root { - continue; - } - - let tests_root = crate_entry.path().join("tests"); - if !tests_root.exists() { - continue; - } - - violations.extend(check_crate_tests_root(&tests_root)); - } - - violations -} - -fn check_crate_tests_root(tests_root: &path::Path) -> Vec { - let mut violations = Vec::new(); - let mut harness_stems = Vec::new(); - - let Ok(entries) = fs::read_dir(tests_root) else { - return violations; - }; - - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() && path.extension().and_then(|value| value.to_str()) == Some("rs") { - let Some(stem) = path.file_stem().and_then(|value| value.to_str()) else { - continue; - }; - harness_stems.push(stem.to_owned()); - violations.extend(check_test_harness_file(&path, stem)); - } else if path.is_dir() { - violations.extend(check_test_area_folder(&path)); - } - } - - for stem in harness_stems { - let area_dir = tests_root.join(&stem); - if !area_dir.is_dir() { - violations.push(models::Violation::new( - "SQT001", - &tests_root.join(format!("{stem}.rs")), - None, - "test harness files must have a sibling area folder with the same stem", - )); - continue; - } - for required_file in ["helpers.rs", "test_types.rs"] { - let required_path = area_dir.join(required_file); - if !required_path.is_file() { - violations.push(models::Violation::new( - "SQT002", - &area_dir, - None, - format!("test area folder must contain {required_file}"), - )); - } - } - violations.extend(check_harness_declares_area_files( - &tests_root.join(format!("{stem}.rs")), - &area_dir, - )); - } - - violations -} - -fn check_harness_declares_area_files( - harness_path: &path::Path, - area_dir: &path::Path, -) -> Vec { - let harness_source = fs::read_to_string(harness_path).unwrap_or_default(); - let Ok(harness_file) = syn::parse_file(&harness_source) else { - return Vec::new(); - }; - let area_name = area_dir - .file_name() - .and_then(|value| value.to_str()) - .unwrap_or_default(); - - let mut declared_files = Vec::new(); - let mut violations = Vec::new(); - for item in &harness_file.items { - let syn::Item::Mod(item_mod) = item else { - continue; - }; - let module_name = item_mod.ident.to_string(); - let declared_file = format!("{module_name}.rs"); - declared_files.push(declared_file.clone()); - - let expected_path_fragment = format!("{area_name}/{declared_file}"); - if !harness_source.contains(&expected_path_fragment) { - continue; - } - if !area_dir.join(&declared_file).is_file() { - violations.push(models::Violation::new( - "SQT019", - harness_path, - Some(item_mod.ident.span().start().line), - "test harness declares a module whose area file does not exist", - )); - } - } - - let Ok(entries) = fs::read_dir(area_dir) else { - return violations; - }; - for entry in entries.flatten() { - let path = entry.path(); - if !path.is_file() || path.extension().and_then(|value| value.to_str()) != Some("rs") { - continue; - } - let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else { - continue; - }; - if matches!(file_name, "helpers.rs" | "test_types.rs") { - continue; - } - if !declared_files.iter().any(|declared| declared == file_name) { - violations.push(models::Violation::new( - "SQT020", - &path, - None, - "test area files must be declared by their top-level harness", - )); - } - } - - violations -} - -fn check_test_area_folder(folder_path: &path::Path) -> Vec { - let mut violations = Vec::new(); - - let Ok(entries) = fs::read_dir(folder_path) else { - return violations; - }; - - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - violations.push(models::Violation::new( - "SQT003", - &path, - None, - "test area folders must stay flat", - )); - continue; - } - let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else { - continue; - }; - if path.extension().and_then(|value| value.to_str()) != Some("rs") { - violations.push(models::Violation::new( - "SQT004", - &path, - None, - "test area folders may contain only Rust files", - )); - continue; - } - - let source = fs::read_to_string(&path).unwrap_or_default(); - let syntax = syn::parse_file(&source).ok(); - if let Some(file) = syntax.as_ref() { - match file_name { - "helpers.rs" => violations.extend(check_test_helpers_file(&path, file)), - "test_types.rs" => violations.extend(check_test_types_file(&path, file)), - _ => violations.extend(check_primary_test_module_file(&path, file)), - } - } - } - - violations -} - -fn check_test_harness_file(file_path: &path::Path, stem: &str) -> Vec { - let source = fs::read_to_string(file_path).unwrap_or_default(); - let Ok(file) = syn::parse_file(&source) else { - return Vec::new(); - }; - - let mut violations = Vec::new(); - - for item in &file.items { - match item { - syn::Item::Mod(item_mod) => { - if item_mod.content.is_some() { - violations.push(models::Violation::new( - "SQT005", - file_path, - Some(item_mod.ident.span().start().line), - "test harness module declarations must point at files, not inline modules", - )); - } - let module_name = item_mod.ident.to_string(); - let expected_path_fragment = format!("{stem}/{module_name}.rs"); - if !source.contains(&expected_path_fragment) { - violations.push(models::Violation::new( - "SQT009", - file_path, - Some(item_mod.ident.span().start().line), - "test harness modules must use #[path = \"/.rs\"]", - )); - } - } - _ => violations.push(models::Violation::new( - "SQT008", - file_path, - item_line(item), - "test harness files may contain only path-based module declarations", - )), - } - } - - violations -} - -fn check_primary_test_module_file( - file_path: &path::Path, - file: &syn::File, -) -> Vec { - let mut violations = Vec::new(); - - for item in &file.items { - match item { - syn::Item::Use(item_use) => { - if contains_wildcard(&item_use.tree) { - violations.push(models::Violation::new( - "SQT006", - file_path, - Some(item_use.use_token.span.start().line), - "test files must not use wildcard imports", - )); - } - } - syn::Item::Const(_) => {} - syn::Item::Fn(item_fn) => { - if !is_test_fn(item_fn) { - violations.push(models::Violation::new( - "SQT007", - file_path, - Some(item_fn.sig.ident.span().start().line), - "primary test module files must not define helper functions", - )); - continue; - } - violations.extend(check_test_function_shape(file_path, item_fn)); - } - _ => violations.push(models::Violation::new( - "SQT008", - file_path, - item_line(item), - "primary test module files may contain only imports, constants, and test functions", - )), - } - } - - violations -} - -fn check_test_function_shape( - file_path: &path::Path, - item_fn: &syn::ItemFn, -) -> Vec { - let mut violations = Vec::new(); - let name = item_fn.sig.ident.to_string(); - if !(name.starts_with("given_") && name.contains("_when_") && name.contains("_then_")) { - violations.push(models::Violation::new( - "SQT010", - file_path, - Some(item_fn.sig.ident.span().start().line), - "test names must follow given__when__then_", - )); - } - - let mut visitor = TestFunctionVisitor::default(); - visitor.visit_block(&item_fn.block); - - if visitor.has_if_or_match { - violations.push(models::Violation::new( - "SQT011", - file_path, - Some(item_fn.sig.ident.span().start().line), - "test functions must not contain if or match; split scenarios into cases or helpers", - )); - } - if visitor.has_unwrap { - violations.push(models::Violation::new( - "SQT012", - file_path, - Some(item_fn.sig.ident.span().start().line), - "test functions must use expect with context instead of unwrap", - )); - } - if visitor.assert_count == 0 { - violations.push(models::Violation::new( - "SQT013", - file_path, - Some(item_fn.sig.ident.span().start().line), - "test functions must contain assertion macros", - )); - } - if visitor.assert_without_expected { - violations.push(models::Violation::new( - "SQT014", - file_path, - Some(item_fn.sig.ident.span().start().line), - "every assertion macro must reference test_case.expected_...", - )); - } - - violations -} - -fn check_test_helpers_file(file_path: &path::Path, file: &syn::File) -> Vec { - let mut violations = Vec::new(); - for item in &file.items { - if let syn::Item::Fn(item_fn) = item { - if is_test_fn(item_fn) { - violations.push(models::Violation::new( - "SQT015", - file_path, - Some(item_fn.sig.ident.span().start().line), - "helpers.rs must not define test functions", - )); - } - } - if let syn::Item::Use(item_use) = item { - if contains_wildcard(&item_use.tree) { - violations.push(models::Violation::new( - "SQT006", - file_path, - Some(item_use.use_token.span.start().line), - "test files must not use wildcard imports", - )); - } - } - } - violations -} - -fn check_test_types_file(file_path: &path::Path, file: &syn::File) -> Vec { - let mut violations = Vec::new(); - for item in &file.items { - match item { - syn::Item::Use(_) | syn::Item::Type(_) => {} - syn::Item::Struct(item_struct) => { - if !item_struct.ident.to_string().ends_with("TestCase") { - continue; - } - let field_names = named_field_names(item_struct); - if !field_names - .iter() - .any(|field_name| field_name == "description") - { - violations.push(models::Violation::new( - "SQT016", - file_path, - Some(item_struct.ident.span().start().line), - "test case structs in test_types.rs must define a description field", - )); - } - if !field_names - .iter() - .any(|field_name| field_name.starts_with("expected_")) - { - violations.push(models::Violation::new( - "SQT017", - file_path, - Some(item_struct.ident.span().start().line), - "test case structs in test_types.rs must define at least one expected_ field", - )); - } - } - syn::Item::Impl(_) => {} - _ => violations.push(models::Violation::new( - "SQT018", - file_path, - item_line(item), - "test_types.rs may contain only imports, type aliases, structs, and impls", - )), - } - } - violations -} - -fn use_tree_paths(tree: &syn::UseTree) -> Vec> { - fn walk(tree: &syn::UseTree, mut prefix: Vec, paths: &mut Vec>) { - match tree { - syn::UseTree::Path(path) => { - prefix.push(path.ident.to_string()); - walk(&path.tree, prefix, paths); - } - syn::UseTree::Name(name) => { - prefix.push(name.ident.to_string()); - paths.push(prefix); - } - syn::UseTree::Rename(rename) => { - prefix.push(rename.ident.to_string()); - paths.push(prefix); - } - syn::UseTree::Glob(_) => { - prefix.push("*".to_owned()); - paths.push(prefix); - } - syn::UseTree::Group(group) => { - for item in &group.items { - walk(item, prefix.clone(), paths); - } - } - } - } - - let mut paths = Vec::new(); - walk(tree, Vec::new(), &mut paths); - paths -} - -fn contains_wildcard(tree: &syn::UseTree) -> bool { - match tree { - syn::UseTree::Glob(_) => true, - syn::UseTree::Path(path) => contains_wildcard(&path.tree), - syn::UseTree::Group(group) => group.items.iter().any(contains_wildcard), - syn::UseTree::Name(_) | syn::UseTree::Rename(_) => false, - } -} - -#[derive(Default)] -struct TestFunctionVisitor { - assert_count: usize, - assert_without_expected: bool, - has_if_or_match: bool, - has_unwrap: bool, -} - -impl<'ast> visit::Visit<'ast> for TestFunctionVisitor { - fn visit_expr_if(&mut self, node: &'ast syn::ExprIf) { - self.has_if_or_match = true; - visit::visit_expr_if(self, node); - } - - fn visit_expr_match(&mut self, node: &'ast syn::ExprMatch) { - self.has_if_or_match = true; - visit::visit_expr_match(self, node); - } - - fn visit_expr_method_call(&mut self, node: &'ast syn::ExprMethodCall) { - if node.method == "unwrap" { - self.has_unwrap = true; - } - visit::visit_expr_method_call(self, node); - } - - fn visit_macro(&mut self, node: &'ast syn::Macro) { - let macro_name = node - .path - .segments - .last() - .map(|segment| segment.ident.to_string()); - if matches!( - macro_name.as_deref(), - Some("assert" | "assert_eq" | "assert_ne") - ) { - self.assert_count += 1; - let tokens = node.tokens.to_string(); - if !(tokens.contains("test_case . expected_") || tokens.contains("test_case.expected_")) - { - self.assert_without_expected = true; - } - } - visit::visit_macro(self, node); - } -} - -fn is_test_fn(item_fn: &syn::ItemFn) -> bool { - item_fn - .attrs - .iter() - .any(|attr| attr.path().is_ident("test")) -} - -fn named_field_names(item_struct: &syn::ItemStruct) -> Vec { - match &item_struct.fields { - syn::Fields::Named(fields) => fields - .named - .iter() - .filter_map(|field| field.ident.as_ref().map(ToString::to_string)) - .collect(), - syn::Fields::Unnamed(_) | syn::Fields::Unit => Vec::new(), - } -} - -fn is_public_item(item: &syn::Item) -> bool { - match item { - syn::Item::Const(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Enum(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Fn(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Mod(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Struct(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Trait(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Type(item) => matches!(item.vis, syn::Visibility::Public(_)), - syn::Item::Use(item) => matches!(item.vis, syn::Visibility::Public(_)), - _ => false, - } -} - -fn item_line(item: &syn::Item) -> Option { - Some(match item { - syn::Item::Const(item) => item.const_token.span.start().line, - syn::Item::Enum(item) => item.enum_token.span.start().line, - syn::Item::Fn(item) => item.sig.ident.span().start().line, - syn::Item::Impl(item) => item.impl_token.span.start().line, - syn::Item::Mod(item) => item.mod_token.span.start().line, - syn::Item::Static(item) => item.static_token.span.start().line, - syn::Item::Struct(item) => item.struct_token.span.start().line, - syn::Item::Trait(item) => item.trait_token.span.start().line, - syn::Item::Type(item) => item.type_token.span.start().line, - syn::Item::Use(item) => item.use_token.span.start().line, - _ => return None, - }) -} - -fn top_sqlquack_module(repo_root: &path::Path, file_path: &path::Path) -> Option { - let parts = relative_parts(repo_root, file_path); - if parts.len() < 4 || parts[..3] != ["crates", "sqlquack", "src"] { - return None; - } - if parts[3] == "lib.rs" { - return None; - } - Some(parts[3].strip_suffix(".rs").unwrap_or(&parts[3]).to_owned()) -} - -fn relative_parts(repo_root: &path::Path, file_path: &path::Path) -> Vec { - file_path - .strip_prefix(repo_root) - .unwrap_or(file_path) - .components() - .filter_map(|component| component.as_os_str().to_str().map(ToOwned::to_owned)) - .collect() -} - -fn relative_path(repo_root: &path::Path, file_path: &path::Path) -> Option { - file_path - .strip_prefix(repo_root) - .ok() - .and_then(|path| path.to_str()) - .map(|path| path.replace('\\', "/")) -} diff --git a/crates/sqlquack-structure-checker/tests/rules.rs b/crates/sqlquack-structure-checker/tests/rules.rs deleted file mode 100644 index 380b23b01..000000000 --- a/crates/sqlquack-structure-checker/tests/rules.rs +++ /dev/null @@ -1,10 +0,0 @@ -#[path = "rules/helpers.rs"] -mod helpers; -#[path = "rules/imports.rs"] -mod imports; -#[path = "rules/role_files.rs"] -mod role_files; -#[path = "rules/test_structure.rs"] -mod test_structure; -#[path = "rules/test_types.rs"] -mod test_types; diff --git a/crates/sqlquack-structure-checker/tests/rules/helpers.rs b/crates/sqlquack-structure-checker/tests/rules/helpers.rs deleted file mode 100644 index 8cab30c62..000000000 --- a/crates/sqlquack-structure-checker/tests/rules/helpers.rs +++ /dev/null @@ -1,133 +0,0 @@ -use crate::test_types; -use sqlquack_structure_checker::rules; -use std::fs; -use std::io::Write; -use std::path; - -pub(crate) fn make_temp_repo(test_case: &test_types::CheckRepoTestCase) -> path::PathBuf { - let repo_root = std::env::temp_dir().join(format!( - "sqlquack-structure-rules-test-{}-{}", - std::process::id(), - unique_suffix() - )); - - write_repo_files(&repo_root, compliant_repo_files().as_slice()); - write_repo_files(&repo_root, test_case.repo_files); - - repo_root -} - -pub(crate) fn collect_source_violation_codes(repo_root: &path::Path) -> Vec<&'static str> { - rules::check_repo(repo_root) - .iter() - .filter(|violation| violation.code.starts_with("SQS")) - .map(|violation| violation.code) - .collect() -} - -pub(crate) fn collect_test_violation_codes(repo_root: &path::Path) -> Vec<&'static str> { - rules::check_repo(repo_root) - .iter() - .filter(|violation| violation.code.starts_with("SQT")) - .map(|violation| violation.code) - .collect() -} - -fn compliant_repo_files() -> Vec { - vec![ - test_types::RepoFile { - path: "Cargo.toml", - contents: "[workspace]\nmembers = [\"crates/sqlquack\", \"crates/example\"]\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/Cargo.toml", - contents: "[package]\nname = \"sqlquack\"\nversion = \"0.1.0\"\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/lib.rs", - contents: "mod analysis;\nmod ast;\npub use analysis::models::AnalysisResult;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/mod.rs", - contents: "pub(crate) mod models;\nmod helpers;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/models.rs", - contents: "pub struct AnalysisResult { pub diagnostics_count: usize }\nimpl AnalysisResult { pub(crate) fn is_empty(&self) -> bool { self.diagnostics_count == 0 } }\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/types.rs", - contents: "pub type AnalysisId = u32;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/constants.rs", - contents: "pub(crate) const MAX_SCOPE_DEPTH: usize = 1024;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/main/mod.rs", - contents: "pub mod analyze_query;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/main/analyze_query.rs", - contents: "use crate::analysis::models;\npub(crate) fn analyze_query() -> models::AnalysisResult { models::AnalysisResult { diagnostics_count: 0 } }\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/helpers/mod.rs", - contents: "pub(super) mod scope_builder;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/analysis/helpers/scope_builder.rs", - contents: "use crate::analysis::models;\nstruct ScopeFrame { depth: usize }\npub(super) fn build_scope() -> models::AnalysisResult { let _frame = ScopeFrame { depth: 0 }; models::AnalysisResult { diagnostics_count: 0 } }\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/ast/mod.rs", - contents: "pub(crate) mod models;\n", - }, - test_types::RepoFile { - path: "crates/sqlquack/src/ast/models.rs", - contents: "pub struct Query;\n", - }, - test_types::RepoFile { - path: "crates/example/Cargo.toml", - contents: "[package]\nname = \"example\"\nversion = \"0.1.0\"\n", - }, - test_types::RepoFile { - path: "crates/example/tests/rules.rs", - contents: "#[path = \"rules/helpers.rs\"]\nmod helpers;\n#[path = \"rules/role_files.rs\"]\nmod role_files;\n#[path = \"rules/test_types.rs\"]\nmod test_types;\n", - }, - test_types::RepoFile { - path: "crates/example/tests/rules/role_files.rs", - contents: "use crate::helpers;\nuse crate::test_types;\nconst TEST_CASES: &[test_types::CheckRepoTestCase] = &[];\n#[test]\nfn given_input_when_running_then_reports_output() {\n for test_case in TEST_CASES {\n assert_eq!(1, test_case.expected_count, \"{}\", test_case.description);\n }\n}\n", - }, - test_types::RepoFile { - path: "crates/example/tests/rules/helpers.rs", - contents: "use crate::test_types;\npub(crate) fn build_count(test_case: &test_types::CheckRepoTestCase) -> usize { test_case.expected_count }\n", - }, - test_types::RepoFile { - path: "crates/example/tests/rules/test_types.rs", - contents: "pub(crate) struct CheckRepoTestCase {\n pub(crate) description: &'static str,\n pub(crate) expected_count: usize,\n}\n", - }, - ] -} - -fn write_repo_files(repo_root: &path::Path, files: &[test_types::RepoFile]) { - for repo_file in files { - let file_path = repo_root.join(repo_file.path); - fs::create_dir_all( - file_path - .parent() - .expect("test repo file path should have a parent"), - ) - .expect("test repo file parent should be created"); - let mut file = fs::File::create(&file_path).expect("test repo file should be created"); - file.write_all(repo_file.contents.as_bytes()) - .expect("test repo file should be written"); - } -} - -fn unique_suffix() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system clock should be after epoch") - .as_nanos() -} diff --git a/crates/sqlquack-structure-checker/tests/rules/imports.rs b/crates/sqlquack-structure-checker/tests/rules/imports.rs deleted file mode 100644 index 989c8f3e8..000000000 --- a/crates/sqlquack-structure-checker/tests/rules/imports.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::helpers; -use crate::test_types; - -const TEST_CASES: &[test_types::CheckRepoTestCase] = &[ - test_types::CheckRepoTestCase { - description: "reports pub use outside lib rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/mod.rs", - contents: "pub use models::AnalysisResult;\n", - }], - expected_violation_codes: &["SQS022"], - }, - test_types::CheckRepoTestCase { - description: "reports pub crate use reexport", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/models.rs", - contents: - "pub(crate) use crate::ast::models as ast_models;\npub struct AnalysisResult;\n", - }], - expected_violation_codes: &["SQS023"], - }, - test_types::CheckRepoTestCase { - description: "reports cross module helper import", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/ast/models.rs", - contents: "use crate::analysis::helpers;\npub struct Query;\n", - }], - expected_violation_codes: &["SQS005", "SQS011"], - }, -]; - -#[test] -fn given_import_rules_when_checking_structure_then_reports_expected_violation_codes() { - for test_case in TEST_CASES { - let repo_root = helpers::make_temp_repo(test_case); - let actual_codes = helpers::collect_source_violation_codes(&repo_root); - - assert_eq!( - actual_codes.as_slice(), - test_case.expected_violation_codes, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlquack-structure-checker/tests/rules/role_files.rs b/crates/sqlquack-structure-checker/tests/rules/role_files.rs deleted file mode 100644 index e74368164..000000000 --- a/crates/sqlquack-structure-checker/tests/rules/role_files.rs +++ /dev/null @@ -1,89 +0,0 @@ -use crate::helpers; -use crate::test_types; - -const TEST_CASES: &[test_types::CheckRepoTestCase] = &[ - test_types::CheckRepoTestCase { - description: "reports no violations for compliant source role files", - repo_files: &[], - expected_violation_codes: &[], - }, - test_types::CheckRepoTestCase { - description: "reports implementation in mod rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/mod.rs", - contents: "pub(crate) fn analyze() {}\n", - }], - expected_violation_codes: &["SQS012"], - }, - test_types::CheckRepoTestCase { - description: "reports free function in models rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/models.rs", - contents: "fn build_result() {}\n", - }], - expected_violation_codes: &["SQS013"], - }, - test_types::CheckRepoTestCase { - description: "reports public method in models rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/models.rs", - contents: "pub struct AnalysisResult;\nimpl AnalysisResult { pub fn new() -> Self { Self } }\n", - }], - expected_violation_codes: &["SQS014"], - }, - test_types::CheckRepoTestCase { - description: "reports enum in types rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/types.rs", - contents: "pub enum AnalysisKind { Basic }\n", - }], - expected_violation_codes: &["SQS015"], - }, - test_types::CheckRepoTestCase { - description: "reports function in constants rs", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/constants.rs", - contents: "fn default_depth() -> usize { 0 }\n", - }], - expected_violation_codes: &["SQS016"], - }, - test_types::CheckRepoTestCase { - description: "reports missing visible main entry", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/main/analyze_query.rs", - contents: "fn helper() {}\n", - }], - expected_violation_codes: &["SQS018"], - }, - test_types::CheckRepoTestCase { - description: "reports too many private main helpers", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/main/analyze_query.rs", - contents: "pub(crate) fn analyze_query() {}\nfn one() {}\nfn two() {}\nfn three() {}\n", - }], - expected_violation_codes: &["SQS019"], - }, - test_types::CheckRepoTestCase { - description: "reports public helper api", - repo_files: &[test_types::RepoFile { - path: "crates/sqlquack/src/analysis/helpers/scope_builder.rs", - contents: "pub fn build_scope() {}\n", - }], - expected_violation_codes: &["SQS021"], - }, -]; - -#[test] -fn given_source_role_files_when_checking_structure_then_reports_expected_violation_codes() { - for test_case in TEST_CASES { - let repo_root = helpers::make_temp_repo(test_case); - let actual_codes = helpers::collect_source_violation_codes(&repo_root); - - assert_eq!( - actual_codes.as_slice(), - test_case.expected_violation_codes, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlquack-structure-checker/tests/rules/test_structure.rs b/crates/sqlquack-structure-checker/tests/rules/test_structure.rs deleted file mode 100644 index de5665033..000000000 --- a/crates/sqlquack-structure-checker/tests/rules/test_structure.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::helpers; -use crate::test_types; - -const TEST_CASES: &[test_types::CheckRepoTestCase] = &[ - test_types::CheckRepoTestCase { - description: "reports no violations for compliant test layout", - repo_files: &[], - expected_violation_codes: &[], - }, - test_types::CheckRepoTestCase { - description: "reports missing expected field assertion", - repo_files: &[test_types::RepoFile { - path: "crates/example/tests/rules/role_files.rs", - contents: "use crate::helpers;\nuse crate::test_types;\nconst TEST_CASES: &[test_types::CheckRepoTestCase] = &[];\n#[test]\nfn given_input_when_running_then_reports_output() {\n for test_case in TEST_CASES {\n assert_eq!(1, 1, \"{}\", test_case.description);\n }\n}\n", - }], - expected_violation_codes: &["SQT014"], - }, - test_types::CheckRepoTestCase { - description: "reports if in test body", - repo_files: &[test_types::RepoFile { - path: "crates/example/tests/rules/role_files.rs", - contents: "use crate::helpers;\nuse crate::test_types;\nconst TEST_CASES: &[test_types::CheckRepoTestCase] = &[];\n#[test]\nfn given_input_when_running_then_reports_output() {\n for test_case in TEST_CASES {\n if true {\n assert_eq!(1, test_case.expected_count, \"{}\", test_case.description);\n }\n }\n}\n", - }], - expected_violation_codes: &["SQT011"], - }, - test_types::CheckRepoTestCase { - description: "reports test case without expected field", - repo_files: &[test_types::RepoFile { - path: "crates/example/tests/rules/test_types.rs", - contents: "pub(crate) struct CheckRepoTestCase {\n pub(crate) description: &'static str,\n}\n", - }], - expected_violation_codes: &["SQT017"], - }, - test_types::CheckRepoTestCase { - description: "reports undeclared area test file", - repo_files: &[test_types::RepoFile { - path: "crates/example/tests/rules/forgotten.rs", - contents: "use crate::test_types;\nconst TEST_CASES: &[test_types::CheckRepoTestCase] = &[];\n#[test]\nfn given_input_when_running_then_reports_output() {\n for test_case in TEST_CASES {\n assert_eq!(1, test_case.expected_count, \"{}\", test_case.description);\n }\n}\n", - }], - expected_violation_codes: &["SQT020"], - }, - test_types::CheckRepoTestCase { - description: "reports harness module pointing to missing area file", - repo_files: &[test_types::RepoFile { - path: "crates/example/tests/rules.rs", - contents: "#[path = \"rules/helpers.rs\"]\nmod helpers;\n#[path = \"rules/missing.rs\"]\nmod missing;\n#[path = \"rules/role_files.rs\"]\nmod role_files;\n#[path = \"rules/test_types.rs\"]\nmod test_types;\n", - }], - expected_violation_codes: &["SQT019"], - }, -]; - -#[test] -fn given_test_layout_rules_when_checking_structure_then_reports_expected_violation_codes() { - for test_case in TEST_CASES { - let repo_root = helpers::make_temp_repo(test_case); - let actual_codes = helpers::collect_test_violation_codes(&repo_root); - - assert_eq!( - actual_codes.as_slice(), - test_case.expected_violation_codes, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlquack-structure-checker/tests/rules/test_types.rs b/crates/sqlquack-structure-checker/tests/rules/test_types.rs deleted file mode 100644 index b88c28d03..000000000 --- a/crates/sqlquack-structure-checker/tests/rules/test_types.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub(crate) struct RepoFile { - pub(crate) path: &'static str, - pub(crate) contents: &'static str, -} - -pub(crate) struct CheckRepoTestCase { - pub(crate) description: &'static str, - pub(crate) repo_files: &'static [RepoFile], - pub(crate) expected_violation_codes: &'static [&'static str], -} diff --git a/crates/sqlquack/Cargo.toml b/crates/sqlquack/Cargo.toml index 7c1eea6fd..47af6f4f2 100644 --- a/crates/sqlquack/Cargo.toml +++ b/crates/sqlquack/Cargo.toml @@ -1,12 +1,35 @@ [package] name = "sqlquack" -version = "0.1.0" +version.workspace = true +description = "Snowflake-to-DuckDB SQL conversion that refuses to guess." edition.workspace = true license.workspace = true repository.workspace = true -readme.workspace = true -description = "Warehouse SQL analysis and fail-closed DuckDB transpilation engine" -keywords = ["sql", "duckdb", "parser", "analysis", "lineage"] -categories = ["parser-implementations", "database"] +homepage.workspace = true +readme = "../../README.md" +keywords = ["sql", "duckdb", "snowflake", "transpiler"] +categories = ["database", "command-line-utilities"] +publish.workspace = true +rust-version.workspace = true + +[[bin]] +name = "sqlquack" +path = "src/bin/sqlquack.rs" [dependencies] +clap.workspace = true +rayon.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlquack-compare.workspace = true +sqlquack-core.workspace = true +sqlquack-duckdb.workspace = true +sqlquack-rules.workspace = true +sqlquack-snowflake.workspace = true +walkdir.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack/src/analysis/main/analyze.rs b/crates/sqlquack/src/analysis/main/analyze.rs new file mode 100644 index 000000000..b51d3517b --- /dev/null +++ b/crates/sqlquack/src/analysis/main/analyze.rs @@ -0,0 +1,46 @@ +//! `analyze`: parse, check fidelity, classify and refuse. + +use sqlquack_core::classification::main::classify::classify; +use sqlquack_core::parsing::main::parse_source::parse_source; +use sqlquack_core::parsing::models::SourceDialect; +use sqlquack_core::refusals::main::refuse_constructs::refuse_constructs; +use sqlquack_core::refusals::main::refuse_infidelity::refuse_infidelity; +use sqlquack_core::refusals::main::refuse_unparsed::refuse_unparsed; +use sqlquack_core::refusals::types::ConstructPolicy; +use sqlquack_core::reparse::main::check_source_fidelity::check_source_fidelity; + +use crate::analysis::models::Analysis; + +/// Parses, checks print-back fidelity, classifies, and refuses every construct no rule converts. +pub fn analyze(sql: &str, dialect: SourceDialect, policy: &dyn ConstructPolicy) -> Analysis { + let statements = match parse_source(sql, dialect) { + Ok(statements) => statements, + Err(failure) => { + return Analysis { + parsed: false, + statements: 0, + occurrences: Vec::new(), + fidelity: Vec::new(), + refusals: vec![refuse_unparsed(&failure)], + }; + } + }; + let mut analysis = Analysis { + parsed: true, + statements: statements.len(), + occurrences: Vec::new(), + fidelity: Vec::with_capacity(statements.len()), + refusals: Vec::new(), + }; + for statement in &statements { + let outcome = check_source_fidelity(statement, dialect); + analysis.refusals.extend(refuse_infidelity(&outcome)); + analysis.fidelity.push(outcome); + let occurrences = classify(statement, sql, dialect); + analysis + .refusals + .extend(refuse_constructs(&occurrences, policy)); + analysis.occurrences.extend(occurrences); + } + analysis +} diff --git a/crates/sqlquack/src/api/main/mod.rs b/crates/sqlquack/src/analysis/main/mod.rs similarity index 100% rename from crates/sqlquack/src/api/main/mod.rs rename to crates/sqlquack/src/analysis/main/mod.rs diff --git a/crates/sqlquack/src/analysis/mod.rs b/crates/sqlquack/src/analysis/mod.rs index 8b1378917..fb51ea02a 100644 --- a/crates/sqlquack/src/analysis/mod.rs +++ b/crates/sqlquack/src/analysis/mod.rs @@ -1 +1,7 @@ +//! The conversion pipeline. With no rules yet, it refuses every construct. +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack/src/analysis/models.rs b/crates/sqlquack/src/analysis/models.rs new file mode 100644 index 000000000..8096f76f7 --- /dev/null +++ b/crates/sqlquack/src/analysis/models.rs @@ -0,0 +1,21 @@ +//! Pipeline results. + +use serde::Serialize; +use sqlquack_core::classification::models::Occurrence; +use sqlquack_core::refusals::models::Refusal; +use sqlquack_core::reparse::models::FidelityOutcome; + +/// What the pipeline found in one SQL input. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct Analysis { + /// Whether the input parsed. + pub parsed: bool, + /// Statements parsed. + pub statements: usize, + /// Every construct used, in source order. + pub occurrences: Vec, + /// The source print-back check for each statement. + pub fidelity: Vec, + /// Every refusal; the input converts only when this is empty. + pub refusals: Vec, +} diff --git a/crates/sqlquack/src/analysis/tests.rs b/crates/sqlquack/src/analysis/tests.rs new file mode 100644 index 000000000..9e3add8d3 --- /dev/null +++ b/crates/sqlquack/src/analysis/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_analyze.rs"] +mod test_analyze; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs new file mode 100644 index 000000000..4ca7421eb --- /dev/null +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -0,0 +1,33 @@ +use std::collections::BTreeSet; + +use sqlquack_core::parsing::models::SourceDialect; +use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; + +use crate::analysis::main::analyze::analyze; +use crate::analysis::tests::test_types::AnalysisSummary; + +pub(crate) fn summarise(sql: &str) -> AnalysisSummary { + let analysis = analyze(sql, SourceDialect::Snowflake, &builtin_catalogue()); + let construct_refusals = analysis + .refusals + .iter() + .filter(|refusal| refusal.construct.is_some()) + .count(); + AnalysisSummary { + parsed: analysis.parsed, + statements: analysis.statements, + categories: analysis + .refusals + .iter() + .map(|refusal| refusal.category) + .collect::>() + .into_iter() + .collect(), + unrefused_constructs: analysis.occurrences.len().abs_diff(construct_refusals), + unlocated_construct_refusals: analysis + .refusals + .iter() + .filter(|refusal| refusal.construct.is_some() && refusal.location.is_none()) + .count(), + } +} diff --git a/crates/sqlquack/src/analysis/tests/test_analyze.rs b/crates/sqlquack/src/analysis/tests/test_analyze.rs new file mode 100644 index 000000000..5fde93d59 --- /dev/null +++ b/crates/sqlquack/src/analysis/tests/test_analyze.rs @@ -0,0 +1,53 @@ +use sqlquack_core::refusals::models::RefusalCategory; + +use crate::analysis::tests::helpers::summarise; +use crate::analysis::tests::test_types::{AnalysisSummary, AnalyzeTestCase}; + +#[test] +fn given_sql_and_builtin_catalogue_when_analyzing_then_everything_is_refused_with_a_reason() { + let test_cases = [ + AnalyzeTestCase { + description: "every construct of a faithful query is a missing rule", + sql: "SELECT c.customer_id, SUM(o.amount) AS total FROM customers c \ + JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id", + expected_summary: AnalysisSummary { + parsed: true, + statements: 1, + categories: vec![RefusalCategory::MissingRule], + unrefused_constructs: 0, + unlocated_construct_refusals: 0, + }, + }, + AnalyzeTestCase { + description: "a canonicalising print-back adds a parse-fidelity refusal", + sql: "SELECT TOP 5 order_id FROM orders", + expected_summary: AnalysisSummary { + parsed: true, + statements: 1, + categories: vec![RefusalCategory::MissingRule, RefusalCategory::ParseFidelity], + unrefused_constructs: 0, + unlocated_construct_refusals: 0, + }, + }, + AnalyzeTestCase { + description: "unparseable SQL is one parse-fidelity refusal", + sql: "SELECT FROM WHERE", + expected_summary: AnalysisSummary { + parsed: false, + statements: 0, + categories: vec![RefusalCategory::ParseFidelity], + unrefused_constructs: 0, + unlocated_construct_refusals: 0, + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + summarise(test_case.sql), + test_case.expected_summary, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/analysis/tests/test_types.rs b/crates/sqlquack/src/analysis/tests/test_types.rs new file mode 100644 index 000000000..ce51e9b27 --- /dev/null +++ b/crates/sqlquack/src/analysis/tests/test_types.rs @@ -0,0 +1,16 @@ +use sqlquack_core::refusals::models::RefusalCategory; + +pub(crate) struct AnalyzeTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_summary: AnalysisSummary, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct AnalysisSummary { + pub(crate) parsed: bool, + pub(crate) statements: usize, + pub(crate) categories: Vec, + pub(crate) unrefused_constructs: usize, + pub(crate) unlocated_construct_refusals: usize, +} diff --git a/crates/sqlquack/src/api/main/analyze.rs b/crates/sqlquack/src/api/main/analyze.rs deleted file mode 100644 index 0acdfbdce..000000000 --- a/crates/sqlquack/src/api/main/analyze.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::api::models; - -pub fn analyze(unit: models::SqlUnit, _options: models::AnalyzeOptions) -> models::AnalysisResult { - let _ = unit; - - models::AnalysisResult { - diagnostics: Vec::new(), - } -} diff --git a/crates/sqlquack/src/api/mod.rs b/crates/sqlquack/src/api/mod.rs deleted file mode 100644 index 5355003ff..000000000 --- a/crates/sqlquack/src/api/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod main; -pub mod models; diff --git a/crates/sqlquack/src/api/models.rs b/crates/sqlquack/src/api/models.rs deleted file mode 100644 index 0beca7214..000000000 --- a/crates/sqlquack/src/api/models.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::diagnostics::models as diagnostics_models; -use crate::dialect::models as dialect_models; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SqlUnit { - pub sql: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AnalyzeOptions { - pub dialect: dialect_models::Dialect, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AnalysisResult { - pub diagnostics: Vec, -} diff --git a/crates/sqlquack/src/ast/mod.rs b/crates/sqlquack/src/ast/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/ast/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/bin/sqlquack.rs b/crates/sqlquack/src/bin/sqlquack.rs new file mode 100644 index 000000000..3a561579b --- /dev/null +++ b/crates/sqlquack/src/bin/sqlquack.rs @@ -0,0 +1,9 @@ +//! `sqlquack`: the command line. + +use std::process::ExitCode; + +use sqlquack::cli::main::run_cli::run_cli; + +fn main() -> ExitCode { + run_cli() +} diff --git a/crates/sqlquack/src/capabilities/mod.rs b/crates/sqlquack/src/capabilities/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/capabilities/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/census/_helpers/aggregate.rs b/crates/sqlquack/src/census/_helpers/aggregate.rs new file mode 100644 index 000000000..10f7cb004 --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/aggregate.rs @@ -0,0 +1,157 @@ +//! Folds per-file censuses into the report. + +use std::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet}; + +use sqlquack_core::classification::models::Construct; +use sqlquack_core::parsing::models::SourceDialect; +use sqlquack_core::refusals::models::RefusalCategory; + +use crate::census::_helpers::coverage::{CoverageInput, convertible_at, greedy_order}; +use crate::census::_helpers::inspect::{FidelityClass, FileCensus}; +use crate::census::models::{ + CensusReport, CensusTotals, ConstructCount, CoveragePoint, FidelityTotals, RefusalTotals, +}; + +pub(crate) fn aggregate( + files: &[FileCensus], + dialect: SourceDialect, + coverage_points: &[usize], +) -> CensusReport { + let mut usage: BTreeMap<&Construct, usize> = BTreeMap::new(); + for file in files { + for construct in &file.constructs { + *usage.entry(construct).or_default() += 1; + } + } + let ids: BTreeMap<&Construct, usize> = usage + .keys() + .enumerate() + .map(|(id, construct)| (*construct, id)) + .collect(); + let mut ranked: Vec<(&Construct, usize)> = usage.iter().map(|(c, n)| (*c, *n)).collect(); + ranked.sort_by_key(|(construct, count)| (Reverse(*count), *construct)); + let frequency_order: Vec = ranked.iter().map(|(construct, _)| ids[construct]).collect(); + let file_ids: Vec>> = + files.iter().map(|file| construct_ids(file, &ids)).collect(); + CensusReport { + dialect, + totals: totals(files, &usage), + fidelity: fidelity_totals(files), + refusals: refusal_totals(files), + constructs: ranked + .iter() + .map(|(construct, count)| ConstructCount { + construct: construct.key(), + files: *count, + }) + .collect(), + coverage: coverage(&file_ids, &frequency_order, coverage_points), + } +} + +fn totals(files: &[FileCensus], usage: &BTreeMap<&Construct, usize>) -> CensusTotals { + let mut constructs_by_family: BTreeMap = BTreeMap::new(); + for construct in usage.keys() { + *constructs_by_family + .entry(construct.family.prefix().to_owned()) + .or_default() += 1; + } + let parsed = files.iter().filter(|file| file.parsed).count(); + CensusTotals { + files: files.len(), + parsed, + unparsed: files.len() - parsed, + empty: files + .iter() + .filter(|file| file.parsed && file.statements == 0) + .count(), + statements: files.iter().map(|file| file.statements).sum(), + distinct_constructs: usage.len(), + constructs_by_family, + } +} + +fn fidelity_totals(files: &[FileCensus]) -> FidelityTotals { + let mut totals = FidelityTotals::default(); + for file in files { + match file.fidelity { + Some(FidelityClass::Identical) => totals.identical += 1, + Some(FidelityClass::Canonicalised) => totals.canonicalised += 1, + Some(FidelityClass::Unstable) => totals.unstable += 1, + Some(FidelityClass::Failed) => totals.failed += 1, + None => {} + } + } + totals +} + +fn refusal_totals(files: &[FileCensus]) -> RefusalTotals { + let count = |category: RefusalCategory| { + files + .iter() + .filter(|file| file.refusal_categories.contains(&category)) + .count() + }; + RefusalTotals { + missing_rule: count(RefusalCategory::MissingRule), + not_expressible: count(RefusalCategory::NotExpressible), + parse_fidelity: count(RefusalCategory::ParseFidelity), + convertible: files + .iter() + .filter(|file| file.refusal_categories.is_empty()) + .count(), + } +} + +fn coverage( + files: &[Option>], + frequency_order: &[usize], + coverage_points: &[usize], +) -> Vec { + let construct_count = frequency_order.len(); + let mut points: BTreeSet = coverage_points + .iter() + .copied() + .filter(|point| *point < construct_count) + .collect(); + points.insert(construct_count); + let points: Vec = points.into_iter().collect(); + let input = CoverageInput { + files, + construct_count, + frequency_order, + }; + let frequency = convertible_at(files, frequency_order, construct_count, &points); + let greedy = convertible_at(files, &greedy_order(&input), construct_count, &points); + let share = |count: usize| { + if files.is_empty() { + 0.0 + } else { + count as f64 / files.len() as f64 + } + }; + points + .iter() + .zip(frequency.iter().zip(&greedy)) + .map(|(rules, (frequency, greedy))| CoveragePoint { + rules: *rules, + frequency_order: *frequency, + greedy_order: *greedy, + frequency_share: share(*frequency), + greedy_share: share(*greedy), + }) + .collect() +} + +fn construct_ids(file: &FileCensus, ids: &BTreeMap<&Construct, usize>) -> Option> { + if !file.parsed { + return None; + } + Some( + file.constructs + .iter() + .map(|construct| ids[construct]) + .collect(), + ) +} diff --git a/crates/sqlquack/src/census/_helpers/coverage.rs b/crates/sqlquack/src/census/_helpers/coverage.rs new file mode 100644 index 000000000..367d83c99 --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/coverage.rs @@ -0,0 +1,83 @@ +//! Convertible files as constructs get rules, in frequency and greedy order. + +use std::cmp::Reverse; + +pub(crate) struct CoverageInput<'a> { + pub(crate) files: &'a [Option>], + pub(crate) construct_count: usize, + pub(crate) frequency_order: &'a [usize], +} + +pub(crate) fn greedy_order(input: &CoverageInput<'_>) -> Vec { + let mut frequency_rank = vec![0usize; input.construct_count]; + for (rank, construct) in input.frequency_order.iter().enumerate() { + frequency_rank[*construct] = rank; + } + let mut chosen = vec![false; input.construct_count]; + let mut missing: Vec = input + .files + .iter() + .map(|file| file.as_ref().map_or(0, Vec::len)) + .collect(); + let mut order: Vec = Vec::with_capacity(input.construct_count); + let mut unlocks = vec![0usize; input.construct_count]; + while order.len() < input.construct_count { + unlocks.iter_mut().for_each(|count| *count = 0); + for (file, remaining) in input.files.iter().zip(&missing) { + if let (Some(constructs), 1) = (file, remaining) + && let Some(last) = constructs.iter().find(|construct| !chosen[**construct]) + { + unlocks[*last] += 1; + } + } + let Some(next) = (0..input.construct_count) + .filter(|construct| !chosen[*construct]) + .min_by_key(|construct| (Reverse(unlocks[*construct]), frequency_rank[*construct])) + else { + break; + }; + chosen[next] = true; + order.push(next); + for (file, remaining) in input.files.iter().zip(missing.iter_mut()) { + if file + .as_ref() + .is_some_and(|constructs| constructs.contains(&next)) + { + *remaining -= 1; + } + } + } + order +} + +pub(crate) fn convertible_at( + files: &[Option>], + order: &[usize], + construct_count: usize, + points: &[usize], +) -> Vec { + let mut rank = vec![usize::MAX; construct_count]; + for (position, construct) in order.iter().enumerate() { + rank[*construct] = position; + } + let mut needed: Vec = Vec::with_capacity(files.len()); + for constructs in files.iter().flatten() { + needed.push(rules_needed(constructs, &rank)); + } + points + .iter() + .map(|point| convertible_within(&needed, *point)) + .collect() +} + +fn convertible_within(needed: &[usize], rules: usize) -> usize { + needed.iter().filter(|need| **need <= rules).count() +} + +fn rules_needed(constructs: &[usize], rank: &[usize]) -> usize { + constructs + .iter() + .map(|construct| rank[*construct].saturating_add(1)) + .max() + .unwrap_or(0) +} diff --git a/crates/sqlquack/src/census/_helpers/files.rs b/crates/sqlquack/src/census/_helpers/files.rs new file mode 100644 index 000000000..686efd5ef --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/files.rs @@ -0,0 +1,46 @@ +//! The SQL files under the census paths. + +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; + +use crate::census::errors::CensusError; + +const SQL_EXTENSION: &str = "sql"; + +pub(crate) fn collect_sql_files(paths: &[PathBuf]) -> Result, CensusError> { + let mut files: Vec = Vec::new(); + for path in paths { + if path.is_file() { + files.push(path.clone()); + continue; + } + if !path.is_dir() { + return Err(CensusError::Read { + path: path.clone(), + message: "no such file or directory".to_owned(), + }); + } + for entry in WalkDir::new(path).follow_links(true) { + let entry = entry.map_err(|error| CensusError::Read { + path: error.path().map_or_else(|| path.clone(), Path::to_path_buf), + message: error.to_string(), + })?; + if entry.file_type().is_file() && is_sql(entry.path()) { + files.push(entry.into_path()); + } + } + } + files.sort(); + files.dedup(); + if files.is_empty() { + return Err(CensusError::NoInputs); + } + Ok(files) +} + +fn is_sql(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case(SQL_EXTENSION)) +} diff --git a/crates/sqlquack/src/census/_helpers/inspect.rs b/crates/sqlquack/src/census/_helpers/inspect.rs new file mode 100644 index 000000000..460abfdf5 --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/inspect.rs @@ -0,0 +1,69 @@ +//! One file's census: its constructs, print-back outcome and refusal categories. + +use std::collections::BTreeSet; +use std::path::Path; + +use sqlquack_core::classification::models::Construct; +use sqlquack_core::parsing::models::SourceDialect; +use sqlquack_core::refusals::models::RefusalCategory; +use sqlquack_core::refusals::types::ConstructPolicy; +use sqlquack_core::reparse::models::FidelityOutcome; + +use crate::analysis::main::analyze::analyze; +use crate::census::_helpers::placeholders::rewrite_placeholders; +use crate::census::errors::CensusError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum FidelityClass { + Identical, + Canonicalised, + Unstable, + Failed, +} + +pub(crate) struct FileCensus { + pub(crate) parsed: bool, + pub(crate) statements: usize, + pub(crate) constructs: BTreeSet, + pub(crate) fidelity: Option, + pub(crate) refusal_categories: BTreeSet, +} + +pub(crate) fn inspect_file( + path: &Path, + dialect: SourceDialect, + policy: &(dyn ConstructPolicy + Sync), +) -> Result { + let bytes = std::fs::read(path).map_err(|error| CensusError::Read { + path: path.to_path_buf(), + message: error.to_string(), + })?; + let sql = rewrite_placeholders(&String::from_utf8_lossy(&bytes)); + let analysis = analyze(&sql, dialect, policy); + Ok(FileCensus { + parsed: analysis.parsed, + statements: analysis.statements, + constructs: analysis + .occurrences + .into_iter() + .map(|occurrence| occurrence.construct) + .collect(), + fidelity: analysis.fidelity.iter().map(fidelity_class).max(), + refusal_categories: analysis + .refusals + .iter() + .map(|refusal| refusal.category) + .collect(), + }) +} + +fn fidelity_class(outcome: &FidelityOutcome) -> FidelityClass { + match outcome { + FidelityOutcome::Identical => FidelityClass::Identical, + FidelityOutcome::Canonicalised { .. } => FidelityClass::Canonicalised, + FidelityOutcome::Unstable { .. } => FidelityClass::Unstable, + FidelityOutcome::PrintFailed { .. } | FidelityOutcome::ReparseFailed { .. } => { + FidelityClass::Failed + } + } +} diff --git a/crates/sqlquack/src/census/_helpers/mod.rs b/crates/sqlquack/src/census/_helpers/mod.rs new file mode 100644 index 000000000..1eaf7833e --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod aggregate; +pub(crate) mod coverage; +pub(crate) mod files; +pub(crate) mod inspect; +pub(crate) mod placeholders; diff --git a/crates/sqlquack/src/census/_helpers/placeholders.rs b/crates/sqlquack/src/census/_helpers/placeholders.rs new file mode 100644 index 000000000..519c2334a --- /dev/null +++ b/crates/sqlquack/src/census/_helpers/placeholders.rs @@ -0,0 +1,150 @@ +//! Replaces sqlbuild reference calls outside comments and quoted text with parseable SQL. + +use sqlquack_core::classification::constants::PROJECT_FUNCTION_PLACEHOLDER; + +use crate::census::constants::{ + CURSOR_CALLS, CURSOR_PLACEHOLDER, RELATION_CALLS, RELATION_PLACEHOLDER, + TABLE_FUNCTION_PLACEHOLDER, +}; + +const PROJECT_FUNCTION_CALL: &str = "__udf"; +const TABLE_FUNCTION_CALL: &str = "__table_fn"; + +pub(crate) fn rewrite_placeholders(sql: &str) -> String { + if !sql.contains("__") { + return sql.to_owned(); + } + let bytes = sql.as_bytes(); + let mut rewritten = String::with_capacity(sql.len()); + let mut position = 0; + let mut copied = 0; + while position < bytes.len() { + let skipped = skip_inert(bytes, position); + if skipped > position { + position = skipped; + continue; + } + if bytes[position] == b'_' + && bytes.get(position + 1) == Some(&b'_') + && (position == 0 || !is_identifier_byte(bytes[position - 1])) + { + let name_end = identifier_end(bytes, position); + let name = sql[position..name_end].to_ascii_lowercase(); + if let Some((replacement, end)) = replacement(bytes, &name, name_end) { + rewritten.push_str(&sql[copied..position]); + rewritten.push_str(replacement); + position = end; + copied = end; + continue; + } + position = name_end; + continue; + } + position += 1; + } + rewritten.push_str(&sql[copied..]); + rewritten +} + +fn replacement(bytes: &[u8], name: &str, name_end: usize) -> Option<(&'static str, usize)> { + let open = skip_whitespace(bytes, name_end); + let call_end = if bytes.get(open) == Some(&b'(') { + Some(matching_paren(bytes, open)?) + } else { + None + }; + if RELATION_CALLS.contains(&name) { + return Some((RELATION_PLACEHOLDER, call_end?)); + } + if CURSOR_CALLS.contains(&name) { + return Some((CURSOR_PLACEHOLDER, call_end.unwrap_or(name_end))); + } + match name { + PROJECT_FUNCTION_CALL => Some((PROJECT_FUNCTION_PLACEHOLDER, call_end?)), + TABLE_FUNCTION_CALL => Some((TABLE_FUNCTION_PLACEHOLDER, call_end?)), + _ => None, + } +} + +fn is_identifier_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'$' +} + +fn identifier_end(bytes: &[u8], start: usize) -> usize { + let mut end = start; + while end < bytes.len() && is_identifier_byte(bytes[end]) { + end += 1; + } + end +} + +fn skip_whitespace(bytes: &[u8], start: usize) -> usize { + let mut end = start; + while end < bytes.len() && bytes[end].is_ascii_whitespace() { + end += 1; + } + end +} + +fn matching_paren(bytes: &[u8], open: usize) -> Option { + let mut depth = 0usize; + let mut position = open; + while position < bytes.len() { + let skipped = skip_inert(bytes, position); + if skipped > position { + position = skipped; + continue; + } + match bytes[position] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(position + 1); + } + } + _ => {} + } + position += 1; + } + None +} + +fn skip_inert(bytes: &[u8], start: usize) -> usize { + match (bytes[start], bytes.get(start + 1)) { + (b'-', Some(b'-')) => find(bytes, start + 2, b"\n").unwrap_or(bytes.len()), + (b'/', Some(b'*')) => find(bytes, start + 2, b"*/").map_or(bytes.len(), |end| end + 2), + (b'$', Some(b'$')) => find(bytes, start + 2, b"$$").map_or(bytes.len(), |end| end + 2), + (b'\'', _) => quoted_end(bytes, start, b'\'', true), + (b'"', _) => quoted_end(bytes, start, b'"', false), + _ => start, + } +} + +fn quoted_end(bytes: &[u8], start: usize, quote: u8, backslash_escapes: bool) -> usize { + let mut position = start + 1; + while position < bytes.len() { + let byte = bytes[position]; + if backslash_escapes && byte == b'\\' { + position += 2; + continue; + } + if byte == quote { + if bytes.get(position + 1) == Some("e) { + position += 2; + continue; + } + return position + 1; + } + position += 1; + } + bytes.len() +} + +fn find(bytes: &[u8], start: usize, needle: &[u8]) -> Option { + bytes + .get(start..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|offset| start + offset) +} diff --git a/crates/sqlquack/src/census/constants.rs b/crates/sqlquack/src/census/constants.rs new file mode 100644 index 000000000..4705bafd6 --- /dev/null +++ b/crates/sqlquack/src/census/constants.rs @@ -0,0 +1,31 @@ +//! Census defaults and the placeholders that stand in for sqlbuild references. + +/// Rule counts at which the convertible share is reported. +pub const COVERAGE_POINTS: &[usize] = &[10, 25, 50, 75, 100, 150, 200, 250]; + +/// Constructs listed in the text report unless asked otherwise. +pub const DEFAULT_TEXT_LIMIT: usize = 50; + +/// Stack for each census worker. Polyglot's trees and their JSON form are deep for generated SQL. +pub const WORKER_STACK_BYTES: usize = 256 * 1024 * 1024; + +/// The table that stands in for sqlbuild model, source, seed and fixture references. +pub const RELATION_PLACEHOLDER: &str = "__sqlquack_relation"; + +/// The table function that stands in for sqlbuild table-function references. +pub const TABLE_FUNCTION_PLACEHOLDER: &str = "__sqlquack_table_fn"; + +/// The column that stands in for sqlbuild cursor bounds. +pub const CURSOR_PLACEHOLDER: &str = "__sqlquack_cursor"; + +/// sqlbuild reference calls replaced by a relation. +pub const RELATION_CALLS: &[&str] = &[ + "__ref", + "__source", + "__seed", + "__dbt_ref", + "__empty_fixture", +]; + +/// sqlbuild cursor intrinsics replaced by a column. +pub const CURSOR_CALLS: &[&str] = &["__cursor_start", "__cursor_end"]; diff --git a/crates/sqlquack/src/census/errors.rs b/crates/sqlquack/src/census/errors.rs new file mode 100644 index 000000000..6f6502c51 --- /dev/null +++ b/crates/sqlquack/src/census/errors.rs @@ -0,0 +1,34 @@ +//! Census failures. + +use std::fmt::{self, Display}; +use std::path::PathBuf; + +/// Why a census could not run. +#[derive(Debug)] +pub enum CensusError { + /// No `.sql` file was found under the given paths. + NoInputs, + /// A path could not be read. + Read { + /// The path. + path: PathBuf, + /// The operating system's message. + message: String, + }, + /// The worker threads could not start. + Workers(String), +} + +impl Display for CensusError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NoInputs => formatter.write_str("no .sql files found under the given paths"), + Self::Read { path, message } => { + write!(formatter, "cannot read {}: {message}", path.display()) + } + Self::Workers(message) => write!(formatter, "cannot start census workers: {message}"), + } + } +} + +impl std::error::Error for CensusError {} diff --git a/crates/sqlquack/src/census/main/mod.rs b/crates/sqlquack/src/census/main/mod.rs new file mode 100644 index 000000000..bdf127706 --- /dev/null +++ b/crates/sqlquack/src/census/main/mod.rs @@ -0,0 +1,2 @@ +pub mod render_census_text; +pub mod run_census; diff --git a/crates/sqlquack/src/census/main/render_census_text.rs b/crates/sqlquack/src/census/main/render_census_text.rs new file mode 100644 index 000000000..585ccbc3d --- /dev/null +++ b/crates/sqlquack/src/census/main/render_census_text.rs @@ -0,0 +1,83 @@ +//! `render_census_text`: the census as a plain-text report. + +use std::fmt::Write; + +use crate::census::models::CensusReport; + +/// The report as text, listing at most `limit` constructs. +pub fn render_census_text(report: &CensusReport, limit: usize) -> String { + let mut text = String::new(); + let totals = &report.totals; + let families: Vec = totals + .constructs_by_family + .iter() + .map(|(family, count)| format!("{count} {family}")) + .collect(); + let _ = writeln!( + text, + "Census of {} {} SQL files", + totals.files, + report.dialect.name() + ); + let _ = writeln!( + text, + " parsed {} ({} statements), unparsed {}, empty {}", + totals.parsed, totals.statements, totals.unparsed, totals.empty + ); + let _ = writeln!( + text, + " distinct constructs {} ({})", + totals.distinct_constructs, + families.join(", ") + ); + let fidelity = &report.fidelity; + let _ = writeln!( + text, + " print-back: identical {}, canonicalised {}, unstable {}, failed {}", + fidelity.identical, fidelity.canonicalised, fidelity.unstable, fidelity.failed + ); + let refusals = &report.refusals; + let _ = writeln!( + text, + " files refused: missing rule {}, not expressible {}, parse fidelity {}; convertible {}", + refusals.missing_rule, + refusals.not_expressible, + refusals.parse_fidelity, + refusals.convertible + ); + let _ = writeln!(text); + let _ = writeln!(text, "Convertible files as the top constructs get rules"); + let _ = writeln!( + text, + " {:>6} {:>16} {:>16}", + "rules", "frequency order", "greedy order" + ); + for point in &report.coverage { + let _ = writeln!( + text, + " {:>6} {:>8} {:>6.1}% {:>8} {:>6.1}%", + point.rules, + point.frequency_order, + point.frequency_share * 100.0, + point.greedy_order, + point.greedy_share * 100.0 + ); + } + let _ = writeln!(text); + let shown = report.constructs.len().min(limit); + let _ = writeln!( + text, + "Constructs by files using them (top {shown} of {})", + report.constructs.len() + ); + for (rank, count) in report.constructs.iter().take(limit).enumerate() { + let _ = writeln!( + text, + " {:>4} {:>6} {}", + rank + 1, + count.files, + count.construct + ); + } + text +} diff --git a/crates/sqlquack/src/census/main/run_census.rs b/crates/sqlquack/src/census/main/run_census.rs new file mode 100644 index 000000000..3f1b98a3a --- /dev/null +++ b/crates/sqlquack/src/census/main/run_census.rs @@ -0,0 +1,35 @@ +//! `run_census`: the census of every SQL file under some paths. + +use std::path::PathBuf; + +use rayon::ThreadPoolBuilder; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; + +use crate::census::_helpers::aggregate::aggregate; +use crate::census::_helpers::files::collect_sql_files; +use crate::census::_helpers::inspect::{FileCensus, inspect_file}; +use crate::census::constants::WORKER_STACK_BYTES; +use crate::census::errors::CensusError; +use crate::census::models::{CensusOptions, CensusReport}; + +/// The census of every `.sql` file under `paths` against the built-in rule catalogue. +pub fn run_census(paths: &[PathBuf], options: &CensusOptions) -> Result { + let files = collect_sql_files(paths)?; + let catalogue = builtin_catalogue(); + let workers = ThreadPoolBuilder::new() + .stack_size(WORKER_STACK_BYTES) + .build() + .map_err(|error| CensusError::Workers(error.to_string()))?; + let censuses: Vec = workers.install(|| { + files + .par_iter() + .map(|path| inspect_file(path, options.dialect, &catalogue)) + .collect::, CensusError>>() + })?; + Ok(aggregate( + &censuses, + options.dialect, + &options.coverage_points, + )) +} diff --git a/crates/sqlquack/src/census/mod.rs b/crates/sqlquack/src/census/mod.rs new file mode 100644 index 000000000..0bf5dcc34 --- /dev/null +++ b/crates/sqlquack/src/census/mod.rs @@ -0,0 +1,10 @@ +//! `sqlquack census`: constructs used, and the files that convert as rules are added. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack/src/census/models.rs b/crates/sqlquack/src/census/models.rs new file mode 100644 index 000000000..8bac57f50 --- /dev/null +++ b/crates/sqlquack/src/census/models.rs @@ -0,0 +1,101 @@ +//! Census options and reports. + +use std::collections::BTreeMap; + +use serde::Serialize; +use sqlquack_core::parsing::models::SourceDialect; + +/// What a census reads and reports. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CensusOptions { + /// The dialect the SQL is written in. + pub dialect: SourceDialect, + /// Rule counts at which the convertible share is reported. + pub coverage_points: Vec, +} + +/// A census report: counts and construct keys only, never SQL text or file names. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct CensusReport { + /// The source dialect. + pub dialect: SourceDialect, + /// File and construct totals. + pub totals: CensusTotals, + /// Files by the source print-back check's worst statement. + pub fidelity: FidelityTotals, + /// Files with at least one refusal of each category. + pub refusals: RefusalTotals, + /// Every construct, by the number of files using it, most used first. + pub constructs: Vec, + /// Files that would convert as the top constructs get rules. + pub coverage: Vec, +} + +/// File and construct totals. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct CensusTotals { + /// SQL files read. + pub files: usize, + /// Files that parsed. + pub parsed: usize, + /// Files that did not parse. + pub unparsed: usize, + /// Files with no statement. + pub empty: usize, + /// Statements parsed. + pub statements: usize, + /// Distinct constructs used. + pub distinct_constructs: usize, + /// Distinct constructs per family. + pub constructs_by_family: BTreeMap, +} + +/// Files by their worst source print-back outcome. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct FidelityTotals { + /// Every statement printed back to the same tree. + pub identical: usize, + /// A print-back canonicalised the tree once, then settled. + pub canonicalised: usize, + /// A print-back kept changing the tree. + pub unstable: usize, + /// A tree did not print, or its print did not parse. + pub failed: usize, +} + +/// Files with at least one refusal of each category. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RefusalTotals { + /// Files with a construct no rule converts. + pub missing_rule: usize, + /// Files with a construct DuckDB cannot express. + pub not_expressible: usize, + /// Files that did not parse or did not print back faithfully. + pub parse_fidelity: usize, + /// Files with no refusal at all. + pub convertible: usize, +} + +/// How many files use one construct. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ConstructCount { + /// The construct key. + pub construct: String, + /// Files using it. + pub files: usize, +} + +/// Files that convert once the first `rules` constructs have rules, in two orders. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct CoveragePoint { + /// Constructs with rules. + pub rules: usize, + /// Convertible files when rules follow file frequency. + pub frequency_order: usize, + /// Convertible files when each next rule is the one that completes the most files. + pub greedy_order: usize, + /// `frequency_order` as a share of all files. + pub frequency_share: f64, + /// `greedy_order` as a share of all files. + pub greedy_share: f64, +} diff --git a/crates/sqlquack/src/census/tests.rs b/crates/sqlquack/src/census/tests.rs new file mode 100644 index 000000000..9b894cf1a --- /dev/null +++ b/crates/sqlquack/src/census/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_coverage.rs"] +mod test_coverage; +#[path = "tests/test_placeholders.rs"] +mod test_placeholders; +#[path = "tests/test_run_census.rs"] +mod test_run_census; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack/src/census/tests/helpers.rs b/crates/sqlquack/src/census/tests/helpers.rs new file mode 100644 index 000000000..a477e0936 --- /dev/null +++ b/crates/sqlquack/src/census/tests/helpers.rs @@ -0,0 +1,98 @@ +use std::path::Path; + +use sqlquack_core::parsing::models::SourceDialect; + +use crate::census::_helpers::coverage::{CoverageInput, greedy_order}; +use crate::census::main::render_census_text::render_census_text; +use crate::census::main::run_census::run_census; +use crate::census::models::{CensusOptions, CensusReport}; +use crate::census::tests::test_types::{CensusSummary, ShopFile}; + +pub(crate) const SHOP: &[ShopFile] = &[ + ShopFile { + path: "marts/customer_totals.sql", + contents: "SELECT customer_id, SUM(amount) AS total\nFROM __ref(\"orders\")\n\ + GROUP BY customer_id", + }, + ShopFile { + path: "staging/customers.sql", + contents: "SELECT customer_id, UPPER(name) AS name FROM __source(\"shop\", \"customers\")", + }, + ShopFile { + path: "staging/latest_orders.sql", + contents: "SELECT * FROM __ref(\"orders\")\n\ + QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) = 1", + }, + ShopFile { + path: "staging/broken.sql", + contents: "SELECT FROM WHERE", + }, + ShopFile { + path: "staging/notes.txt", + contents: "not SQL", + }, +]; + +pub(crate) const NOT_SQL: &[ShopFile] = &[ShopFile { + path: "notes.txt", + contents: "not SQL", +}]; + +fn write_files(root: &Path, files: &[ShopFile]) { + for file in files { + let path = root.join(file.path); + std::fs::create_dir_all(path.parent().expect("fixture files have a parent")) + .expect("fixture directory is created"); + std::fs::write(path, file.contents).expect("fixture file is written"); + } +} + +fn files_using(report: &CensusReport, key: &str) -> usize { + report + .constructs + .iter() + .find(|count| count.construct == key) + .map_or(0, |count| count.files) +} + +fn summarise(report: &CensusReport) -> CensusSummary { + let text = render_census_text(report, 5); + CensusSummary { + files: report.totals.files, + parsed: report.totals.parsed, + unparsed: report.totals.unparsed, + missing_rule: report.refusals.missing_rule, + parse_fidelity: report.refusals.parse_fidelity, + convertible: report.refusals.convertible, + most_used_files: report.constructs.first().map_or(0, |count| count.files), + select_files: files_using(report, "node:select"), + qualify_files: files_using(report, "clause:qualify"), + all_rules_convertible: report + .coverage + .last() + .map_or(0, |point| point.frequency_order.min(point.greedy_order)), + text_mentions_sql: text.contains("customers") || text.contains("orders\""), + } +} + +pub(crate) fn census_summary(files: &[ShopFile]) -> Result { + let directory = tempfile::tempdir().expect("temporary directory"); + write_files(directory.path(), files); + run_census( + &[directory.path().to_path_buf()], + &CensusOptions { + dialect: SourceDialect::Snowflake, + coverage_points: vec![1, 5], + }, + ) + .map(|report| summarise(&report)) + .map_err(|error| error.to_string()) +} + +pub(crate) fn greedy(files: &[Option>], frequency_order: &[usize]) -> Vec { + greedy_order(&CoverageInput { + files, + construct_count: frequency_order.len(), + frequency_order, + }) +} diff --git a/crates/sqlquack/src/census/tests/test_coverage.rs b/crates/sqlquack/src/census/tests/test_coverage.rs new file mode 100644 index 000000000..b6e34461e --- /dev/null +++ b/crates/sqlquack/src/census/tests/test_coverage.rs @@ -0,0 +1,57 @@ +use crate::census::_helpers::coverage::convertible_at; +use crate::census::tests::helpers::greedy; +use crate::census::tests::test_types::{ConvertibleTestCase, GreedyTestCase}; + +#[test] +fn given_rule_order_when_counting_then_files_convert_once_all_their_constructs_have_rules() { + let test_cases = [ + ConvertibleTestCase { + description: "files need every construct they use; unparsed files never convert", + files: vec![Some(vec![0]), Some(vec![0, 1]), Some(vec![0, 2]), None], + order: vec![0, 1, 2], + points: vec![0, 1, 2, 3], + expected_convertible: vec![0, 1, 2, 3], + }, + ConvertibleTestCase { + description: "a file using only late constructs waits for them", + files: vec![Some(vec![2]), Some(vec![0, 1])], + order: vec![0, 1, 2], + points: vec![1, 2, 3], + expected_convertible: vec![0, 1, 2], + }, + ]; + + for test_case in test_cases { + assert_eq!( + convertible_at(&test_case.files, &test_case.order, 3, &test_case.points), + test_case.expected_convertible, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_construct_usage_when_ordering_greedily_then_rules_completing_most_files_come_first() { + let test_cases = [GreedyTestCase { + description: "a rare construct completing two files beats a common one completing none", + files: vec![ + Some(vec![1]), + Some(vec![1]), + Some(vec![0, 2]), + Some(vec![0, 3]), + Some(vec![0, 4]), + ], + frequency_order: vec![0, 1, 2, 3, 4], + expected_order: vec![1, 0, 2, 3, 4], + }]; + + for test_case in test_cases { + assert_eq!( + greedy(&test_case.files, &test_case.frequency_order), + test_case.expected_order, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/census/tests/test_placeholders.rs b/crates/sqlquack/src/census/tests/test_placeholders.rs new file mode 100644 index 000000000..4783a6ec4 --- /dev/null +++ b/crates/sqlquack/src/census/tests/test_placeholders.rs @@ -0,0 +1,53 @@ +use crate::census::_helpers::placeholders::rewrite_placeholders; +use crate::census::tests::test_types::RewriteTestCase; + +#[test] +fn given_sqlbuild_compiled_sql_when_rewriting_then_references_become_parseable_sql() { + let test_cases = [ + RewriteTestCase { + description: "model, source and seed references become one relation", + sql: "SELECT * FROM __ref(\"orders\") o JOIN __source(\"shop\", \"customers\") c \ + ON TRUE JOIN __seed( \"regions\" ) r ON TRUE", + expected_sql: "SELECT * FROM __sqlquack_relation o JOIN __sqlquack_relation c \ + ON TRUE JOIN __sqlquack_relation r ON TRUE", + }, + RewriteTestCase { + description: "project functions keep their argument list", + sql: "SELECT __udf(\"is_gift\")(o.note) FROM t", + expected_sql: "SELECT __sqlquack_udf(o.note) FROM t", + }, + RewriteTestCase { + description: "table functions keep their argument list", + sql: "SELECT * FROM TABLE(__table_fn(\"open_orders\")(1))", + expected_sql: "SELECT * FROM TABLE(__sqlquack_table_fn(1))", + }, + RewriteTestCase { + description: "cursor bounds become a column", + sql: "SELECT * FROM t WHERE updated_at >= __cursor_start()", + expected_sql: "SELECT * FROM t WHERE updated_at >= __sqlquack_cursor", + }, + RewriteTestCase { + description: "references in strings and comments stay", + sql: "SELECT '__ref(\"x\")' -- __ref(\"y\")\nFROM t /* __seed(\"z\") */", + expected_sql: "SELECT '__ref(\"x\")' -- __ref(\"y\")\nFROM t /* __seed(\"z\") */", + }, + RewriteTestCase { + description: "other double-underscore names stay", + sql: "SELECT __internal_flag, col__ref(1) FROM t", + expected_sql: "SELECT __internal_flag, col__ref(1) FROM t", + }, + RewriteTestCase { + description: "a parenthesis inside a reference string does not end the call", + sql: "SELECT * FROM __ref(\"odd)name\")", + expected_sql: "SELECT * FROM __sqlquack_relation", + }, + ]; + for test_case in test_cases { + assert_eq!( + rewrite_placeholders(test_case.sql), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/census/tests/test_run_census.rs b/crates/sqlquack/src/census/tests/test_run_census.rs new file mode 100644 index 000000000..75215736b --- /dev/null +++ b/crates/sqlquack/src/census/tests/test_run_census.rs @@ -0,0 +1,39 @@ +use crate::census::tests::helpers::{NOT_SQL, SHOP, census_summary}; +use crate::census::tests::test_types::{CensusSummary, CensusTestCase}; + +#[test] +fn given_sql_directory_when_running_census_then_counts_files_constructs_and_coverage() { + let test_cases = [ + CensusTestCase { + description: "compiled shop models with one unparseable file and one non-SQL file", + fixtures: SHOP, + expected_summary: Ok(CensusSummary { + files: 4, + parsed: 3, + unparsed: 1, + missing_rule: 3, + parse_fidelity: 1, + convertible: 0, + most_used_files: 3, + select_files: 3, + qualify_files: 1, + all_rules_convertible: 3, + text_mentions_sql: false, + }), + }, + CensusTestCase { + description: "a directory with no SQL files", + fixtures: NOT_SQL, + expected_summary: Err("no .sql files found under the given paths".to_owned()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + census_summary(test_case.fixtures), + test_case.expected_summary, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/census/tests/test_types.rs b/crates/sqlquack/src/census/tests/test_types.rs new file mode 100644 index 000000000..0ff780187 --- /dev/null +++ b/crates/sqlquack/src/census/tests/test_types.rs @@ -0,0 +1,46 @@ +pub(crate) struct RewriteTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: &'static str, +} + +pub(crate) struct ConvertibleTestCase { + pub(crate) description: &'static str, + pub(crate) files: Vec>>, + pub(crate) order: Vec, + pub(crate) points: Vec, + pub(crate) expected_convertible: Vec, +} + +pub(crate) struct GreedyTestCase { + pub(crate) description: &'static str, + pub(crate) files: Vec>>, + pub(crate) frequency_order: Vec, + pub(crate) expected_order: Vec, +} + +pub(crate) struct CensusTestCase { + pub(crate) description: &'static str, + pub(crate) fixtures: &'static [ShopFile], + pub(crate) expected_summary: Result, +} + +pub(crate) struct ShopFile { + pub(crate) path: &'static str, + pub(crate) contents: &'static str, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct CensusSummary { + pub(crate) files: usize, + pub(crate) parsed: usize, + pub(crate) unparsed: usize, + pub(crate) missing_rule: usize, + pub(crate) parse_fidelity: usize, + pub(crate) convertible: usize, + pub(crate) most_used_files: usize, + pub(crate) select_files: usize, + pub(crate) qualify_files: usize, + pub(crate) all_rules_convertible: usize, + pub(crate) text_mentions_sql: bool, +} diff --git a/crates/sqlquack/src/cli/_helpers/census_command.rs b/crates/sqlquack/src/cli/_helpers/census_command.rs new file mode 100644 index 000000000..698393b1c --- /dev/null +++ b/crates/sqlquack/src/cli/_helpers/census_command.rs @@ -0,0 +1,62 @@ +//! `sqlquack census`. + +use std::io::Write; +use std::process::ExitCode; +use std::time::Instant; + +use sqlquack_core::parsing::models::SourceDialect; + +use crate::census::constants::COVERAGE_POINTS; +use crate::census::main::render_census_text::render_census_text; +use crate::census::main::run_census::run_census; +use crate::census::models::CensusOptions; +use crate::cli::models::CensusArguments; + +pub(crate) fn census_command( + arguments: &CensusArguments, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> ExitCode { + let Some(dialect) = SourceDialect::from_name(&arguments.dialect) else { + let _ = writeln!( + stderr, + "sqlquack census: unsupported dialect '{}'; supported: snowflake", + arguments.dialect + ); + return ExitCode::from(2); + }; + let options = CensusOptions { + dialect, + coverage_points: COVERAGE_POINTS.to_vec(), + }; + let started = Instant::now(); + let _ = writeln!(stderr, "sqlquack census: reading SQL files…"); + let report = match run_census(&arguments.paths, &options) { + Ok(report) => report, + Err(error) => { + let _ = writeln!(stderr, "sqlquack census: failed: {error}"); + return ExitCode::FAILURE; + } + }; + let rendered = if arguments.json { + match serde_json::to_string_pretty(&report) { + Ok(json) => json + "\n", + Err(error) => { + let _ = writeln!(stderr, "sqlquack census: failed: {error}"); + return ExitCode::FAILURE; + } + } + } else { + render_census_text(&report, arguments.limit) + }; + if stdout.write_all(rendered.as_bytes()).is_err() { + return ExitCode::FAILURE; + } + let _ = writeln!( + stderr, + "sqlquack census: done, {} files in {:.1} s", + report.totals.files, + started.elapsed().as_secs_f64() + ); + ExitCode::SUCCESS +} diff --git a/crates/sqlquack/src/cli/_helpers/mod.rs b/crates/sqlquack/src/cli/_helpers/mod.rs new file mode 100644 index 000000000..4342abca6 --- /dev/null +++ b/crates/sqlquack/src/cli/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod census_command; diff --git a/crates/sqlquack/src/cli/main/mod.rs b/crates/sqlquack/src/cli/main/mod.rs new file mode 100644 index 000000000..5a23098c0 --- /dev/null +++ b/crates/sqlquack/src/cli/main/mod.rs @@ -0,0 +1 @@ +pub mod run_cli; diff --git a/crates/sqlquack/src/cli/main/run_cli.rs b/crates/sqlquack/src/cli/main/run_cli.rs new file mode 100644 index 000000000..3c5890a58 --- /dev/null +++ b/crates/sqlquack/src/cli/main/run_cli.rs @@ -0,0 +1,28 @@ +//! `run_cli`: parse arguments and run a command. + +use std::process::ExitCode; + +use clap::Parser; + +use crate::cli::_helpers::census_command::census_command; +use crate::cli::models::{Cli, Command}; + +/// Runs `sqlquack` with the process arguments. +pub fn run_cli() -> ExitCode { + let cli = match Cli::try_parse() { + Ok(cli) => cli, + Err(error) => { + let _ = error.print(); + return if error.use_stderr() { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + }; + } + }; + let mut stdout = std::io::stdout().lock(); + let mut stderr = std::io::stderr().lock(); + match &cli.command { + Command::Census(arguments) => census_command(arguments, &mut stdout, &mut stderr), + } +} diff --git a/crates/sqlquack/src/cli/mod.rs b/crates/sqlquack/src/cli/mod.rs new file mode 100644 index 000000000..c640691b6 --- /dev/null +++ b/crates/sqlquack/src/cli/mod.rs @@ -0,0 +1,5 @@ +//! The `sqlquack` command line. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs new file mode 100644 index 000000000..241e915ec --- /dev/null +++ b/crates/sqlquack/src/cli/models.rs @@ -0,0 +1,40 @@ +//! Command-line arguments. + +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; + +use crate::census::constants::DEFAULT_TEXT_LIMIT; + +/// Snowflake-to-DuckDB SQL conversion that refuses to guess. +#[derive(Debug, Parser)] +#[command(name = "sqlquack", version)] +pub struct Cli { + /// The command to run. + #[command(subcommand)] + pub command: Command, +} + +/// `sqlquack` commands. +#[derive(Debug, Subcommand)] +pub enum Command { + /// Count the constructs SQL files use and estimate how many files convert as rules are added. + Census(CensusArguments), +} + +/// Arguments of `sqlquack census`. +#[derive(Debug, Args)] +pub struct CensusArguments { + /// SQL files, or directories searched for `.sql` files: one query or compiled model per file. + #[arg(required = true)] + pub paths: Vec, + /// The dialect the SQL is written in. + #[arg(long, default_value = "snowflake")] + pub dialect: String, + /// Print the report as JSON on standard output. + #[arg(long)] + pub json: bool, + /// Constructs listed in the text report. + #[arg(long, default_value_t = DEFAULT_TEXT_LIMIT)] + pub limit: usize, +} diff --git a/crates/sqlquack/src/diagnostics/mod.rs b/crates/sqlquack/src/diagnostics/mod.rs deleted file mode 100644 index c446ac883..000000000 --- a/crates/sqlquack/src/diagnostics/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod models; diff --git a/crates/sqlquack/src/diagnostics/models.rs b/crates/sqlquack/src/diagnostics/models.rs deleted file mode 100644 index 47c39e355..000000000 --- a/crates/sqlquack/src/diagnostics/models.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Diagnostic { - pub severity: DiagnosticSeverity, - pub code: String, - pub message: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DiagnosticSeverity { - Error, - Warning, - Info, -} diff --git a/crates/sqlquack/src/dialect/mod.rs b/crates/sqlquack/src/dialect/mod.rs deleted file mode 100644 index c446ac883..000000000 --- a/crates/sqlquack/src/dialect/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod models; diff --git a/crates/sqlquack/src/dialect/models.rs b/crates/sqlquack/src/dialect/models.rs deleted file mode 100644 index 818556a2c..000000000 --- a/crates/sqlquack/src/dialect/models.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Dialect { - Generic, - Snowflake, - BigQuery, - Databricks, - DuckDb, -} diff --git a/crates/sqlquack/src/lexer/mod.rs b/crates/sqlquack/src/lexer/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/lexer/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/lib.rs b/crates/sqlquack/src/lib.rs index 2a867d4fd..ce9ab0599 100644 --- a/crates/sqlquack/src/lib.rs +++ b/crates/sqlquack/src/lib.rs @@ -1,19 +1,7 @@ -#![forbid(unsafe_code)] +//! SQLQuack converts Snowflake SQL into DuckDB SQL and refuses whatever it cannot prove correct. -mod analysis; -mod api; -mod ast; -mod capabilities; -pub mod diagnostics; -mod dialect; -mod lexer; -mod lineage; -mod parser; -mod span; -mod translation; -mod types; +#![forbid(unsafe_code)] -pub use api::main::analyze::analyze; -pub use api::models::{AnalysisResult, AnalyzeOptions, SqlUnit}; -pub use diagnostics::models::{Diagnostic, DiagnosticSeverity}; -pub use dialect::models::Dialect; +pub mod analysis; +pub mod census; +pub mod cli; diff --git a/crates/sqlquack/src/lineage/mod.rs b/crates/sqlquack/src/lineage/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/lineage/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/parser/mod.rs b/crates/sqlquack/src/parser/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/parser/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/span/mod.rs b/crates/sqlquack/src/span/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/span/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/translation/mod.rs b/crates/sqlquack/src/translation/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/translation/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/src/types/mod.rs b/crates/sqlquack/src/types/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/crates/sqlquack/src/types/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/sqlquack/tests/api.rs b/crates/sqlquack/tests/api.rs deleted file mode 100644 index d71e5dcc1..000000000 --- a/crates/sqlquack/tests/api.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[path = "api/analyze.rs"] -mod analyze; -#[path = "api/helpers.rs"] -mod helpers; -#[path = "api/test_types.rs"] -mod test_types; diff --git a/crates/sqlquack/tests/api/analyze.rs b/crates/sqlquack/tests/api/analyze.rs deleted file mode 100644 index e23ceff9c..000000000 --- a/crates/sqlquack/tests/api/analyze.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::helpers; -use crate::test_types; - -const TEST_CASES: &[test_types::AnalyzeTestCase] = &[test_types::AnalyzeTestCase { - description: "simple select returns no diagnostics in placeholder API", - sql: "select a from raw.orders", - expected_diagnostic_count: 0, -}]; - -#[test] -fn given_sql_unit_when_analyzing_then_returns_expected_diagnostics() { - for test_case in TEST_CASES { - let result = helpers::analyze_sql(test_case); - - assert_eq!( - result.diagnostics.len(), - test_case.expected_diagnostic_count, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlquack/tests/api/helpers.rs b/crates/sqlquack/tests/api/helpers.rs deleted file mode 100644 index ec6087ee9..000000000 --- a/crates/sqlquack/tests/api/helpers.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::test_types; - -pub(crate) fn analyze_sql(test_case: &test_types::AnalyzeTestCase) -> sqlquack::AnalysisResult { - sqlquack::analyze( - sqlquack::SqlUnit { - sql: test_case.sql.to_owned(), - }, - sqlquack::AnalyzeOptions { - dialect: sqlquack::Dialect::Generic, - }, - ) -} diff --git a/crates/sqlquack/tests/api/test_types.rs b/crates/sqlquack/tests/api/test_types.rs deleted file mode 100644 index fc234909e..000000000 --- a/crates/sqlquack/tests/api/test_types.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) struct AnalyzeTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_diagnostic_count: usize, -} diff --git a/crates/sqlquack/tests/census.rs b/crates/sqlquack/tests/census.rs new file mode 100644 index 000000000..c1e2cade0 --- /dev/null +++ b/crates/sqlquack/tests/census.rs @@ -0,0 +1,6 @@ +#[path = "census/helpers.rs"] +mod helpers; +#[path = "census/test_census_binary.rs"] +mod test_census_binary; +#[path = "census/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack/tests/census/helpers.rs b/crates/sqlquack/tests/census/helpers.rs new file mode 100644 index 000000000..6ce6e89fc --- /dev/null +++ b/crates/sqlquack/tests/census/helpers.rs @@ -0,0 +1,27 @@ +use std::process::Command; + +use crate::test_types::BinaryOutcome; + +const ORDERS_SQL: &str = "SELECT customer_id, COUNT(*) AS orders FROM __ref(\"orders\") \ + GROUP BY customer_id"; +const PRODUCTS_SQL: &str = "SELECT sku, price::NUMBER(10, 2) AS price FROM products"; + +pub(crate) fn run_on_shop(arguments: &[&str]) -> BinaryOutcome { + let directory = tempfile::tempdir().expect("temporary directory"); + std::fs::write(directory.path().join("orders.sql"), ORDERS_SQL).expect("fixture written"); + std::fs::write(directory.path().join("products.sql"), PRODUCTS_SQL).expect("fixture written"); + let output = Command::new(env!("CARGO_BIN_EXE_sqlquack")) + .args(arguments) + .arg(directory.path()) + .output() + .expect("sqlquack runs"); + let report: serde_json::Value = + serde_json::from_slice(&output.stdout).unwrap_or(serde_json::Value::Null); + BinaryOutcome { + exit_code: output.status.code(), + files: report["totals"]["files"].as_u64(), + missing_rule: report["refusals"]["missing_rule"].as_u64(), + completion_reported: String::from_utf8_lossy(&output.stderr) + .contains("sqlquack census: done, 2 files"), + } +} diff --git a/crates/sqlquack/tests/census/test_census_binary.rs b/crates/sqlquack/tests/census/test_census_binary.rs new file mode 100644 index 000000000..cf867c037 --- /dev/null +++ b/crates/sqlquack/tests/census/test_census_binary.rs @@ -0,0 +1,37 @@ +use crate::helpers::run_on_shop; +use crate::test_types::{BinaryOutcome, BinaryTestCase}; + +#[test] +fn given_sql_directory_when_running_census_binary_then_reports_on_the_right_streams() { + let test_cases = [ + BinaryTestCase { + description: "JSON goes to stdout and the completion message to stderr", + arguments: &["census", "--json"], + expected_outcome: BinaryOutcome { + exit_code: Some(0), + files: Some(2), + missing_rule: Some(2), + completion_reported: true, + }, + }, + BinaryTestCase { + description: "an unknown dialect is a usage error", + arguments: &["census", "--dialect", "unknown"], + expected_outcome: BinaryOutcome { + exit_code: Some(2), + files: None, + missing_rule: None, + completion_reported: false, + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + run_on_shop(test_case.arguments), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/tests/census/test_types.rs b/crates/sqlquack/tests/census/test_types.rs new file mode 100644 index 000000000..4daaaa313 --- /dev/null +++ b/crates/sqlquack/tests/census/test_types.rs @@ -0,0 +1,13 @@ +pub(crate) struct BinaryTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_outcome: BinaryOutcome, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct BinaryOutcome { + pub(crate) exit_code: Option, + pub(crate) files: Option, + pub(crate) missing_rule: Option, + pub(crate) completion_reported: bool, +} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 000000000..d59984355 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,94 @@ +# Architecture + +## Principle + +SQLQuack never passes a construct through unconverted. Identical syntax in two dialects is not +evidence of identical meaning, so every node of the output must come from a rule that is verified +against recorded Snowflake results, under stated type conditions. Anything else is refused. + +## Pipeline + +``` +source SQL ─ parse (polyglot, source dialect) ─▶ tree + ├─ fidelity: print back in the source dialect, re-parse, compare (else refuse: parse fidelity) + ├─ classify: every node, function, cast type, join kind and clause (one construct each) + ├─ convert: each construct must be claimed by a rule (else refuse: missing rule, + │ or not expressible) + ├─ print: polyglot's DuckDB printer + └─ check: re-parse the DuckDB SQL and compare with the intended tree (else refuse: printer) +``` + +Today the rule catalogue is empty, so the convert step refuses every construct. The source dialect is +a parameter of every entry point; Snowflake is the only one implemented. + +## Crates + +| Crate | Role | +|---|---| +| `sqlquack-core` | Parsing with complexity guards, construct classification, refusals, the tree comparator and both re-parse checks. | +| `sqlquack-snowflake` | The Snowflake typing layer (empty). Polyglot leaves many Snowflake expressions untyped, so exact types (`NUMBER(p, s)`, timestamp kinds, semi-structured values) are computed here. | +| `sqlquack-rules` | The rule catalogue and declared refusals (empty). | +| `sqlquack-duckdb` | DuckDB building blocks shared by rules (empty). | +| `sqlquack-compare` | The strict comparator for recorded results (stub). | +| `sqlquack` | The `sqlquack` command line and the library entry points: the pipeline and the census. | + +Polyglot is pinned to one revision with `default-features = false`. Its `transpile` feature stays +off: none of its own conversions are used. + +## Constructs + +A construct is a key `:`: + +- `node:`: a polyglot AST node kind, such as `node:select`, `node:eq` or `node:coalesce`; +- `fn:`: a function polyglot keeps as a generic call, by upper-case name; calls to project + functions are one construct, `fn:`; +- `cast:`: the base target type of a cast, as polyglot prints it in the source dialect; +- `join:`: a join kind; +- `clause:`: a query clause or set-operation form, such as `clause:qualify` or + `clause:union_all`; +- `syntax:`: a spelling recorded separately, such as `syntax:double_colon_cast`. + +Clause classification destructures polyglot's query structs field by field, so a polyglot upgrade +that adds a clause fails to compile until the clause is classified. + +Each construct carries a source position. Polyglot keeps positions only on identifiers, columns, +tables, stars and some calls; any other node takes the first position inside it, or else its +parent's. + +## Refusals + +Every refusal has a category, a construct (absent when a whole statement is refused), a position and +a reason: + +- **missing rule**: no verified rule converts the construct yet; +- **not expressible in DuckDB**: declared in the catalogue with a reason and evidence; +- **parse fidelity**: the SQL does not parse, or printing it back changes its tree. + +## Tree comparison + +Both re-parse checks compare trees as JSON after removing a declared list of spelling-only fields +(source positions, comments, `::` versus `CAST`, optional keywords and similar) and applying a +declared list of normalisations (function-name case, dollar-quoted strings, comment wrappers). Each +entry weakens the checks, so each has a reason and a test showing that two spellings parse to trees +that differ only there. A mismatch reports the first differing JSON path. + +## Source fidelity + +A statement is printed back in its own dialect and re-parsed. If the tree is unchanged it is +faithful. If it changes once and then settles, the change is a canonicalisation (for example `TOP` +becoming `LIMIT`); canonicalisations are refused until a recorded case proves both forms agree on +Snowflake. A tree that keeps changing is refused as unstable. + +## Census + +`sqlquack census` runs the pipeline over every `.sql` file and reports, per construct, how many files +use it, plus how many files would convert as the top N constructs get rules, in frequency order and +in greedy order (each next rule is the one that completes the most files). SQLBuild reference calls +(`__ref`, `__source`, `__seed`, `__udf`, `__table_fn`, cursor bounds) are replaced with neutral +placeholders before parsing. The report holds counts and construct keys only. + +## Evidence + +Recorded warehouse results are the only arbiter of correctness. Recording runs outside CI, through +the maintainer; results are append-only and named by the hash of their case, and everything that +could be weakened to make a failing case pass lives in one reviewed policy file. diff --git a/fensu.toml b/fensu.toml new file mode 100644 index 000000000..6ead1ad07 --- /dev/null +++ b/fensu.toml @@ -0,0 +1,12 @@ +[targets.rust] +analyzer = "rust" +roots = ["crates"] +tests = [] +tooling = [] +rule_packs = ["rust"] +select = ["FPRS"] + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlquack-core/src/reparse/main/check_duckdb_print.rs" +reason = "The DuckDB printer check is public library API; its first importer is the rule engine, which lands with the first rules." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..735c0aeb6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "sqlquack-workspace" +version = "0.0.0" +description = "Development tooling for the SQLQuack Rust workspace." +requires-python = ">=3.12" +dependencies = [] + +[dependency-groups] +dev = [ + "fensu>=0.27.0", + "ruff>=0.12.0", + "ty>=0.0.1a19", +] + +[tool.uv] +package = false + +[tool.ruff] +line-length = 100 +target-version = "py312" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..35440c48c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.99.1" +components = ["clippy", "rustfmt"] +profile = "minimal" diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..a1c9d6e6d --- /dev/null +++ b/uv.lock @@ -0,0 +1,103 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "fensu" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fensu-cli" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, + { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, + { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, +] + +[[package]] +name = "fensu-cli" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, + { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, + { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, + { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/58/690cf56da80c8f2c8bf12042b6017beac2c34fe961f65420bcb3fb7fac38/ruff-0.17.2.tar.gz", hash = "sha256:84d632402d14c23b488f941db23712d6736a013225162062d34558c3f3c780eb", size = 5333400, upload-time = "2026-10-22T18:38:00.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/b8/8b6d69c57e8fe1cc6af1aa7493b72a1a5b5a8afe34694e302025441729d4/ruff-0.17.2-py3-none-linux_armv6l.whl", hash = "sha256:96a65e755d8dba094707fb23ab602111d9a31ec2835b3a6992a1914a8fcb211e", size = 10183857, upload-time = "2026-10-22T18:37:14.956Z" }, + { url = "https://files.pythonhosted.org/packages/e8/61/12cc2ff7b2fa90a268ce1e2162c23aaf76e601d22623f6a31b0eeaa764b0/ruff-0.17.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:39c5b55c960515cdb32be16e69efb5734c589c06a0eb38a22694524a336b889d", size = 10210904, upload-time = "2026-10-22T18:37:17.985Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7c/b8850a8e4ad29f64a827dd6c6902db4a9aa538fb2b9eef4e26084001a3bb/ruff-0.17.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:92a80d2f44584dabb0189b3a6c393e37e525e04942838482010575cb81945be7", size = 9408439, upload-time = "2026-10-22T18:37:20.964Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b6/6a64fae86ca5e5e29d5adc311e5481de726cedf9aee607435a16c31c442a/ruff-0.17.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7d0906677d89bbf3cf73cde64b5bbe2ab35df3ae43e49ee2627301e91265bac", size = 10047078, upload-time = "2026-10-22T18:37:23.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/19/23d376547715175aa1bd46df48df6aacc7c6439d9daf19cac15975180aa5/ruff-0.17.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cfd783590edf2b77317882631900f6a13ea2e8eacefe6ebf2370d2715299160", size = 9990478, upload-time = "2026-10-22T18:37:26.584Z" }, + { url = "https://files.pythonhosted.org/packages/93/6c/34cda32cbe13adac5c6b291a307a5570d8ebd37c0c86339f0fe628e55632/ruff-0.17.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122258f6ecde0e0dd1bb60b24263c3edde4338caf723fca711cc729f9bfd414a", size = 10673624, upload-time = "2026-10-22T18:37:29.767Z" }, + { url = "https://files.pythonhosted.org/packages/bf/10/5d762a4dbc851d8bb47863cf3a28a61bf31c804d393b70d891fac899f5d1/ruff-0.17.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f8fba429d6ac4128ab29cfb8fa6240d2fa671d04adf7f84d3b22fc7b6b6bea4", size = 11351797, upload-time = "2026-10-22T18:37:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/3f/04/a0e92a81faf5c81c1c98d4374c8401734ccf2b8cca7d0170cb0fec08c6e2/ruff-0.17.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:583bd67bb9e7061dd09bc46a0a884bae2bbd214dbcd48e509813612619683869", size = 10949414, upload-time = "2026-10-22T18:37:35.501Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d8/59894cf630d2d86882457a2a6a0a1533410c5c805bfbac6f6eb4b5aef69d/ruff-0.17.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c60f93fec7142dba5b8f487f94bf2f6231dae24dc11e4c48648347c8b0720004", size = 10528000, upload-time = "2026-10-22T18:37:37.88Z" }, + { url = "https://files.pythonhosted.org/packages/ab/95/d6e30c51e851d37e10762ad91fa1e76ddaad998589b5667962a3cfbc9edd/ruff-0.17.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c555a877f8f41447e318101cfefea9bf12dfa9f82eedf90c376e57e069f917d0", size = 10571779, upload-time = "2026-10-22T18:37:40.444Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5a/a7e1d74fb464250137d3d4eca645d323536533b672004d04ee0e46e0caf3/ruff-0.17.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ba405cbd5ec7bda1f555d5e52be1fd68d02608d4d95d24591bdac92cfacb579f", size = 10051707, upload-time = "2026-10-22T18:37:42.812Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f2/02fca29ca7999380f0350a7212fe755c11ab9c22d61327356cce8decdbc6/ruff-0.17.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8c21b4ab24485358f17291a4f8dc5399b834e963b8efedd920fe4fdb02081bb4", size = 9998227, upload-time = "2026-10-22T18:37:45.224Z" }, + { url = "https://files.pythonhosted.org/packages/d1/00/55aa604a093dfdc1526e570fae0bb7d37a2221c7b4728d5f080b2add16ba/ruff-0.17.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5875a16b0de70b43cb81eabea5ae5e8f5d1b2fd1b5e6ed07f80cfe5019d0bd5a", size = 10377692, upload-time = "2026-10-22T18:37:47.704Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5a/a7a640f47bb906ec8fa233ea84319d503e6dee02b59e3901bd312ebd1f9b/ruff-0.17.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2d086be452eed686aa51271ecf2cbe5037956517cc6ee39a17b4489ac12f8bbe", size = 10790587, upload-time = "2026-10-22T18:37:50.191Z" }, + { url = "https://files.pythonhosted.org/packages/2c/d4/c3b74e1b4808438ea7f14bb914478025e2d51261f60774a54a0fa222ec46/ruff-0.17.2-py3-none-win32.whl", hash = "sha256:6bce7d1f5165701ae18f5df85c65afe78cddfa6c5fb4b9bdb0014d60388f3edd", size = 9870537, upload-time = "2026-10-22T18:37:52.486Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/c7da96c2c96d20704cf9027f7a5121da272a8c05f1d1531ba949e3b3df04/ruff-0.17.2-py3-none-win_amd64.whl", hash = "sha256:1b59d447f197bef9ef48a22ba1a6016137753c6099e9bac9c36eb566e422424c", size = 10282826, upload-time = "2026-10-22T18:37:55.2Z" }, + { url = "https://files.pythonhosted.org/packages/81/60/a3604c79cb02777b6cb84da894104021986f06c4bc41375102cc95049a32/ruff-0.17.2-py3-none-win_arm64.whl", hash = "sha256:195ad918027f817c119310ca89a42d347887dfc28934aaa869325e9c5dc9d962", size = 9827036, upload-time = "2026-10-22T18:37:58.031Z" }, +] + +[[package]] +name = "sqlquack-workspace" +version = "0.0.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "fensu" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "fensu", specifier = ">=0.27.0" }, + { name = "ruff", specifier = ">=0.12.0" }, + { name = "ty", specifier = ">=0.0.1a19" }, +] + +[[package]] +name = "ty" +version = "0.0.91" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/c0/9e26817a5c94d504aab30bffbbf53f836f4763dd0a947089246ff555eebe/ty-0.0.91.tar.gz", hash = "sha256:b9c434cc1e3b106b6b9353b78ef304d5904c82cf440abcafef7134270a22b41a", size = 8035848, upload-time = "2026-10-22T18:24:52.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/ad/0a850c1ca8e6e802e6b0f730f0d59c9769845b2fc956e3acd7f429b665f2/ty-0.0.91-py3-none-linux_armv6l.whl", hash = "sha256:5d39e6e485854838eb7fcdb747495f5b90283d58bce869c17445120fe0c7e40a", size = 14255787, upload-time = "2026-10-22T18:24:11.284Z" }, + { url = "https://files.pythonhosted.org/packages/33/87/9ccfa2d97b8b043e4d6fab10febc50ac7856b02cf7cd64910085db806bbe/ty-0.0.91-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dcf953e25f90fbccf6cc27ef43fc4ef0961c89f54866e32cd1d7f24891b96084", size = 13822267, upload-time = "2026-10-22T18:24:14.31Z" }, + { url = "https://files.pythonhosted.org/packages/13/33/be8f62444c7840d1b0928c6d174540c0fe6b1b193ae9324c8249e8ed52b9/ty-0.0.91-py3-none-macosx_11_0_arm64.whl", hash = "sha256:78e3e66e88a9ae903afe0981ff9bd225d452ade2aeb070715842510dfcbf6de4", size = 12957620, upload-time = "2026-10-22T18:24:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/9f/a742f273c065c4f5a054cc2b44076207136ff1cd346adfb23da38d8894d8/ty-0.0.91-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ecaf87b3d7b508f9ea1ef8715d4cf47942719659028c92fad819d82a82ccfa", size = 13746018, upload-time = "2026-10-22T18:24:19.221Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a0/3eecf8717cbe4e05d6e02b53679aebad79f1bfcff6c9c03d7601cb7df5f9/ty-0.0.91-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0ec8fe4243aab445abece38a2b8e9b3010230d5e6b3f4e15bfa811a914d99e73", size = 13993220, upload-time = "2026-10-22T18:24:21.328Z" }, + { url = "https://files.pythonhosted.org/packages/09/b0/fb3edee2857e97d85b6c735901469fd523cf6792382e4fbbb00b065c186c/ty-0.0.91-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:577701ec2a9cf7a50f6059069399b9437bb8af7a99df24012776e95d8a55831f", size = 14936911, upload-time = "2026-10-22T18:24:24.393Z" }, + { url = "https://files.pythonhosted.org/packages/b7/2c/82f29ee86f4171807af8ab2feaf4e51211143118a3e53e9da2c2e77eca7d/ty-0.0.91-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc683f49197d63ec47b2eaa9763ddb32dafb2a31190d938b9bb8ebecc5b25173", size = 15336384, upload-time = "2026-10-22T18:24:26.472Z" }, + { url = "https://files.pythonhosted.org/packages/0b/85/e2419b19577ead84b8afa9137c12eb4bde19f926423bfe01e615db12c52c/ty-0.0.91-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8bb2784572fa22f77e417844936f6a9f831a6b0ba64749ca1065c407aefcd23", size = 15332319, upload-time = "2026-10-22T18:24:28.979Z" }, + { url = "https://files.pythonhosted.org/packages/83/06/dc28cea0796ee0268bf987d1d6f0f168605bec8898dbdbc7676049d2667b/ty-0.0.91-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f68bb6bfc8171910335a47c7e84070f28bbbc0326bcd6ef08fe3690d0509556b", size = 14600250, upload-time = "2026-10-22T18:24:31.359Z" }, + { url = "https://files.pythonhosted.org/packages/1e/71/83dd28e628698059ba6fe7081acafa7a8f65cfd90fe6573907ca8219d561/ty-0.0.91-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:63f268c80697d614d18e988476d2517177d66ec732ae104d86e57b60f197a416", size = 14812339, upload-time = "2026-10-22T18:24:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/5a/63c0eae2d5b02cef98a9b327b1a16b36287c140b3a7d2306ff7560177f9c/ty-0.0.91-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b0f34630aa92e222d7754d66222d116f027025cb2b9ed66e1b4b73623de74e84", size = 13703060, upload-time = "2026-10-22T18:24:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/fd6d20d00202949d775043c5f70679f38d0e48ea30eeacbffaf6e0a9754e/ty-0.0.91-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e62f8bbc17d481e01cce39bdfbe36beefda0478cb210e8045ef618d3f674a731", size = 14006236, upload-time = "2026-10-22T18:24:38.464Z" }, + { url = "https://files.pythonhosted.org/packages/fa/26/53dddd4f927dfaa73ea14cd71d97b4d92d685e365b5b037bac457e271403/ty-0.0.91-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f8bf4514a383b13d7cf86eddb55f0eb9f356fc7efeae0de714ad4fe5fc54fee8", size = 14356612, upload-time = "2026-10-22T18:24:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/ccebba145dd244b2f6058ce2e4a6ff34dea36d4a5b38dd7a6ae983b65a3b/ty-0.0.91-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a6d93ab988c4c5ed92b52389b888d46ddede78e4646fe28db615bbeff89cdf82", size = 14674843, upload-time = "2026-10-22T18:24:43.325Z" }, + { url = "https://files.pythonhosted.org/packages/08/6f/8a7701adf1f539a893274358f12ba62773d5bb40920bfc53e0d852bfd576/ty-0.0.91-py3-none-win32.whl", hash = "sha256:b9f4bba8486a3d13babc11f8115369d0a0dc8bef127b7cf4f4c5eed9d4dbfa01", size = 13260545, upload-time = "2026-10-22T18:24:45.715Z" }, + { url = "https://files.pythonhosted.org/packages/fc/63/27c8051a72328ee81f47dace5e2c63e7db43bcae935db25a9dbd4d0e59b2/ty-0.0.91-py3-none-win_amd64.whl", hash = "sha256:5ac9eaf938e7fba4dd6c3358f852e6eb3127aa7bde0a76d9807c41f60ad73a29", size = 13759484, upload-time = "2026-10-22T18:24:48.111Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a9/1f6c3694b7b33be4ae9a272f71a99a7dd27ec020c32a22049e9c0e79f576/ty-0.0.91-py3-none-win_arm64.whl", hash = "sha256:35820957639db1fb8c643273670967192c308038836005bdf3bfef111aff64a0", size = 13318343, upload-time = "2026-10-22T18:24:50.116Z" }, +]