diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50aac042e..eabf7f4e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,7 +211,7 @@ jobs: with: workspaces: fuzz shared-key: fuzz - - name: Seed the corpus with the exact check's statements + - name: Seed the corpus with the corpora's statements run: python3 -m scripts.seed_fuzz_corpus --out fuzz/corpus/tolerant - name: Fuzz working-directory: fuzz diff --git a/Cargo.lock b/Cargo.lock index 9f79068a3..ecbaa6013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,36 +1006,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" -[[package]] -name = "polyglot-sql" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e#7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e" -dependencies = [ - "polyglot-sql-ast-derive", - "polyglot-sql-function-catalogs", - "regex", - "serde", - "serde_json", - "stacker", - "thiserror 1.0.69", - "unicode-segmentation", -] - -[[package]] -name = "polyglot-sql-ast-derive" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e#7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "polyglot-sql-function-catalogs" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e#7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e" - [[package]] name = "portable-atomic" version = "1.15.0" @@ -1570,25 +1540,6 @@ dependencies = [ "toml", ] -[[package]] -name = "sqlquack-polyglot-check" -version = "0.5.1" -dependencies = [ - "clap", - "polyglot-sql", - "serde", - "serde_json", - "sqlquack", - "sqlquack-convert", - "sqlquack-diagnostics", - "sqlquack-evidence", - "sqlquack-policy", - "sqlquack-syntax", - "sqlquack-syntax-snowflake", - "toml", - "walkdir", -] - [[package]] name = "sqlquack-python" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 1072b987c..effaadf0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ members = [ "crates/sqlquack-compare", "crates/sqlquack-evidence", "crates/sqlquack", - "tools/sqlquack-polyglot-check", "tools/sqlquack-coverage-check", "crates/sqlquack-syntax-postgres", "crates/sqlquack-lint", @@ -45,17 +44,6 @@ anstream = "1.0" arrow = { version = "60", default-features = false, features = ["ipc"] } clap = { version = "4.5", features = ["derive"] } duckdb = "=1.20000.0" -polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "7d5fe2ccce7b3cbd1413a2823560f8dfc63ecc2e", default-features = false, features = [ - "dialect-snowflake", - "dialect-duckdb", - "dialect-bigquery", - "dialect-databricks", - "dialect-postgresql", - "dialect-sqlite", - "generate", - "semantic", - "stacker", -] } proc-macro2 = "1" quote = "1" rayon = "1" diff --git a/Makefile b/Makefile index 650d15b4a..83c2275b6 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build polyglot-guard engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -116,10 +116,6 @@ evidence-pin-check: clutter-guard: uv run python -m scripts.check_clutter_guard -# Fails when a crate other than the dev-only exact check reaches polyglot at runtime. -polyglot-guard: - uv run python -m scripts.check_polyglot_guard - # Fails when the conversion pipeline crate, which builds without DuckDB's engine, reaches it. engine-free: @tree="$$(cargo tree --locked -p sqlquack-convert -e normal --prefix none)" || exit 1; \ @@ -170,6 +166,15 @@ generate-cases: corpus-rerecord: SQLQUACK_RERECORD=1 cargo test --locked -p sqlquack --lib everyday_corpus +# Checks SQLQuack's parse verdict on every statement of the corpora under corpora/polyglot-check +# against coverage/syntax/corpus-verdicts.json, and fails on any change. +corpus-verdicts: gate-build + $(GATE_DIR)/syntax-coverage corpus-verdicts + +# Records the verdicts corpus-verdicts checks. Run it in the change that moves one, and commit it. +corpus-verdicts-baseline: gate-build + $(GATE_DIR)/syntax-coverage corpus-verdicts --update + # Fails when a generated case file differs from what its generator writes. generated-cases: uv run python -m scripts.generate_cases --check $(GENERATED_CASE_ARGS) @@ -220,7 +225,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts evidence-pin-check clutter-guard polyglot-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. +check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. # Each distribution ships the licence and third-party notices; its copies must match the root files. distribution-notices: @@ -484,14 +489,6 @@ duckshop-compiled: (cd $(DUCKSHOP_DIR)/duckshop-10k && $(abspath $(DUCKSHOP_DIR)/venv/bin/sqb) compile); \ echo "$(SQLBUILD_REF) $(SQLBUILD_VERSION)" > $(DUCKSHOP_DIR)/.ref -# The exact parser check over each compiled DuckShop project in DUCKSHOP_COMPILED. -polyglot-duckshop: - @for compiled in $(DUCKSHOP_COMPILED); do test -d "$$compiled" || \ - { echo "polyglot-duckshop: $$compiled is missing; run make duckshop-compiled" >&2; exit 1; }; done - for compiled in $(DUCKSHOP_COMPILED); do \ - cargo run --locked --release -p sqlquack-polyglot-check -- directory "$$compiled" --dialect snowflake || exit 1; \ - done - # Converting each compiled DuckShop project, with chained models and a diagnostic for each blocked # file, must finish within DUCKSHOP_CENSUS_SECONDS. DuckShop 10K takes about 25 s on a development # machine; 300 s is about twelve times that, room for slower CI runners, so only a pathological @@ -537,7 +534,7 @@ duckshop-measure: duckshop-compiled --sqlquack "$$target/release/sqlquack" --library "$$library" \ --work "$$out/work" --out "$$out/snowflake.json" -# Fuzzes each of FUZZ_TARGETS from the exact check's statements for FUZZ_SECONDS: tolerant parsing, +# Fuzzes each of FUZZ_TARGETS from the corpora's statements for FUZZ_SECONDS: tolerant parsing, # and strict parsing through to rendered diagnostics. fuzz-smoke: uv run python -m scripts.seed_fuzz_corpus --out $(FUZZ_CORPUS) @@ -546,8 +543,8 @@ fuzz-smoke: -max_total_time=$(FUZZ_SECONDS) -max_len=4096 -timeout=10 -rss_limit_mb=2048) || exit 1; \ done -# Everything CI runs: the static gate, the tests, the recorded-case replay, the exact check and the -# timed conversion on DuckShop, and the fuzz smoke. +# Everything CI runs: the static gate, the tests, the recorded-case replay, the timed conversion on +# DuckShop, and the fuzz smoke. # After the static gate, the independent targets run in parallel, each target's output kept # together; any failure fails the gate. The fuzz smoke and then the SQL timing benchmark run alone # afterwards, so each gets the machine to itself. @@ -556,7 +553,7 @@ gate: check-ci $(if $(filter command line,$(origin DUCKSHOP_COMPILED)),,duckshop $(MAKE) fuzz-smoke $(MAKE) sql-benchmark -gate-parallel: test-rust test-python verify-recorded polyglot-duckshop census-duckshop +gate-parallel: test-rust test-python verify-recorded census-duckshop # Example: make census CENSUS_ARGS="path/to/sql --json" census: diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index de5b44d4b..01a5f876e 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -11,7 +11,7 @@ answer from Snowflake itself. sqlglot's expected DuckDB output is kept in the `s field as reference side data only: it is never asserted, does not count toward verification and is not part of the case hash. -The exact-check corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test +The corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test fixtures at v30.14.0 (tests/fixtures/identity.sql, tests/test_parser.py and tests/dialects/), as extracted by polyglot's fixture tooling. polyglot, below, is itself inspired by sqlglot and ships sqlglot's licence. corpora/polyglot-check/sqlglot_postgres.json holds the SQL of @@ -60,7 +60,7 @@ polyglot SQLQuack's tokenizer, parser and syntax tree are a modified fork of polyglot (https://github.com/tobilg/polyglot), as forked at https://github.com/sqlstack-io/polyglot. The derived code is in crates/sqlquack-syntax-snowflake, which now holds most of it, -crates/sqlquack-syntax and crates/sqlquack-derive. The exact-check corpus +crates/sqlquack-syntax and crates/sqlquack-derive. The corpus corpora/polyglot-check/polyglot_tests.json holds the SQL inputs of polyglot's tests. SQLQuack's DuckDB result-type rules, in crates/sqlquack-semantics/src/duckdb/profile/_helpers/typing.rs, are ported from polyglot's engine type rules, and the fixture diff --git a/corpora/README.md b/corpora/README.md index 92bbbee91..bd86e1860 100644 --- a/corpora/README.md +++ b/corpora/README.md @@ -22,6 +22,6 @@ Inputs of checks that never ship. Their outcomes are not recorded here. | Directory | Read by | |---|---| -| `polyglot-check/` | `tools/sqlquack-polyglot-check`, which proves that `sqlquack-syntax` parses exactly as polyglot does, and the fuzz corpus seed (`scripts/fuzz_corpus`). `expected_differences.toml` lists the reviewed differences. The `*.NOTICE` files and `THIRD_PARTY_NOTICES` give the sources' licences. | +| `polyglot-check/` | `syntax-coverage corpus-verdicts` (`make corpus-verdicts`), against SQLQuack's verdicts recorded in `coverage/syntax/corpus-verdicts.json`, and the fuzz corpus seed (`scripts/fuzz_corpus`). The `*.NOTICE` files and `THIRD_PARTY_NOTICES` give the sources' licences. | | `duckdb-core/` | The DuckDB parser's corpus tests in `crates/sqlquack-syntax-duckdb`; see its README. | | `databricks-core/` | The acceptance corpus for the Databricks parser, each statement tagged with the catalogue forms it uses; no test reads it yet. See its README. | diff --git a/corpora/polyglot-check/expected_differences.toml b/corpora/polyglot-check/expected_differences.toml deleted file mode 100644 index d9a7b6d58..000000000 --- a/corpora/polyglot-check/expected_differences.toml +++ /dev/null @@ -1,1647 +0,0 @@ -# The call spellings SQLQuack keeps as written where polyglot lowers them into a cross-dialect -# node, in Snowflake. The exact check (tools/sqlquack-polyglot-check) puts each kept call back -# into polyglot's form by parsing the call's own text with polyglot's lowering; the result must -# equal polyglot's tree, and every (name, node) pair it puts back must be listed here. -# -# Every entry changes spelling only: the kept call is the written name and arguments, covering -# exactly the tokens polyglot's node covers. Where a note says polyglot merges spellings, keeping -# the call is what stops a rule written for one spelling from silently converting another. -# Calls in keyword syntax (EXTRACT(x FROM y), POSITION(a IN b), TRIM(BOTH ...)) stay typed. - -[[spelling]] -name = "ABS" -polyglot = ["abs"] -note = "polyglot lowers it to node:abs" - -[[spelling]] -name = "ACOS" -polyglot = ["acos"] -note = "polyglot lowers it to node:acos" - -[[spelling]] -name = "ADD_MONTHS" -polyglot = ["add_months"] -note = "polyglot lowers it to node:add_months" - -[[spelling]] -name = "ANY_VALUE" -polyglot = ["any_value"] -note = "polyglot lowers it to node:any_value" - -[[spelling]] -name = "APPROX_COUNT_DISTINCT" -polyglot = ["approx_count_distinct"] -note = "polyglot lowers it to node:approx_count_distinct" - -[[spelling]] -name = "ARRAY_AGG" -polyglot = ["array_agg"] -note = "polyglot lowers it to node:array_agg" - -[[spelling]] -name = "ARRAY_APPEND" -polyglot = ["array_append"] -note = "polyglot lowers it to node:array_append" - -[[spelling]] -name = "ARRAY_COMPACT" -polyglot = ["array_compact"] -note = "polyglot lowers it to node:array_compact" - -[[spelling]] -name = "ARRAY_CONTAINS" -polyglot = ["array_contains"] -note = "polyglot lowers it to node:array_contains" - -[[spelling]] -name = "ARRAY_DISTINCT" -polyglot = ["array_distinct"] -note = "polyglot lowers it to node:array_distinct" - -[[spelling]] -name = "ARRAY_EXCEPT" -polyglot = ["array_except"] -note = "polyglot lowers it to node:array_except" - -[[spelling]] -name = "ARRAY_INTERSECT" -polyglot = ["array_intersect"] -note = "polyglot lowers it to node:array_intersect" - -[[spelling]] -name = "ARRAY_LENGTH" -polyglot = ["array_length"] -note = "polyglot lowers it to node:array_length" - -[[spelling]] -name = "ARRAY_POSITION" -polyglot = ["array_position"] -note = "polyglot lowers it to node:array_position" - -[[spelling]] -name = "ARRAY_PREPEND" -polyglot = ["array_prepend"] -note = "polyglot lowers it to node:array_prepend" - -[[spelling]] -name = "ARRAY_REMOVE" -polyglot = ["array_remove"] -note = "polyglot lowers it to node:array_remove" - -[[spelling]] -name = "ARRAY_REVERSE" -polyglot = ["array_reverse"] -note = "polyglot lowers it to node:array_reverse" - -[[spelling]] -name = "ARRAY_SIZE" -polyglot = ["array_size"] -note = "polyglot lowers it to node:array_size" - -[[spelling]] -name = "ASIN" -polyglot = ["asin"] -note = "polyglot lowers it to node:asin" - -[[spelling]] -name = "ATAN" -polyglot = ["atan"] -note = "polyglot lowers it to node:atan" - -[[spelling]] -name = "ATAN2" -polyglot = ["atan2"] -note = "polyglot lowers it to node:atan2" - -[[spelling]] -name = "AVG" -polyglot = ["avg"] -note = "polyglot lowers it to node:avg" - -[[spelling]] -name = "BOOL_AND" -polyglot = ["logical_and"] -note = "polyglot merges BOOL_AND and EVERY" - -[[spelling]] -name = "BOOL_OR" -polyglot = ["logical_or"] -note = "polyglot lowers it to node:logical_or" - -[[spelling]] -name = "CEIL" -polyglot = ["ceil"] -note = "polyglot lowers it to node:ceil" - -[[spelling]] -name = "CEILING" -polyglot = ["ceil"] -note = "polyglot merges CEIL and CEILING" - -[[spelling]] -name = "CHARACTER_LENGTH" -polyglot = ["length"] -note = "polyglot merges LEN, LENGTH, CHAR_LENGTH and CHARACTER_LENGTH" - -[[spelling]] -name = "CHAR_LENGTH" -polyglot = ["length"] -note = "polyglot merges LEN, LENGTH, CHAR_LENGTH and CHARACTER_LENGTH" - -[[spelling]] -name = "COALESCE" -polyglot = ["coalesce"] -note = "polyglot lowers it to node:coalesce" - -[[spelling]] -name = "CONTAINS" -polyglot = ["contains"] -note = "polyglot lowers it to node:contains" - -[[spelling]] -name = "COS" -polyglot = ["cos"] -note = "polyglot lowers it to node:cos" - -[[spelling]] -name = "COUNT" -polyglot = ["count"] -note = "polyglot lowers it to node:count" - -[[spelling]] -name = "COUNTIF" -polyglot = ["count_if"] -note = "polyglot merges COUNTIF and COUNT_IF" - -[[spelling]] -name = "COUNT_IF" -polyglot = ["count_if"] -note = "polyglot lowers it to node:count_if" - -[[spelling]] -name = "CUME_DIST" -polyglot = ["cume_dist"] -note = "polyglot lowers it to node:cume_dist" - -[[spelling]] -name = "CURRENT_SCHEMAS" -polyglot = ["current_schemas"] -note = "polyglot lowers it to node:current_schemas" - -[[spelling]] -name = "DATEDIFF" -polyglot = ["date_diff"] -note = "polyglot merges DATEDIFF, DATE_DIFF, TIMEDIFF and TIMESTAMPDIFF" - -[[spelling]] -name = "DATEPART" -polyglot = ["function"] -note = "polyglot renames it DATE_PART" - -[[spelling]] -name = "DATE_DIFF" -polyglot = ["date_diff"] -note = "polyglot merges DATEDIFF, DATE_DIFF, TIMEDIFF and TIMESTAMPDIFF" - -[[spelling]] -name = "DAY" -polyglot = ["day"] -note = "polyglot lowers it to node:day" - -[[spelling]] -name = "DAYOFMONTH" -polyglot = ["day_of_month"] -note = "polyglot lowers it to node:day_of_month" - -[[spelling]] -name = "DAYOFWEEK" -polyglot = ["day_of_week"] -note = "polyglot lowers it to node:day_of_week" - -[[spelling]] -name = "DAYOFYEAR" -polyglot = ["day_of_year"] -note = "polyglot lowers it to node:day_of_year" - -[[spelling]] -name = "DAY_OF_MONTH" -polyglot = ["day_of_month"] -note = "polyglot lowers it to node:day_of_month" - -[[spelling]] -name = "DAY_OF_WEEK" -polyglot = ["day_of_week"] -note = "polyglot lowers it to node:day_of_week" - -[[spelling]] -name = "DAY_OF_YEAR" -polyglot = ["day_of_year"] -note = "polyglot lowers it to node:day_of_year" - -[[spelling]] -name = "DEGREES" -polyglot = ["degrees"] -note = "polyglot lowers it to node:degrees" - -[[spelling]] -name = "DENSE_RANK" -polyglot = ["dense_rank"] -note = "polyglot lowers it to node:dense_rank" - -[[spelling]] -name = "ELEMENT_AT" -polyglot = ["element_at"] -note = "polyglot lowers it to node:element_at" - -[[spelling]] -name = "ENDSWITH" -polyglot = ["ends_with"] -note = "polyglot lowers it to node:ends_with" - -[[spelling]] -name = "ENDS_WITH" -polyglot = ["ends_with"] -note = "polyglot lowers it to node:ends_with" - -[[spelling]] -name = "EPOCH" -polyglot = ["epoch"] -note = "polyglot lowers it to node:epoch" - -[[spelling]] -name = "EPOCH_MS" -polyglot = ["epoch_ms"] -note = "polyglot lowers it to node:epoch_ms" - -[[spelling]] -name = "EVERY" -polyglot = ["logical_and"] -note = "polyglot merges BOOL_AND and EVERY" - -[[spelling]] -name = "EXP" -polyglot = ["exp"] -note = "polyglot lowers it to node:exp" - -[[spelling]] -name = "EXPLODE" -polyglot = ["explode"] -note = "polyglot lowers it to node:explode" - -[[spelling]] -name = "EXTRACT" -polyglot = ["extract"] -note = "the call form EXTRACT(part, expr) with a comma; polyglot lowers it to node:extract, the node of the keyword form" - -[[spelling]] -name = "FIRST" -polyglot = ["first"] -note = "polyglot lowers it to node:first" - -[[spelling]] -name = "FIRST_VALUE" -polyglot = ["first_value"] -note = "polyglot lowers it to node:first_value" - -[[spelling]] -name = "FLOOR" -polyglot = ["floor"] -note = "polyglot lowers it to node:floor" - -[[spelling]] -name = "GROUP_CONCAT" -polyglot = ["group_concat"] -note = "polyglot lowers it to node:group_concat" - -[[spelling]] -name = "HOUR" -polyglot = ["hour"] -note = "polyglot lowers it to node:hour" - -[[spelling]] -name = "IF" -polyglot = ["if_func"] -note = "polyglot merges IIF, IF and IFF into one node" - -[[spelling]] -name = "IFF" -polyglot = ["if_func"] -note = "polyglot merges IIF, IF and IFF into one node" - -[[spelling]] -name = "IFNULL" -polyglot = ["coalesce"] -note = "polyglot merges it into COALESCE" - -[[spelling]] -name = "IIF" -polyglot = ["if_func"] -note = "polyglot merges IIF, IF and IFF into one node" - -[[spelling]] -name = "INITCAP" -polyglot = ["initcap"] -note = "polyglot lowers it to node:initcap" - -[[spelling]] -name = "JSON_ARRAY_LENGTH" -polyglot = ["json_array_length"] -note = "polyglot lowers it to node:json_array_length" - -[[spelling]] -name = "JSON_EXTRACT" -polyglot = ["json_extract"] -note = "polyglot lowers it to node:json_extract" - -[[spelling]] -name = "JSON_EXTRACT_SCALAR" -polyglot = ["json_extract_scalar"] -note = "polyglot lowers it to node:json_extract_scalar" - -[[spelling]] -name = "JSON_KEYS" -polyglot = ["json_keys"] -note = "polyglot lowers it to node:json_keys" - -[[spelling]] -name = "JSON_OBJECT" -polyglot = ["json_object"] -note = "polyglot lowers it to node:json_object" - -[[spelling]] -name = "LAG" -polyglot = ["lag"] -note = "polyglot lowers it to node:lag" - -[[spelling]] -name = "LAST" -polyglot = ["last"] -note = "MATCH_RECOGNIZE's LAST(expr) in MEASURES and DEFINE; polyglot lowers it to node:last" - -[[spelling]] -name = "LAST_DAY" -polyglot = ["last_day"] -note = "polyglot lowers it to node:last_day" - -[[spelling]] -name = "LAST_VALUE" -polyglot = ["last_value"] -note = "polyglot lowers it to node:last_value" - -[[spelling]] -name = "LCASE" -polyglot = ["lower"] -note = "polyglot merges LCASE and LOWER" - -[[spelling]] -name = "LEAD" -polyglot = ["lead"] -note = "polyglot lowers it to node:lead" - -[[spelling]] -name = "LEN" -polyglot = ["length"] -note = "polyglot merges LEN, LENGTH, CHAR_LENGTH and CHARACTER_LENGTH" - -[[spelling]] -name = "LENGTH" -polyglot = ["length"] -note = "polyglot merges LEN, LENGTH, CHAR_LENGTH and CHARACTER_LENGTH" - -[[spelling]] -name = "LISTAGG" -polyglot = ["list_agg"] -note = "polyglot lowers it to node:list_agg" - -[[spelling]] -name = "LN" -polyglot = ["ln"] -note = "polyglot lowers it to node:ln" - -[[spelling]] -name = "LOCATE" -polyglot = ["str_position"] -note = "polyglot merges POSITION and LOCATE, reordering arguments" - -[[spelling]] -name = "LOG" -polyglot = ["log"] -note = "polyglot lowers it to node:log" - -[[spelling]] -name = "LOWER" -polyglot = ["lower"] -note = "polyglot lowers it to node:lower" - -[[spelling]] -name = "MAP_CONTAINS_KEY" -polyglot = ["map_contains_key"] -note = "polyglot lowers it to node:map_contains_key" - -[[spelling]] -name = "MAP_KEYS" -polyglot = ["map_keys"] -note = "polyglot lowers it to node:map_keys" - -[[spelling]] -name = "MAP_VALUES" -polyglot = ["map_values"] -note = "polyglot lowers it to node:map_values" - -[[spelling]] -name = "MAX" -polyglot = ["max"] -note = "polyglot lowers it to node:max" - -[[spelling]] -name = "MEDIAN" -polyglot = ["median"] -note = "polyglot lowers it to node:median" - -[[spelling]] -name = "MIN" -polyglot = ["min"] -note = "polyglot lowers it to node:min" - -[[spelling]] -name = "MINUTE" -polyglot = ["minute"] -note = "polyglot lowers it to node:minute" - -[[spelling]] -name = "MOD" -polyglot = ["mod_func"] -note = "polyglot lowers it to a modulo node, which prints as an operator" - -[[spelling]] -name = "MODE" -polyglot = ["mode"] -note = "polyglot lowers it to node:mode" - -[[spelling]] -name = "MONTH" -polyglot = ["month"] -note = "polyglot lowers it to node:month" - -[[spelling]] -name = "MONTHS_BETWEEN" -polyglot = ["months_between"] -note = "polyglot lowers it to node:months_between" - -[[spelling]] -name = "NEXT_DAY" -polyglot = ["next_day"] -note = "polyglot lowers it to node:next_day" - -[[spelling]] -name = "NTH_VALUE" -polyglot = ["nth_value"] -note = "polyglot lowers it to node:nth_value" - -[[spelling]] -name = "NTILE" -polyglot = ["n_tile"] -note = "polyglot lowers it to node:n_tile" - -[[spelling]] -name = "NVL" -polyglot = ["nvl"] -note = "polyglot gives it a node shared with other dialects' NVL" - -[[spelling]] -name = "NVL2" -polyglot = ["nvl2"] -note = "polyglot lowers it to node:nvl2" - -[[spelling]] -name = "PARSE_JSON" -polyglot = ["parse_json"] -note = "polyglot lowers it to node:parse_json" - -[[spelling]] -name = "PERCENT_RANK" -polyglot = ["percent_rank"] -note = "polyglot lowers it to node:percent_rank" - -[[spelling]] -name = "PI" -polyglot = ["pi"] -note = "polyglot lowers it to node:pi" - -[[spelling]] -name = "POSITION" -polyglot = ["str_position"] -note = "polyglot merges POSITION and LOCATE, reordering arguments" - -[[spelling]] -name = "POW" -polyglot = ["power"] -note = "polyglot renames it POWER" - -[[spelling]] -name = "QUARTER" -polyglot = ["quarter"] -note = "polyglot lowers it to node:quarter" - -[[spelling]] -name = "RADIANS" -polyglot = ["radians"] -note = "polyglot lowers it to node:radians" - -[[spelling]] -name = "RAND" -polyglot = ["rand"] -note = "polyglot lowers it to node:rand" - -[[spelling]] -name = "RANDOM" -polyglot = ["rand", "random"] -note = "polyglot lowers it to RANDOM or RAND by arity" - -[[spelling]] -name = "RANK" -polyglot = ["rank"] -note = "polyglot lowers it to node:rank" - -[[spelling]] -name = "ROW_NUMBER" -polyglot = ["row_number"] -note = "polyglot lowers it to node:row_number" - -[[spelling]] -name = "SECOND" -polyglot = ["second"] -note = "polyglot lowers it to node:second" - -[[spelling]] -name = "SIN" -polyglot = ["sin"] -note = "polyglot lowers it to node:sin" - -[[spelling]] -name = "SQRT" -polyglot = ["sqrt"] -note = "polyglot lowers it to node:sqrt" - -[[spelling]] -name = "STARTSWITH" -polyglot = ["starts_with"] -note = "polyglot lowers it to node:starts_with" - -[[spelling]] -name = "STARTS_WITH" -polyglot = ["starts_with"] -note = "polyglot lowers it to node:starts_with" - -[[spelling]] -name = "STDDEV" -polyglot = ["stddev"] -note = "polyglot lowers it to node:stddev" - -[[spelling]] -name = "STDDEV_POP" -polyglot = ["stddev_pop"] -note = "polyglot lowers it to node:stddev_pop" - -[[spelling]] -name = "STDDEV_SAMP" -polyglot = ["stddev_samp"] -note = "polyglot lowers it to node:stddev_samp" - -[[spelling]] -name = "STRING_AGG" -polyglot = ["string_agg"] -note = "polyglot lowers it to node:string_agg" - -[[spelling]] -name = "SUBSTR" -polyglot = ["substring"] -note = "polyglot merges SUBSTR and SUBSTRING" - -[[spelling]] -name = "SUBSTRING" -polyglot = ["substring"] -note = "polyglot merges SUBSTR and SUBSTRING" - -[[spelling]] -name = "SUM" -polyglot = ["sum"] -note = "polyglot lowers it to node:sum" - -[[spelling]] -name = "TAN" -polyglot = ["tan"] -note = "polyglot lowers it to node:tan" - -[[spelling]] -name = "TIMEDIFF" -polyglot = ["date_diff"] -note = "polyglot merges DATEDIFF, DATE_DIFF, TIMEDIFF and TIMESTAMPDIFF" - -[[spelling]] -name = "TIMESTAMPDIFF" -polyglot = ["date_diff"] -note = "polyglot merges DATEDIFF, DATE_DIFF, TIMEDIFF and TIMESTAMPDIFF" - -[[spelling]] -name = "TIME_TO_TIME_STR" -polyglot = ["cast"] -note = "polyglot lowers it to a cast to text" - -[[spelling]] -name = "TOTYPENAME" -polyglot = ["typeof"] -note = "polyglot merges TOTYPENAME and TYPEOF" - -[[spelling]] -name = "TO_DECIMAL" -polyglot = ["to_number"] -note = "polyglot merges TO_DECIMAL, TO_NUMBER and TO_NUMERIC" - -[[spelling]] -name = "TO_JSON" -polyglot = ["to_json"] -note = "polyglot lowers it to node:to_json" - -[[spelling]] -name = "TO_NUMBER" -polyglot = ["to_number"] -note = "polyglot merges TO_DECIMAL, TO_NUMBER and TO_NUMERIC" - -[[spelling]] -name = "TO_NUMERIC" -polyglot = ["to_number"] -note = "polyglot merges TO_DECIMAL, TO_NUMBER and TO_NUMERIC" - -[[spelling]] -name = "TRIM" -polyglot = ["trim"] -note = "polyglot lowers it to node:trim" - -[[spelling]] -name = "TRY_TO_DECIMAL" -polyglot = ["to_number"] -note = "polyglot merges TRY_ and non-TRY spellings into one node with a flag" - -[[spelling]] -name = "TRY_TO_NUMBER" -polyglot = ["to_number"] -note = "polyglot merges TRY_ and non-TRY spellings into one node with a flag" - -[[spelling]] -name = "TRY_TO_NUMERIC" -polyglot = ["to_number"] -note = "polyglot merges TRY_ and non-TRY spellings into one node with a flag" - -[[spelling]] -name = "TYPEOF" -polyglot = ["typeof"] -note = "polyglot lowers it to node:typeof" - -[[spelling]] -name = "UCASE" -polyglot = ["upper"] -note = "polyglot merges UCASE and UPPER" - -[[spelling]] -name = "UPPER" -polyglot = ["upper"] -note = "polyglot lowers it to node:upper" - -[[spelling]] -name = "VARIANCE" -polyglot = ["variance"] -note = "polyglot lowers it to node:variance" - -[[spelling]] -name = "VAR_POP" -polyglot = ["var_pop"] -note = "polyglot lowers it to node:var_pop" - -[[spelling]] -name = "VAR_SAMP" -polyglot = ["var_samp"] -note = "polyglot lowers it to node:var_samp" - -[[spelling]] -name = "WEEKOFYEAR" -polyglot = ["week_of_year"] -note = "polyglot lowers it to node:week_of_year" - -[[spelling]] -name = "WEEK_OF_YEAR" -polyglot = ["week_of_year"] -note = "polyglot lowers it to node:week_of_year" - -[[spelling]] -name = "YEAR" -polyglot = ["year"] -note = "polyglot lowers it to node:year" - -# Valid Snowflake polyglot rejects and SQLQuack parses. The check counts an entry as fixed only -# when polyglot's error contains one of `polyglot_errors` and the statement, without whitespace and -# ignoring case, contains `marker`; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins -# SQLQuack's tree for each fix. - -[[fix]] -id = "index_then_key_path" -marker = "]:" -polyglot_errors = ["Invalid expression / Unexpected token", "Expected RParen, got Colon (':')"] -note = "v[3]:a is a key path on an array element; polyglot accepts a path only after a column, path or call" - -[[fix]] -id = "double_colon_object_type" -marker = "::OBJECT(" -polyglot_errors = ["Invalid expression / Unexpected token", "Expected RParen, got LParen ('(')"] -note = "x::OBJECT(a INT) is the structured OBJECT type CAST(x AS OBJECT(a INT)) parses; polyglot's :: types stop at OBJECT" - -[[fix]] -id = "nested_object_type_in_array" -marker = "(OBJECT(" -polyglot_errors = ["Expected RParen, got LParen ('(')"] -note = "x::ARRAY(OBJECT(k INT)) nests a structured OBJECT as CAST(x AS ARRAY(OBJECT(k INT))) does" - -[[fix]] -id = "nested_object_type_in_map" -marker = ",OBJECT(" -polyglot_errors = ["Expected RParen, got LParen ('(')"] -note = "x::MAP(VARCHAR, OBJECT(k INT)) nests a structured OBJECT as the CAST form does" - -[[fix]] -id = "negative_autoincrement_step" -marker = "INCREMENT-" -polyglot_errors = ["Unexpected token: Dash"] -note = "AUTOINCREMENT START INCREMENT takes a negative step, which counts down; polyglot's INCREMENT reads only an unsigned number" - -[[fix]] -id = "parenthesized_typed_lambda" -marker = "varchar)->" -polyglot_errors = ["Expected RParen, got Var ('VARCHAR')"] -note = "(x VARCHAR) -> ... is a lambda with a typed parameter; polyglot reads a parenthesized lambda's parameters as names only" - -[[fix]] -id = "parenthesized_typed_lambda_parameters" -marker = "variant)->" -polyglot_errors = ["Expected RParen, got Var ('NUMBER')"] -note = "(acc NUMBER(18, 2), i VARIANT) -> ... is a lambda with several typed parameters, one parameterised; polyglot reads a parenthesized lambda's parameters as names only" - -[[fix]] -id = "parenthesized_typed_lambda_int" -marker = "int)->" -polyglot_errors = ["Expected RParen, got Var ('INT')"] -note = "(v INT) -> ... is a lambda with an INT parameter, as ACCUMULATE's documented examples write it; polyglot reads a parenthesized lambda's parameters as names only" - -[[fix]] -id = "hash_agg_distinct" -marker = "HASH_AGG(DISTINCT" -polyglot_errors = ["Expected RParen, got Var"] -note = "HASH_AGG(DISTINCT a) is documented and recorded; polyglot reads no DISTINCT in HASH_AGG" - -[[fix]] -id = "object_agg_distinct" -marker = "OBJECT_AGG(DISTINCT" -polyglot_errors = ["Expected RParen, got Var"] -note = "OBJECT_AGG(DISTINCT k, v) is documented and recorded; polyglot reads no DISTINCT in OBJECT_AGG" - -[[fix]] -id = "snowflake_star_ilike" -marker = "*ILIKE'" -polyglot_errors = ["Invalid expression / Unexpected token"] -note = "SELECT * ILIKE 'pattern' is documented and recorded, alone and before REPLACE or RENAME; polyglot reads no ILIKE after a star" - -[[fix]] -id = "snowflake_interval_day_time_cast" -marker = "::INTERVALDAY" -polyglot_errors = ["Invalid expression / Unexpected token"] -note = "x::INTERVAL DAY(2) TO SECOND(2) casts to a day-time interval type with its qualifiers, as the interval data type page documents; polyglot reads no qualifier after INTERVAL in a :: cast" - -[[fix]] -id = "snowflake_interval_year_month_cast" -marker = "::INTERVALYEAR" -polyglot_errors = ["Invalid expression / Unexpected token"] -note = "x::INTERVAL YEAR(2) TO MONTH casts to a year-month interval type with its qualifiers, as the interval data type page documents; polyglot reads no qualifier after INTERVAL in a :: cast" - -[[fix]] -id = "snowflake_ipaddress_literal" -marker = "IPADDRESS'" -polyglot_errors = ["Invalid expression / Unexpected token"] -note = "IPADDRESS '
' is the documented typed literal of the IPADDRESS type and is recorded; polyglot knows no IPADDRESS literal" - -[[fix]] -id = "snowflake_array_spread" -marker = "(**" -polyglot_errors = ["Unexpected token: DStar"] -note = "** spreads an array into an IN list or a function's arguments, as the expansion operator page documents and Snowflake recorded; polyglot reads ** only as a binary operator" - -[[fix]] -id = "snowflake_resample" -marker = "RESAMPLE(USING" -polyglot_errors = ['Expected identifier, got \"Using\"'] -note = "FROM t RESAMPLE(USING col INCREMENT BY step ...) is documented and recorded; polyglot reads RESAMPLE as a table alias" - -[[fix]] -id = "snowflake_bare_trailing_limit" -marker = "1LIMIT" -polyglot_errors = ["Unexpected token: Limit"] -note = "Snowflake accepts a bare trailing LIMIT (recorded as accepted) and reads it as LIMIT NULL; polyglot needs a count" - -# Statements SQLQuack rejects where polyglot parses them or fails with another error. The check counts an entry as rejected only -# for this exact statement in a listed dialect when SQLQuack's error contains `sqlquack_error`; -# crates/sqlquack-syntax-snowflake/src/parser/tests/test_typed_expressions.rs pins the error. - -[[rejection]] -id = "malformed_typed_type_text" -sql = "SELECT __polyglot_typed(amount, 'NOT A TYPE (') FROM orders" -dialects = ["DuckDB"] -sqlquack_error = "Unexpected token after data type" -note = "malformed typed-expression type text errors instead of degrading to an ordinary call" - -[[rejection]] -id = "snowflake_reserved_type_name" -sql = "SELECT __polyglot_typed(amount, 'NOT A TYPE (') FROM orders" -dialects = ["Snowflake"] -sqlquack_error = "Reserved word NOT cannot be an unquoted type name" -note = "the same malformed type text fails earlier in Snowflake, where NOT is reserved and cannot name a type" - -[[rejection]] -id = "snowflake_cte_without_as" -sql = "WITH t (SELECT 1 AS c) SELECT c FROM t" -dialects = ["Snowflake"] -sqlquack_error = "Expected AS before the CTE body" -note = "Snowflake needs AS between a CTE's name and its body: it refuses WITH v1 ( SELECT 1 ) SELECT 1 with syntax error 1003 (case sf.query.with.select.k8b29420d51a4)" - -[[rejection]] -id = "snowflake_cte_without_as_prose" -sql = " with the correct AST shape." -dialects = ["Snowflake"] -sqlquack_error = "Expected AS before the CTE body" -note = "prose read as a CTE without AS; polyglot fails here too, with another error" - -[[rejection]] -id = "snowflake_copy_prose_into_with" -sql = "COPY INTO with ? source failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a table name" -note = "an assertion message read as a COPY; the COPY grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_copy_prose_on_error" -sql = "COPY ON_ERROR=? failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected INTO after COPY" -note = "an assertion message read as a COPY; the COPY grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_copy_prose_reversed" -sql = "COPY reversed param order failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected INTO after COPY" -note = "an assertion message read as a COPY; the COPY grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_copy_prose_minimal" -sql = "COPY minimal with ? source failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected INTO after COPY" -note = "an assertion message read as a COPY; the COPY grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_copy_prose_roundtrip" -sql = "COPY roundtrip failed to re-parse: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected INTO after COPY" -note = "an assertion message read as a COPY; the COPY grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_reserved_cte_name" -sql = "WITH all AS (SELECT 1 AS count) SELECT all.count FROM all" -dialects = ["Snowflake"] -sqlquack_error = "Reserved word ALL cannot be an unquoted object name" -note = "ALL is a Snowflake reserved word, so it cannot name a CTE unquoted; Snowflake refuses WITH SELECT AS ( SELECT 1 ) SELECT 1 with syntax error 1003" - -[[rejection]] -id = "snowflake_reserved_truncate_target" -sql = "TRUNCATE TABLE table" -dialects = ["Snowflake"] -sqlquack_error = "Reserved word TABLE cannot be an unquoted object name" -note = "TABLE is a Snowflake reserved word, so it cannot name a table unquoted" - -[[rejection]] -id = "snowflake_copy_unload_reserved_table_credentials" -sql = "COPY INTO 's3://example/data.csv'\n FROM EXTRA.EXAMPLE.TABLE\n CREDENTIALS = ()\n FILE_FORMAT = (TYPE = CSV COMPRESSION = NONE NULL_IF = ('') FIELD_OPTIONALLY_ENCLOSED_BY = '\"')\n HEADER = TRUE\n OVERWRITE = TRUE\n SINGLE = TRUE\n " -dialects = ["Snowflake"] -sqlquack_error = "Expected a table name" -note = "TABLE is a Snowflake reserved word, so it cannot be the unquoted last part of the unloaded table's name either; Snowflake refuses d1.s1.SELECT as an object name with syntax error 1003 (DEFINE STREAM d1.s1.v1 ON TABLE d1.s1.SELECT and DEFINE SEQUENCE d1.s1.SELECT)" - -[[rejection]] -id = "snowflake_copy_unload_reserved_table_credentials_flat" -sql = "COPY INTO 's3://example/data.csv'\nFROM EXTRA.EXAMPLE.TABLE\nCREDENTIALS = ()\nFILE_FORMAT = (TYPE=CSV COMPRESSION=NONE NULL_IF=(\n ''\n) FIELD_OPTIONALLY_ENCLOSED_BY='\"')\nHEADER = TRUE\nOVERWRITE = TRUE\nSINGLE = TRUE" -dialects = ["Snowflake"] -sqlquack_error = "Expected a table name" -note = "TABLE is a Snowflake reserved word, so it cannot be the unquoted last part of the unloaded table's name either; Snowflake refuses d1.s1.SELECT as an object name with syntax error 1003 (DEFINE STREAM d1.s1.v1 ON TABLE d1.s1.SELECT and DEFINE SEQUENCE d1.s1.SELECT)" - -[[rejection]] -id = "snowflake_copy_unload_reserved_table_integration" -sql = "COPY INTO 's3://example/data.csv'\n FROM EXTRA.EXAMPLE.TABLE\n STORAGE_INTEGRATION = S3_INTEGRATION\n FILE_FORMAT = (TYPE=CSV COMPRESSION=NONE NULL_IF=('') FIELD_OPTIONALLY_ENCLOSED_BY='\"')\n HEADER = TRUE\n OVERWRITE = TRUE\n SINGLE = TRUE\n " -dialects = ["Snowflake"] -sqlquack_error = "Expected a table name" -note = "TABLE is a Snowflake reserved word, so it cannot be the unquoted last part of the unloaded table's name either; Snowflake refuses d1.s1.SELECT as an object name with syntax error 1003 (DEFINE STREAM d1.s1.v1 ON TABLE d1.s1.SELECT and DEFINE SEQUENCE d1.s1.SELECT)" - -[[rejection]] -id = "snowflake_copy_unload_reserved_table_integration_flat" -sql = "COPY INTO 's3://example/data.csv' FROM EXTRA.EXAMPLE.TABLE STORAGE_INTEGRATION = S3_INTEGRATION FILE_FORMAT = (TYPE=CSV COMPRESSION=NONE NULL_IF=('') FIELD_OPTIONALLY_ENCLOSED_BY='\"') HEADER = TRUE OVERWRITE = TRUE SINGLE = TRUE" -dialects = ["Snowflake"] -sqlquack_error = "Expected a table name" -note = "TABLE is a Snowflake reserved word, so it cannot be the unquoted last part of the unloaded table's name either; Snowflake refuses d1.s1.SELECT as an object name with syntax error 1003 (DEFINE STREAM d1.s1.v1 ON TABLE d1.s1.SELECT and DEFINE SEQUENCE d1.s1.SELECT)" - -[[rejection]] -id = "snowflake_empty_call_argument" -sql = "SELECT ROUND(SCALE => 1, EXPR => 2.25, , ROUNDING_MODE => 'HALF_TO_EVEN') AS value" -dialects = ["Snowflake"] -sqlquack_error = "Expected a function argument between commas" -note = "Snowflake refuses an empty argument between commas with syntax error 1003; polyglot drops it" - -[[rejection]] -id = "snowflake_bare_values_cte" -sql = "WITH RECURSIVE T(n) AS (VALUES (1) UNION ALL SELECT n + 1 FROM t WHERE n < 100) SELECT SUM(n) FROM t" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake does not accept a bare VALUES list as the body of CTE 'T'" -note = "Snowflake takes VALUES only in FROM; polyglot added this rule after SQLQuack's fork (rev 8767d68), with the same message" - -[[rejection]] -id = "snowflake_table_unset_projection_policy" -sql = "ALTER TABLE t UNSET PROJECTION POLICY" -dialects = ["Snowflake"] -sqlquack_error = "Expected the end of the statement" -note = "Snowflake refuses UNSET PROJECTION POLICY on a table, which takes it only on a column (recorded errno 1003)" - -[[rejection]] -id = "snowflake_put_prose_with_quoted_file_uri" -sql = "PUT with quoted file URI failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_placeholder" -sql = "PUT with ? placeholder failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected an option after the location" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_quoted_stage" -sql = "PUT with quoted stage failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_minimal_with" -sql = "PUT minimal with ? failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_roundtrip" -sql = "PUT roundtrip failed to re-parse: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_dollar_quoted_stage_target" -sql = "PUT with dollar-quoted stage target failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_uuid_name_path" -sql = "PUT with @~/UUID/name path failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected an option after the location" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_uuid_compressupload_path" -sql = "PUT with @~/UUID/compressUpload path failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected an option after the location" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_stage_uuid_auto_compress_false" -sql = "PUT with @stage/UUID AUTO_COMPRESS=FALSE failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected an option after the location" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_stage_uuid_auto_compress_true" -sql = "PUT with @stage/UUID AUTO_COMPRESS=TRUE failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected an option after the location" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_put_prose_with_long_path_stage_uuid" -sql = "PUT with long path + @stage/UUID failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Expected a file:// source and then the stage" -note = "an assertion message read as a PUT; the PUT grammar check refuses it before polyglot's parser fails on it with another error" - -[[rejection]] -id = "snowflake_empty_select_list" -sql = "SELECT" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "Snowflake rejects a SELECT with no items (recorded errno 1003); polyglot parses it with an empty expression list" - -[[rejection]] -id = "snowflake_empty_select_list_lower_case" -sql = "select" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "the lower-case spelling of the empty SELECT above" - -[[rejection]] -id = "snowflake_empty_select_list_trailing_space" -sql = "SELECT " -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "the empty SELECT above with a trailing space" - -[[rejection]] -id = "snowflake_reserved_is_alias" -sql = "SELECT 1 is" -dialects = ["Snowflake"] -sqlquack_error = "Invalid expression / Unexpected token" -note = "IS is fully reserved in Snowflake, so it is no implicit alias; Snowflake rejects a reserved implicit alias (recorded errno 1003)" - -[[rejection]] -id = "snowflake_empty_select_list_before_from" -sql = "SELECT FROM users" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "SELECT FROM t has no select item; Snowflake rejects it (recorded errno 1003)" - -[[rejection]] -id = "snowflake_empty_select_list_keyword_qualifier" -sql = "SELECT from.x, date.x FROM orders" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "FROM is fully reserved in Snowflake, so this SELECT has no item; polyglot fails later, at the dot" - -[[rejection]] -id = "snowflake_empty_select_list_trailing_order_by" -sql = "SELECT FROM x ORDER BY" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "an empty SELECT with a dangling ORDER BY; SQLQuack reports the missing item, polyglot the dangling BY" - -[[rejection]] -id = "snowflake_empty_select_list_stage_text" -sql = "SELECT FROM dollar-quoted stage path failed: {:?}" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake requires at least one select item" -note = "a test's message text, not SQL; SQLQuack reports the missing select item, polyglot the dash" - -[[rejection]] -id = "snowflake_reserved_distinct_alias" -sql = "SELECT (a OR b) IS {}DISTINCT FROM (c AND d)" -dialects = ["Snowflake"] -sqlquack_error = "Invalid expression / Unexpected token" -note = "a test's format template, not SQL; both reject it, SQLQuack where IS meets the braces" - -[[rejection]] -id = "snowflake_set_operation_join_side" -sql = "SELECT 1 AS a INNER UNION ALL BY NAME SELECT 2 AS a" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake set operations take no LEFT, RIGHT, FULL, INNER or OUTER" -note = "INNER UNION ... BY NAME is BigQuery's; Snowflake rejects a join side before a set operation (recorded errno 1003)" - -[[rejection]] -id = "snowflake_role_grant_without_role" -sql = "GRANT developer_role TO john;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_without_role_quoted_user" -sql = "GRANT abluva TO \"kk8529p@abluva.com\";" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_without_role_admin_option" -sql = "GRANT admin_role TO mary WITH ADMIN OPTION;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_without_role_lists" -sql = "GRANT role1, role2 TO user1, \"user2@example.com\";" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_without_role_user_list" -sql = "GRANT purchases_reader_role TO george, maria;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_quoted_roles" -sql = "GRANT 'role1', 'role2' TO 'user1'@'localhost', 'user2'@'localhost';" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "MySQL grants quoted roles to user@host accounts; Snowflake writes GRANT ROLE TO { ROLE | USER } , with identifiers, not strings" - -[[rejection]] -id = "snowflake_role_grant_without_role_after_select" -sql = "SELECT set_config('session_authorization', 'abluva', false); GRANT abluva TO \"kk8529p@abluva.com\";" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_grant_without_role_unterminated" -sql = "GRANT abluva TO \"kk8529p@abluva.com\"" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres grants role membership without ROLE; Snowflake writes GRANT ROLE TO { ROLE | USER } and refuses GRANT v1 TO USER v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_role_revoke_without_role" -sql = "REVOKE developer_role FROM john;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres revokes role membership without ROLE; Snowflake writes REVOKE ROLE FROM { ROLE | USER } , as GRANT ROLE, which refuses the form without ROLE with syntax error 1003" - -[[rejection]] -id = "snowflake_role_revoke_admin_option_without_role" -sql = "REVOKE ADMIN OPTION FOR developer_role FROM john CASCADE;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres revokes role membership without ROLE; Snowflake writes REVOKE ROLE FROM { ROLE | USER } , as GRANT ROLE, which refuses the form without ROLE with syntax error 1003" - -[[rejection]] -id = "snowflake_role_revoke_without_role_lists" -sql = "REVOKE role1, role2 FROM user1, \"user2@example.com\" RESTRICT;" -dialects = ["Snowflake"] -sqlquack_error = "Expected a privilege" -note = "Postgres revokes role membership without ROLE; Snowflake writes REVOKE ROLE FROM { ROLE | USER } , as GRANT ROLE, which refuses the form without ROLE with syntax error 1003" - -[[rejection]] -id = "snowflake_describe_bare_name" -sql = "DESCRIBE x" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "Snowflake DESCRIBE names its object kind before the name: it refuses DESC v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_show_profile" -sql = "SHOW PROFILE FOR QUERY 5" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after SHOW" -note = "SHOW PROFILE is MySQL's; PROFILE is no Snowflake SHOW kind" - -[[rejection]] -id = "snowflake_show_groups" -sql = "SHOW GROUPS FOR ROLE" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after SHOW" -note = "GROUPS is no Snowflake SHOW kind; Snowflake shows user groups as SHOW ORGANIZATION USER GROUPS" - -[[rejection]] -id = "snowflake_show_create_table" -sql = "SHOW CREATE TABLE t" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after SHOW" -note = "SHOW CREATE TABLE is MySQL's; Snowflake returns DDL through GET_DDL, and CREATE is no SHOW kind" - -[[rejection]] -id = "snowflake_describe_extended" -sql = "DESCRIBE EXTENDED a.b" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "EXTENDED is Hive's and Spark's; Snowflake DESCRIBE names an object kind, and EXTENDED is none" - -[[rejection]] -id = "snowflake_describe_formatted" -sql = "DESCRIBE FORMATTED a.b" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "FORMATTED is Hive's; Snowflake DESCRIBE names an object kind, and FORMATTED is none" - -[[rejection]] -id = "snowflake_describe_query" -sql = "DESCRIBE SELECT 1" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "Snowflake describes a query's result with DESCRIBE RESULT and a query id, not the query itself" - -[[rejection]] -id = "snowflake_describe_bare_quoted_name" -sql = "DESCRIBE \"history\"" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "a quoted name is still a bare name: Snowflake refuses DESC v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_describe_bare_qualified_name" -sql = "DESCRIBE db.table" -dialects = ["Snowflake"] -sqlquack_error = "Expected an object kind after DESCRIBE" -note = "a qualified name is still a bare name: Snowflake refuses DESC v1 with syntax error 1003" - -[[rejection]] -id = "snowflake_by_name_on_intersect" -sql = "SELECT 1 AS a, 2 AS b INTERSECT DISTINCT BY NAME ON (b, a) SELECT 3 AS b, 4 AS a" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake allows BY NAME only after UNION, without an ON column list" -note = "BY NAME ON (...) after INTERSECT is BigQuery's; Snowflake takes BY NAME only after UNION (recorded errno 1003)" - -[[rejection]] -id = "snowflake_by_name_on_except" -sql = "SELECT 1 AS a, 2 AS b EXCEPT DISTINCT BY NAME ON (b, a) SELECT 3 AS b, 4 AS a" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake allows BY NAME only after UNION, without an ON column list" -note = "BY NAME ON (...) after EXCEPT is BigQuery's; Snowflake takes BY NAME only after UNION (recorded errno 1003)" - -[[rejection]] -id = "snowflake_function_body_number" -sql = "CREATE FUNCTION f(x SMALLDECIMAL ARRAY) RETURNS INT AS 1" -dialects = ["Snowflake"] -sqlquack_error = "Expected the function body as a string" -note = "Snowflake reads a function body only as a string (recorded errno 1003 for CREATE FUNCTION v1 ( ) RETURNS INT AS m1)" - -[[rejection]] -id = "snowflake_function_body_return" -sql = "CREATE FUNCTION a.b(x INT) RETURNS INT AS RETURN x + 1" -dialects = ["Snowflake"] -sqlquack_error = "Expected the function body as a string" -note = "Snowflake has no RETURN body; a function body is a string or $$ block (recorded errno 1003 for CREATE FUNCTION v1 ( ) RETURNS INT AS m1)" - -[[rejection]] -id = "snowflake_function_body_return_contains_sql" -sql = "CREATE FUNCTION a.b(x TEXT) RETURNS TEXT CONTAINS SQL AS RETURN x" -dialects = ["Snowflake"] -sqlquack_error = "Expected the function body as a string" -note = "Snowflake has no RETURN body; a function body is a string or $$ block (recorded errno 1003 for CREATE FUNCTION v1 ( ) RETURNS INT AS m1)" - -[[rejection]] -id = "snowflake_function_body_return_modifies_sql" -sql = "CREATE FUNCTION a.b(x TEXT) RETURNS TEXT LANGUAGE SQL MODIFIES SQL DATA AS RETURN x" -dialects = ["Snowflake"] -sqlquack_error = "Expected the function body as a string" -note = "Snowflake has no RETURN body; a function body is a string or $$ block (recorded errno 1003 for CREATE FUNCTION v1 ( ) RETURNS INT AS m1)" - -[[rejection]] -id = "snowflake_function_body_return_reads_sql" -sql = "CREATE FUNCTION a.b(x TEXT) LANGUAGE SQL READS SQL DATA RETURNS TEXT AS RETURN x" -dialects = ["Snowflake"] -sqlquack_error = "Expected the function body as a string" -note = "Snowflake has no RETURN body; a function body is a string or $$ block (recorded errno 1003 for CREATE FUNCTION v1 ( ) RETURNS INT AS m1)" - -[[rejection]] -id = "snowflake_string_part_extract" -sql = "SELECT EXTRACT('{quoted}' FROM created_at) FROM events" -dialects = ["Snowflake"] -sqlquack_error = "Expected a date or time part keyword before FROM" -note = "Snowflake refuses a string date part before FROM (recorded errno 1003 for EXTRACT ( 'm' FROM 1 ))" - -[[rejection]] -id = "snowflake_string_part_date_part" -sql = "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))" -dialects = ["Snowflake"] -sqlquack_error = "Expected a date or time part keyword before FROM" -note = "the same rule for DATE_PART (recorded errno 1003 for DATE_PART ( 'm' FROM 1 ))" - -[[rejection]] -id = "snowflake_collate_needs_string" -sql = "SELECT 1 || 2 COLLATE NOCASE * 3" -dialects = ["Snowflake"] -sqlquack_error = "Expected a collation specification string" -note = "Snowflake's COLLATE takes a collation specification string such as 'en-ci' (recorded errno 1003 for 'm' COLLATE m1); NOCASE is SQLite's" - -[[rejection]] -id = "snowflake_dot_after_literal" -sql = "SELECT 1..2" -dialects = ["Snowflake"] -sqlquack_error = "A literal has no fields to read with a dot" -note = "both parsers refuse it; polyglot says \"Expected field name after dot\"" - -[[rejection]] -id = "snowflake_frame_bare_preceding" -sql = "SELECT SUM(x) OVER (PARTITION BY a ROWS BETWEEN UNBOUNDED PRECEDING AND PRECEDING)" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake frame bounds need a value or UNBOUNDED before PRECEDING" -note = "Snowflake needs a value or UNBOUNDED before PRECEDING (recorded errno 1003 for PERCENT_RANK ( ) OVER ( ORDER BY 1 RANGE PRECEDING ))" - -[[rejection]] -id = "snowflake_frame_value_without_direction" -sql = "SELECT SUM(x) OVER (PARTITION BY a RANGE BETWEEN 1 AND 3)" -dialects = ["Snowflake"] -sqlquack_error = "Expected PRECEDING or FOLLOWING after a frame bound value" -note = "Snowflake needs PRECEDING or FOLLOWING after a frame bound value, as its window frame grammar documents" - -[[rejection]] -id = "snowflake_frame_value_without_direction_end" -sql = "SELECT SUM(x) OVER (PARTITION BY a RANGE BETWEEN 1 FOLLOWING AND 3)" -dialects = ["Snowflake"] -sqlquack_error = "Expected PRECEDING or FOLLOWING after a frame bound value" -note = "the same rule at the frame's end" - -# Statements both parsers parse where polyglot drops the quoting of a name SQLQuack keeps quoted. -# The check counts an entry as kept quoting only when the statement, without whitespace and ignoring -# case, contains `marker`, and the trees differ in nothing but identifiers whose `quoted` is false in -# polyglot's tree and true in SQLQuack's; tools/sqlquack-polyglot-check/tests/runs/test_kept_quoting.rs -# pins both. - -[[kept_quoting]] -id = "star_exclude_quoted_name" -marker = "EXCLUDE(\"" -note = "SELECT * EXCLUDE (\"note\") excludes the case-sensitive column note, not NOTE; polyglot drops the quotes of a star's EXCLUDE names" - -[[kept_quoting]] -id = "comment_on_quoted_name" -marker = "COMMENTON" -note = "COMMENT ON TABLE \"v1\" names the case-sensitive object v1, not V1; polyglot drops the quotes of a COMMENT ON name; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -# Wrapper nodes SQLQuack keeps where polyglot drops them. The check counts an entry only when the -# statement, without whitespace and ignoring case, contains `marker`, and both trees are equal once -# every `{ "": { "this": … } }` in SQLQuack's tree is replaced by its operand. - -[[kept_node]] -id = "unary_plus" -marker = "+" -node = "pos" -note = "SELECT + 1 keeps its unary plus, a documented Snowflake operator; polyglot drops it and keeps only the operand; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -# Fields SQLQuack keeps as written where polyglot drops or infers them. The check counts an entry -# only when the statement, without whitespace and ignoring case, contains `marker`, the trees differ, -# and both are equal once every `` key is removed from each. - -[[kept_field]] -id = "snowflake_secondary_role_list" -marker = "SECONDARYROLES" -field = "roles" -note = "USE SECONDARY ROLES a, b, c lists its roles one by one with their quoting; polyglot joins them into one unquoted name; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -[[kept_field]] -id = "snowflake_show_unnamed_container" -marker = "SHOW" -field = "scope_kind" -note = "SHOW ... IN a.b names no container kind, so SQLQuack records none; polyglot infers SCHEMA from the two-part name, which makes IN a.b and IN SCHEMA a.b one tree" - -[[kept_field]] -id = "snowflake_timestamp_without_time_zone" -marker = "WITHOUTTIMEZONE" -field = "to" -note = "Snowflake documents TIMESTAMP WITHOUT TIME ZONE as a synonym of TIMESTAMP_NTZ, so SQLQuack's cast target is TIMESTAMPNTZ, as for TIMESTAMP_NTZ; polyglot reads it as bare TIMESTAMP, whose meaning follows TIMESTAMP_TYPE_MAPPING; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -[[kept_field]] -id = "snowflake_clone_time_travel" -marker = "CLONE" -field = "at_clause" -note = "a database or schema clone's AT | BEFORE ( TIMESTAMP | OFFSET | STATEMENT => ) parses into its keyword, kind and expression; polyglot keeps the clause as raw text; crates/sqlquack-syntax-snowflake/tests/parser/test_snowflake_create_clone.rs pins the tree" - -[[kept_field]] -id = "snowflake_table_clone_time_travel" -marker = "CLONE" -field = "clone_at_clause" -note = "a table clone's AT | BEFORE ( TIMESTAMP | OFFSET | STATEMENT => ) parses into its keyword, kind and expression; polyglot keeps the clause as raw text; crates/sqlquack-syntax-snowflake/tests/parser/test_snowflake_create_clone.rs pins the tree" - -# Clauses whose ALL SQLQuack keeps apart from its keys, where polyglot reads ALL as a bare-word key. -# The check counts an entry only when the statement, without whitespace and ignoring case, contains -# `marker`, the trees differ, and both are equal once each `` in SQLQuack's tree that has no -# keys and an `all` is rewritten as polyglot's one key `{ "var": { "this": "ALL" } }` with the same -# DESC and NULLS options. The run counts them with the kept fields. - -[[kept_all]] -id = "snowflake_order_by_all" -marker = "ORDERBYALL" -clause = "order_by" -note = "ORDER BY ALL sorts by every SELECT-list column in order, so SQLQuack records it as an ALL of the clause with its DESC and NULLS options; polyglot reads ALL as a key named ALL, which a column called ALL would also give; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -# Statements polyglot keeps as one opaque command where SQLQuack parses them into structured nodes. -# The check counts an entry only when polyglot's tree is a single command, SQLQuack's tree starts -# with `sqlquack_node`, and the statement, without whitespace and ignoring case, contains `marker`. - -[[structured]] -id = "snowflake_storage_integration_properties" -marker = "CREATESTORAGEINTEGRATION" -sqlquack_node = "create_object" -note = "a Snowflake CREATE STORAGE INTEGRATION parses into its name and its property list, each property's kind and value; polyglot keeps the statement as raw SQL; crates/sqlquack-syntax-snowflake/tests/parser/test_snowflake_create_properties.rs pins the tree" - -[[structured]] -id = "snowflake_scripting_block" -marker = "BEGIN" -sqlquack_node = "scripting_block" -note = "a Snowflake Scripting block parses into its declarations, statements and handlers; crates/sqlquack-syntax-snowflake/tests/parser/test_scripting.rs pins the tree" - -[[structured]] -id = "snowflake_create_stage" -marker = "STAGE" -sqlquack_node = "create_object" -note = "a Snowflake CREATE STAGE parses into a STAGE object with its properties, such as FILE_FORMAT = ( FORMAT_NAME = ... ); polyglot keeps it as raw text; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -[[structured]] -id = "snowflake_create_file_format" -marker = "FILEFORMAT" -sqlquack_node = "create_object" -note = "a Snowflake CREATE FILE FORMAT parses into a FILE FORMAT object with its modifier and options; polyglot keeps it as raw text; crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs pins the tree" - -[[rejection]] -id = "snowflake_stage_reserved_name" -sql = "SELECT @select" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage name after @" -note = "Snowflake reads @ as a stage reference and refuses a bare reserved word as the stage name (recorded syntax_error)" - -[[rejection]] -id = "snowflake_stage_quoted_blank_name" -sql = "SELECT @`a b`" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage name after @" -note = "Snowflake refuses a stage reference whose quoted name holds a blank (recorded syntax_error); backticks are no Snowflake quote either" - -[[rejection]] -id = "snowflake_stage_string_name" -sql = "SELECT @JOIN, @'foo'" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage name after @" -note = "Snowflake reads @ as a stage reference and refuses a string as the stage name (recorded syntax_error for @'v1')" - -[[rejection]] -id = "snowflake_bare_interval_type_double_colon" -sql = "SELECT interval '00:00:01'::interval AS foo" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_bare_interval_type_cast" -sql = "SELECT CAST(INTERVAL '00:00:01' AS INTERVAL) AS foo" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_bare_interval_type_double_colon_window" -sql = "SELECT ROW_NUMBER() OVER(PARTITION BY event_time + interval '00:00:01'::interval) AS foo FROM t" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_bare_interval_type_cast_window" -sql = "SELECT ROW_NUMBER() OVER (PARTITION BY event_time + CAST(INTERVAL '00:00:01' AS INTERVAL)) AS foo FROM t" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_bare_interval_type_cast_string" -sql = "CAST('1 DAY' AS INTERVAL)" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_type_bare_interval_cast" -sql = "SELECT CAST(x AS INTERVAL)" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_type_bare_interval_try_cast" -sql = "SELECT TRY_CAST(x AS INTERVAL)" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_type_bare_interval_double_colon" -sql = "SELECT x::INTERVAL" -dialects = ["Snowflake"] -sqlquack_error = "Expected an interval field such as DAY or YEAR TO MONTH" -note = "Snowflake's INTERVAL type needs a field (recorded errno 1003 for CAST ( NULL AS INTERVAL ))" - -[[rejection]] -id = "snowflake_list_function_call" -sql = "LIST(a)" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage after the command" -note = "Snowflake's LIST is a stage command; DuckDB's LIST(a) aggregate is no Snowflake statement, and LIST needs @stage first" - -[[rejection]] -id = "snowflake_list_function_order_by" -sql = "LIST(a ORDER BY b)" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage after the command" -note = "Snowflake's LIST is a stage command; DuckDB's ordered LIST aggregate is no Snowflake statement, and LIST needs @stage first" - -[[rejection]] -id = "snowflake_list_without_stage" -sql = "LIST" -dialects = ["Snowflake"] -sqlquack_error = "Expected a stage after the command" -note = "Snowflake's LIST needs the @stage it lists" - -[[rejection]] -id = "snowflake_unpivot_reserved_value_column" -sql = "SELECT * FROM q UNPIVOT(values FOR quarter IN (b, c))" -dialects = ["Snowflake"] -sqlquack_error = "Expected a value column name in UNPIVOT" -note = "VALUES is a Snowflake reserved word, so it cannot name UNPIVOT's value column unquoted" - - - - - - - - - - - - -[[rejection]] -id = "snowflake_show_grants_for" -sql = "SHOW GRANTS FOR foo" -dialects = ["Snowflake"] -sqlquack_error = "Invalid expression / Unexpected token" -note = "SHOW GRANTS FOR is MySQL's; Snowflake's SHOW GRANTS takes ON, TO, OF or IN, so FOR is refused" - -[[rejection]] -id = "snowflake_int_precision_default_uuid" -sql = "CREATE TABLE z (a INT(11) DEFAULT UUID())" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake's INT and INTEGER take no precision" -note = "Snowflake's INT is NUMBER(38, 0) with no precision: it refuses CREATE TABLE v1 ( v1 INT ( 1 ) ) ; with syntax error 1003" - -[[rejection]] -id = "snowflake_int_precision_default_null_comment" -sql = "CREATE TABLE z (a INT(11) DEFAULT NULL COMMENT '客户id')" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake's INT and INTEGER take no precision" -note = "Snowflake's INT is NUMBER(38, 0) with no precision: it refuses CREATE TABLE v1 ( v1 INT ( 1 ) ) ; with syntax error 1003" - -[[rejection]] -id = "snowflake_int_precision_not_null_default" -sql = "CREATE TABLE z (a INT(11) NOT NULL DEFAULT 1)" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake's INT and INTEGER take no precision" -note = "Snowflake's INT is NUMBER(38, 0) with no precision: it refuses CREATE TABLE v1 ( v1 INT ( 1 ) ) ; with syntax error 1003" - -[[rejection]] -id = "snowflake_int_precision_not_null_default_negative" -sql = "CREATE TABLE z (a INT(11) NOT NULL DEFAULT -1)" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake's INT and INTEGER take no precision" -note = "Snowflake's INT is NUMBER(38, 0) with no precision: it refuses CREATE TABLE v1 ( v1 INT ( 1 ) ) ; with syntax error 1003" - -[[rejection]] -id = "snowflake_int_precision_collate_auto_increment" -sql = "CREATE TABLE z (a INT(11) NOT NULL COLLATE utf8_bin AUTO_INCREMENT)" -dialects = ["Snowflake"] -sqlquack_error = "Snowflake's INT and INTEGER take no precision" -note = "Snowflake's INT is NUMBER(38, 0) with no precision: it refuses CREATE TABLE v1 ( v1 INT ( 1 ) ) ; with syntax error 1003" - -[[rejection]] -id = "snowflake_view_without_query_columns" -sql = "CREATE VIEW z (a, b)" -dialects = ["Snowflake"] -sqlquack_error = "Expected AS and the view's query" -note = "a Snowflake view needs AS and its query; this is a column list with no query" - -[[rejection]] -id = "snowflake_view_without_query_column_comments" -sql = "CREATE VIEW z (a, b COMMENT 'b', c COMMENT 'c')" -dialects = ["Snowflake"] -sqlquack_error = "Expected AS and the view's query" -note = "a Snowflake view needs AS and its query; this is a commented column list with no query" - -[[rejection]] -id = "snowflake_merge_insert_star" -sql = "\n MERGE INTO target USING source ON target.id = source.id\n WHEN MATCHED THEN UPDATE *\n WHEN NOT MATCHED THEN INSERT *\n " -dialects = ["Snowflake"] -sqlquack_error = "Expected VALUES in MERGE INSERT" -note = "Snowflake's MERGE INSERT takes VALUES ( ... ) and refuses an INSERT without VALUES with syntax error 1003; INSERT * is Databricks'" - -[[rejection]] -id = "snowflake_view_row_access_policy_without_on" -sql = "CREATE VIEW v WITH ROW ACCESS POLICY #unknown_policy AS SELECT 1" -dialects = ["Snowflake"] -sqlquack_error = "Expected On" -note = "a Snowflake view's ROW ACCESS POLICY reads ON ( , ... ); this one has no ON list and names the policy with sqlglot's #unknown placeholder" diff --git a/coverage/syntax/corpus-verdicts.json b/coverage/syntax/corpus-verdicts.json new file mode 100644 index 000000000..537299be3 --- /dev/null +++ b/coverage/syntax/corpus-verdicts.json @@ -0,0 +1,6338 @@ +{ + "about": "SQLQuack's parse verdict on every statement of the corpora under corpora/polyglot-check: per file and dialect, how many statements it holds and the positions of those SQLQuack refuses. Any change, and any statement whose lossless tokens do not rebuild it or whose located parse differs from its faithful parse, fails `make corpus-verdicts`; record a reviewed change with `make corpus-verdicts-baseline`.", + "corpora": { + "polyglot_tests.json": { + "duckdb": { + "statements": 1425, + "refused": [ + 297, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 322, + 350, + 359, + 361, + 371, + 378, + 401, + 402, + 414, + 422, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 442, + 444, + 449, + 451, + 471, + 492, + 498, + 561, + 570, + 573, + 579, + 610, + 623, + 624, + 627, + 629, + 630, + 636, + 637, + 638, + 639, + 640, + 647, + 658, + 659, + 660, + 661, + 667, + 672, + 682, + 700, + 702, + 703, + 704, + 705, + 706, + 707, + 714, + 715, + 716, + 717, + 718, + 719, + 791, + 792, + 793, + 841, + 842, + 843, + 845, + 846, + 872, + 874, + 881, + 882, + 883, + 909, + 910, + 911, + 912, + 913, + 914, + 916, + 917, + 918, + 919, + 978, + 981, + 982, + 985, + 986, + 987, + 988, + 989, + 990, + 991, + 992, + 993, + 1007, + 1009, + 1015, + 1017, + 1039, + 1056, + 1057, + 1058, + 1059, + 1060, + 1061, + 1063, + 1065, + 1066, + 1067, + 1069, + 1071, + 1072, + 1073, + 1074, + 1075, + 1076, + 1077, + 1078, + 1079, + 1080, + 1081, + 1082, + 1083, + 1084, + 1085, + 1086, + 1088, + 1103, + 1104, + 1105, + 1107, + 1108, + 1109, + 1111, + 1112, + 1113, + 1114, + 1120, + 1130, + 1131, + 1132, + 1133, + 1135, + 1136, + 1137, + 1138, + 1139, + 1140, + 1141, + 1142, + 1143, + 1144, + 1145, + 1186, + 1193, + 1194, + 1197, + 1198, + 1199, + 1200, + 1201, + 1212, + 1214, + 1216, + 1217, + 1218, + 1219, + 1223, + 1261, + 1263, + 1265, + 1267, + 1269, + 1271, + 1272, + 1273, + 1274, + 1275, + 1276, + 1278, + 1279, + 1281, + 1285, + 1289, + 1291, + 1293, + 1295, + 1297, + 1299, + 1301, + 1303, + 1305, + 1312, + 1314, + 1315, + 1316, + 1317, + 1318, + 1321, + 1326, + 1327, + 1328, + 1329, + 1330, + 1331, + 1333, + 1334, + 1335, + 1336, + 1347, + 1349, + 1355, + 1356, + 1361, + 1362, + 1363, + 1374, + 1375, + 1376, + 1377, + 1382, + 1389, + 1390, + 1391, + 1392, + 1393, + 1394, + 1400, + 1401, + 1402, + 1405, + 1406, + 1407, + 1408, + 1409, + 1410, + 1411, + 1412, + 1413, + 1414, + 1415, + 1416, + 1417, + 1418, + 1419, + 1420, + 1421, + 1422 + ] + }, + "snowflake": { + "statements": 1425, + "refused": [ + 166, + 270, + 294, + 297, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 322, + 347, + 348, + 350, + 353, + 354, + 355, + 357, + 366, + 370, + 371, + 377, + 378, + 382, + 384, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 399, + 400, + 401, + 402, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 426, + 430, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 449, + 451, + 457, + 458, + 459, + 460, + 461, + 466, + 467, + 468, + 471, + 492, + 493, + 501, + 508, + 511, + 512, + 513, + 515, + 516, + 517, + 561, + 573, + 577, + 578, + 579, + 580, + 609, + 610, + 623, + 624, + 625, + 627, + 629, + 630, + 634, + 635, + 636, + 637, + 638, + 639, + 640, + 647, + 649, + 650, + 651, + 652, + 653, + 654, + 655, + 656, + 658, + 660, + 661, + 663, + 664, + 665, + 666, + 667, + 672, + 673, + 675, + 676, + 679, + 680, + 682, + 697, + 700, + 701, + 702, + 703, + 704, + 705, + 706, + 707, + 714, + 715, + 716, + 717, + 718, + 719, + 735, + 737, + 738, + 739, + 740, + 773, + 791, + 792, + 793, + 806, + 808, + 822, + 823, + 824, + 825, + 826, + 827, + 841, + 842, + 843, + 844, + 845, + 846, + 847, + 848, + 862, + 863, + 864, + 865, + 870, + 872, + 874, + 878, + 881, + 882, + 883, + 896, + 897, + 898, + 899, + 900, + 901, + 902, + 903, + 904, + 905, + 906, + 909, + 910, + 911, + 912, + 913, + 914, + 915, + 916, + 917, + 918, + 919, + 950, + 956, + 957, + 958, + 959, + 978, + 981, + 982, + 983, + 986, + 987, + 988, + 989, + 990, + 991, + 992, + 993, + 1007, + 1009, + 1015, + 1017, + 1039, + 1056, + 1057, + 1058, + 1059, + 1060, + 1061, + 1062, + 1063, + 1065, + 1066, + 1067, + 1069, + 1071, + 1072, + 1073, + 1074, + 1075, + 1076, + 1077, + 1078, + 1079, + 1080, + 1081, + 1082, + 1083, + 1084, + 1085, + 1086, + 1088, + 1103, + 1104, + 1105, + 1107, + 1108, + 1109, + 1111, + 1112, + 1113, + 1114, + 1120, + 1130, + 1131, + 1132, + 1133, + 1135, + 1136, + 1137, + 1138, + 1139, + 1140, + 1141, + 1142, + 1143, + 1144, + 1145, + 1186, + 1193, + 1194, + 1195, + 1197, + 1198, + 1199, + 1200, + 1201, + 1206, + 1216, + 1217, + 1218, + 1261, + 1263, + 1265, + 1267, + 1269, + 1271, + 1273, + 1276, + 1278, + 1279, + 1285, + 1289, + 1291, + 1293, + 1295, + 1297, + 1299, + 1301, + 1303, + 1305, + 1306, + 1307, + 1312, + 1313, + 1314, + 1315, + 1316, + 1317, + 1318, + 1321, + 1326, + 1327, + 1328, + 1329, + 1330, + 1331, + 1332, + 1333, + 1334, + 1335, + 1336, + 1347, + 1349, + 1355, + 1356, + 1361, + 1362, + 1363, + 1367, + 1368, + 1369, + 1370, + 1371, + 1372, + 1373, + 1374, + 1375, + 1376, + 1377, + 1382, + 1388, + 1389, + 1390, + 1391, + 1392, + 1393, + 1394, + 1400, + 1401, + 1402, + 1404, + 1405, + 1406, + 1407, + 1408, + 1409, + 1410, + 1411, + 1412, + 1413, + 1414, + 1415, + 1416, + 1417, + 1418, + 1419, + 1420, + 1421, + 1422 + ] + } + }, + "postgres_regress.json": { + "postgres": { + "statements": 37500, + "refused": [ + 0, + 22, + 249, + 250, + 251, + 252, + 253, + 254, + 260, + 261, + 262, + 263, + 264, + 266, + 267, + 268, + 269, + 317, + 326, + 431, + 499, + 501, + 502, + 503, + 510, + 643, + 652, + 659, + 670, + 673, + 676, + 679, + 683, + 688, + 693, + 698, + 703, + 706, + 714, + 720, + 727, + 795, + 796, + 797, + 798, + 801, + 802, + 803, + 849, + 850, + 851, + 852, + 853, + 854, + 855, + 861, + 897, + 898, + 899, + 900, + 901, + 902, + 934, + 935, + 967, + 971, + 976, + 977, + 978, + 979, + 980, + 981, + 983, + 986, + 989, + 990, + 1011, + 1012, + 1013, + 1016, + 1018, + 1024, + 1028, + 1029, + 1030, + 1035, + 1036, + 1038, + 1049, + 1058, + 1088, + 1090, + 1092, + 1124, + 1125, + 1129, + 1136, + 1137, + 1138, + 1143, + 1147, + 1173, + 1189, + 1190, + 1194, + 1267, + 1320, + 1321, + 1322, + 1323, + 1477, + 1481, + 1482, + 1483, + 1485, + 1486, + 1515, + 1516, + 1517, + 1521, + 1524, + 1527, + 1545, + 1547, + 1549, + 1551, + 1553, + 1558, + 1572, + 1573, + 1574, + 1599, + 1627, + 1628, + 1631, + 1634, + 1637, + 1641, + 1642, + 1645, + 1646, + 1647, + 1648, + 1649, + 1652, + 1667, + 1669, + 1723, + 1744, + 1757, + 1758, + 1759, + 1760, + 1761, + 1762, + 1763, + 1764, + 1766, + 1767, + 1800, + 1801, + 1817, + 1880, + 1884, + 1898, + 1899, + 1900, + 1904, + 1908, + 1910, + 1914, + 1917, + 1919, + 1920, + 1923, + 1924, + 1932, + 1935, + 1936, + 1945, + 1950, + 1952, + 1956, + 1961, + 1964, + 1965, + 1967, + 1973, + 1975, + 1977, + 1980, + 1983, + 1984, + 1988, + 1991, + 1996, + 2000, + 2001, + 2002, + 2011, + 2013, + 2014, + 2015, + 2020, + 2022, + 2030, + 2031, + 2034, + 2039, + 2042, + 2043, + 2044, + 2046, + 2048, + 2049, + 2051, + 2052, + 2053, + 2055, + 2061, + 2066, + 2068, + 2069, + 2071, + 2072, + 2073, + 2095, + 2098, + 2099, + 2114, + 2116, + 2122, + 2126, + 2129, + 2135, + 2138, + 2140, + 2146, + 2147, + 2148, + 2161, + 2163, + 2169, + 2172, + 2176, + 2181, + 2185, + 2189, + 2196, + 2198, + 2206, + 2221, + 2223, + 2224, + 2225, + 2227, + 2228, + 2229, + 2230, + 2238, + 2239, + 2240, + 2246, + 2261, + 2262, + 2263, + 2272, + 2273, + 2274, + 2275, + 2283, + 2284, + 2285, + 2286, + 2287, + 2291, + 2292, + 2293, + 2294, + 2295, + 2296, + 2297, + 2298, + 2299, + 2300, + 2301, + 2302, + 2304, + 2353, + 2433, + 2434, + 2437, + 2438, + 2439, + 2440, + 2441, + 2513, + 2514, + 2619, + 2620, + 2664, + 2673, + 2689, + 2690, + 2691, + 2696, + 2756, + 2757, + 2758, + 2760, + 2761, + 2762, + 2763, + 2776, + 2777, + 2778, + 2779, + 2780, + 2781, + 2782, + 2808, + 2809, + 2810, + 2811, + 2812, + 2813, + 2814, + 2815, + 2816, + 2817, + 2818, + 2819, + 2820, + 2821, + 2822, + 2823, + 2824, + 2825, + 2826, + 2827, + 2828, + 2829, + 2830, + 2834, + 2835, + 2836, + 2837, + 2838, + 2839, + 2840, + 2841, + 2842, + 2849, + 2850, + 2851, + 2853, + 2854, + 2855, + 2856, + 2857, + 2859, + 2860, + 2861, + 2862, + 2863, + 2867, + 2868, + 2928, + 2929, + 2930, + 2947, + 2948, + 2949, + 2950, + 2951, + 2952, + 2953, + 2954, + 2959, + 2960, + 2980, + 2981, + 2982, + 2983, + 2986, + 3000, + 3001, + 3006, + 3007, + 3020, + 3061, + 3064, + 3066, + 3067, + 3068, + 3072, + 3073, + 3085, + 3100, + 3102, + 3103, + 3109, + 3110, + 3113, + 3114, + 3117, + 3132, + 3188, + 3193, + 3199, + 3211, + 3222, + 3223, + 3224, + 3225, + 3269, + 3283, + 3288, + 3297, + 3299, + 3303, + 3363, + 3454, + 3464, + 3467, + 3472, + 3473, + 3482, + 3483, + 3507, + 3510, + 3515, + 3524, + 3525, + 3529, + 3534, + 3541, + 3543, + 3544, + 3548, + 3553, + 3566, + 3568, + 3576, + 3729, + 3731, + 3739, + 3751, + 3752, + 3753, + 3756, + 3758, + 3763, + 3766, + 3767, + 3770, + 3771, + 3846, + 3848, + 3854, + 3874, + 3876, + 3882, + 3902, + 3904, + 3909, + 4078, + 4081, + 4086, + 4156, + 4158, + 4162, + 4163, + 4164, + 4165, + 4180, + 4206, + 4224, + 4245, + 4248, + 4256, + 4262, + 4263, + 4268, + 4278, + 4279, + 4283, + 4285, + 4286, + 4292, + 4299, + 4302, + 4304, + 4305, + 4310, + 4318, + 4320, + 4330, + 4337, + 4358, + 4387, + 4392, + 4413, + 4453, + 4470, + 4473, + 4474, + 4482, + 4487, + 4492, + 4495, + 4496, + 4498, + 4503, + 4504, + 4508, + 4509, + 4527, + 4528, + 4545, + 4551, + 4552, + 4553, + 4554, + 4555, + 4556, + 4557, + 4558, + 4562, + 4563, + 4566, + 4567, + 4568, + 4578, + 4579, + 4581, + 4582, + 4636, + 4638, + 4650, + 4651, + 4655, + 4657, + 4659, + 4664, + 4675, + 4703, + 4721, + 4733, + 4735, + 4737, + 4739, + 4741, + 4743, + 4751, + 4754, + 4755, + 4756, + 4757, + 4758, + 4759, + 4760, + 4762, + 4772, + 4773, + 4774, + 4777, + 4778, + 4786, + 4787, + 4810, + 4813, + 4816, + 4819, + 4820, + 4831, + 4832, + 4858, + 4872, + 4873, + 4890, + 4891, + 4906, + 4907, + 4908, + 4909, + 4910, + 4911, + 4913, + 4914, + 4917, + 4920, + 4927, + 4933, + 4955, + 4963, + 4965, + 4967, + 4968, + 4970, + 4972, + 4975, + 4978, + 4979, + 4980, + 4985, + 4986, + 4987, + 4991, + 4992, + 5016, + 5048, + 5051, + 5063, + 5064, + 5065, + 5066, + 5068, + 5077, + 5078, + 5092, + 5101, + 5135, + 5138, + 5151, + 5155, + 5160, + 5165, + 5172, + 5174, + 5175, + 5181, + 5182, + 5183, + 5188, + 5189, + 5190, + 5191, + 5196, + 5197, + 5198, + 5203, + 5204, + 5216, + 5218, + 5219, + 5220, + 5221, + 5233, + 5234, + 5236, + 5237, + 5240, + 5244, + 5269, + 5284, + 5288, + 5290, + 5293, + 5296, + 5310, + 5312, + 5318, + 5320, + 5346, + 5368, + 5369, + 5370, + 5386, + 5387, + 5388, + 5399, + 5433, + 5455, + 5456, + 5462, + 5463, + 5464, + 5466, + 5479, + 5487, + 5500, + 5510, + 5519, + 5524, + 5534, + 5536, + 5539, + 5541, + 5544, + 5545, + 5548, + 5549, + 5551, + 5552, + 5553, + 5584, + 5585, + 5621, + 5624, + 5625, + 5631, + 5637, + 5638, + 5642, + 5643, + 5644, + 5650, + 5651, + 5655, + 5656, + 5661, + 5663, + 5666, + 5667, + 5668, + 5675, + 5676, + 5677, + 5678, + 5679, + 5680, + 5681, + 5682, + 5683, + 5686, + 5687, + 5688, + 5689, + 5691, + 5692, + 5693, + 5694, + 5696, + 5703, + 5704, + 5705, + 5706, + 5707, + 5708, + 5710, + 5715, + 5716, + 5717, + 5718, + 5719, + 5720, + 5721, + 5722, + 5723, + 5729, + 5732, + 5733, + 5740, + 5746, + 5754, + 5755, + 5759, + 5760, + 5764, + 5768, + 5775, + 5777, + 5782, + 5786, + 5791, + 5792, + 5795, + 5816, + 5817, + 5818, + 5826, + 5828, + 5832, + 5834, + 5843, + 5844, + 5845, + 5863, + 5864, + 5865, + 5866, + 5867, + 5868, + 5879, + 5888, + 5890, + 5894, + 5896, + 5964, + 5965, + 5972, + 5973, + 5974, + 5975, + 5976, + 5984, + 5986, + 5992, + 5993, + 5994, + 5995, + 5996, + 5997, + 5998, + 6001, + 6002, + 6005, + 6033, + 6034, + 6041, + 6042, + 6057, + 6058, + 6080, + 6084, + 6096, + 6116, + 6129, + 6135, + 6138, + 6141, + 6142, + 6143, + 6144, + 6164, + 6169, + 6186, + 6237, + 6247, + 6248, + 6263, + 6265, + 6266, + 6267, + 6268, + 6269, + 6270, + 6271, + 6272, + 6273, + 6275, + 6276, + 6277, + 6278, + 6279, + 6280, + 6281, + 6282, + 6283, + 6284, + 6285, + 6286, + 6287, + 6288, + 6289, + 6290, + 6294, + 6295, + 6296, + 6297, + 6298, + 6300, + 6301, + 6302, + 6303, + 6304, + 6306, + 6307, + 6308, + 6309, + 6310, + 6311, + 6312, + 6313, + 6314, + 6315, + 6316, + 6319, + 6320, + 6322, + 6329, + 6330, + 6344, + 6346, + 6350, + 6351, + 6353, + 6371, + 6373, + 6395, + 6436, + 6530, + 6531, + 6539, + 6542, + 6550, + 6551, + 6556, + 6557, + 6574, + 6581, + 6611, + 6717, + 6720, + 6722, + 6725, + 6726, + 6730, + 6732, + 6752, + 6768, + 6769, + 6770, + 6771, + 6772, + 6773, + 6785, + 6786, + 6791, + 6792, + 6794, + 6796, + 6810, + 6829, + 6831, + 6836, + 6837, + 6838, + 6839, + 6841, + 6853, + 6855, + 6857, + 6858, + 6859, + 6882, + 6890, + 6937, + 6938, + 6939, + 6940, + 6941, + 6942, + 6943, + 6944, + 6945, + 6949, + 6950, + 6952, + 6953, + 6976, + 6977, + 6981, + 6983, + 7015, + 7049, + 7090, + 7091, + 7093, + 7120, + 7350, + 7351, + 7354, + 7357, + 7358, + 7359, + 7364, + 7365, + 7366, + 7367, + 7368, + 7374, + 7388, + 7389, + 7390, + 7391, + 7392, + 7393, + 7395, + 7399, + 7400, + 7401, + 7402, + 7420, + 7421, + 7432, + 7433, + 7441, + 7446, + 7462, + 7471, + 7472, + 7473, + 7476, + 7488, + 7502, + 7503, + 7505, + 7506, + 7512, + 7513, + 7514, + 7515, + 7516, + 7517, + 7524, + 7525, + 7545, + 7547, + 7548, + 7555, + 7557, + 7558, + 7561, + 7562, + 7563, + 7585, + 7586, + 7587, + 7589, + 7590, + 7600, + 7629, + 7666, + 7667, + 7668, + 7744, + 7745, + 7760, + 7818, + 7819, + 7820, + 7821, + 7822, + 7823, + 7824, + 7825, + 7826, + 7827, + 7828, + 7829, + 7831, + 7832, + 7833, + 7834, + 7835, + 7837, + 7838, + 7839, + 7840, + 7841, + 7843, + 7844, + 7845, + 7850, + 7851, + 7852, + 7853, + 7854, + 7855, + 7857, + 7858, + 7859, + 7860, + 7861, + 7870, + 7871, + 7872, + 7873, + 7874, + 7875, + 7877, + 7878, + 7879, + 7880, + 7881, + 7882, + 7883, + 7884, + 7885, + 7886, + 7887, + 7888, + 7889, + 7890, + 7891, + 7894, + 7895, + 7896, + 7897, + 7898, + 7899, + 7900, + 7901, + 7902, + 7904, + 7910, + 7911, + 7912, + 7913, + 7914, + 7915, + 7916, + 7919, + 7920, + 7921, + 7922, + 7936, + 7939, + 7940, + 7941, + 7942, + 7945, + 7948, + 7951, + 7952, + 8101, + 8105, + 8153, + 8164, + 8180, + 8182, + 8184, + 8197, + 8198, + 8200, + 8201, + 8202, + 8203, + 8204, + 8205, + 8206, + 8207, + 8209, + 8210, + 8212, + 8213, + 8214, + 8215, + 8216, + 8217, + 8218, + 8219, + 8220, + 8221, + 8222, + 8223, + 8224, + 8225, + 8226, + 8227, + 8228, + 8229, + 8230, + 8244, + 8245, + 8246, + 8247, + 8248, + 8249, + 8250, + 8251, + 8252, + 8253, + 8254, + 8255, + 8274, + 8296, + 8310, + 8311, + 8312, + 8313, + 8314, + 8315, + 8338, + 8342, + 8344, + 8345, + 8346, + 8351, + 8353, + 8354, + 8355, + 8356, + 8362, + 8367, + 8368, + 8378, + 8388, + 8389, + 8394, + 8404, + 8405, + 8406, + 8408, + 8411, + 8412, + 8415, + 8417, + 8422, + 8423, + 8425, + 8426, + 8427, + 8443, + 8445, + 8446, + 8447, + 8452, + 8460, + 8462, + 8464, + 8562, + 8582, + 8583, + 8592, + 8608, + 8627, + 8628, + 8629, + 8659, + 8711, + 8721, + 8722, + 8723, + 8734, + 8798, + 8832, + 8889, + 8896, + 8897, + 8899, + 8900, + 8901, + 8902, + 8903, + 8904, + 8905, + 8906, + 8907, + 8908, + 8909, + 8910, + 8911, + 8912, + 8913, + 8914, + 8915, + 8916, + 8917, + 8918, + 8919, + 8920, + 8921, + 8922, + 8923, + 8924, + 8925, + 8926, + 8927, + 8928, + 8929, + 8930, + 8931, + 8932, + 8935, + 8945, + 8946, + 8947, + 8948, + 8949, + 8950, + 8951, + 8952, + 8953, + 8954, + 8955, + 8956, + 8957, + 8958, + 8959, + 8960, + 8961, + 8962, + 8963, + 8964, + 8965, + 8966, + 8967, + 8969, + 8997, + 9005, + 9007, + 9016, + 9024, + 9026, + 9036, + 9061, + 9062, + 9063, + 9069, + 9083, + 9084, + 9085, + 9094, + 9095, + 9104, + 9107, + 9108, + 9109, + 9110, + 9113, + 9114, + 9139, + 9140, + 9141, + 9142, + 9143, + 9144, + 9146, + 9147, + 9157, + 9164, + 9165, + 9170, + 9171, + 9172, + 9175, + 9176, + 9180, + 9181, + 9184, + 9185, + 9186, + 9227, + 9262, + 9276, + 9277, + 9278, + 9279, + 9287, + 9288, + 9292, + 9293, + 9296, + 9297, + 9298, + 9300, + 9301, + 9304, + 9305, + 9306, + 9310, + 9311, + 9314, + 9316, + 9318, + 9326, + 9343, + 9360, + 9361, + 9362, + 9363, + 9369, + 9372, + 9375, + 9376, + 9379, + 9383, + 9385, + 9386, + 9387, + 9388, + 9389, + 9390, + 9391, + 9392, + 9393, + 9394, + 9397, + 9399, + 9400, + 9402, + 9407, + 9410, + 9413, + 9418, + 9419, + 9421, + 9422, + 9423, + 9424, + 9425, + 9426, + 9427, + 9428, + 9429, + 9450, + 9451, + 9452, + 9547, + 9548, + 9549, + 9550, + 9638, + 9646, + 9667, + 9675, + 9678, + 9681, + 9682, + 9724, + 9739, + 9763, + 9767, + 9772, + 9774, + 9807, + 9835, + 9839, + 9852, + 9854, + 9861, + 9862, + 9863, + 9866, + 9867, + 9868, + 9869, + 9874, + 9878, + 9895, + 9898, + 9902, + 9904, + 9906, + 9907, + 9908, + 9909, + 9920, + 9925, + 9929, + 9930, + 9933, + 9935, + 9936, + 9941, + 9949, + 9955, + 9958, + 9963, + 10003, + 10004, + 10005, + 10006, + 10007, + 10015, + 10029, + 10030, + 10034, + 10038, + 10043, + 10046, + 10056, + 10068, + 10069, + 10078, + 10111, + 10116, + 10121, + 10127, + 10147, + 10149, + 10152, + 10181, + 10184, + 10187, + 10188, + 10191, + 10192, + 10195, + 10225, + 10228, + 10229, + 10309, + 10315, + 10317, + 10319, + 10394, + 10398, + 10399, + 10408, + 10437, + 10450, + 10451, + 10464, + 10467, + 10469, + 10473, + 10475, + 10477, + 10487, + 10488, + 10489, + 10494, + 10513, + 10514, + 10515, + 10516, + 10517, + 10523, + 10526, + 10527, + 10562, + 10574, + 10581, + 10582, + 10583, + 10587, + 10590, + 10591, + 10593, + 10594, + 10596, + 10599, + 10609, + 10610, + 10614, + 10615, + 10617, + 10618, + 10629, + 10630, + 10649, + 10659, + 10667, + 10668, + 10669, + 10670, + 10674, + 10680, + 10688, + 10689, + 10690, + 10691, + 10704, + 10725, + 10735, + 10743, + 10744, + 10766, + 10775, + 10776, + 10821, + 10830, + 11004, + 11012, + 11014, + 11016, + 11018, + 11020, + 11022, + 11024, + 11026, + 11029, + 11030, + 11042, + 11044, + 11087, + 11088, + 11089, + 11090, + 11091, + 11092, + 11093, + 11094, + 11095, + 11096, + 11097, + 11098, + 11099, + 11100, + 11101, + 11102, + 11103, + 11104, + 11105, + 11106, + 11107, + 11108, + 11109, + 11110, + 11111, + 11113, + 11115, + 11116, + 11117, + 11119, + 11122, + 11123, + 11134, + 11135, + 11136, + 11137, + 11178, + 11179, + 11180, + 11185, + 11186, + 11277, + 11278, + 11279, + 11280, + 11281, + 11282, + 11283, + 11284, + 11285, + 11286, + 11287, + 11288, + 11289, + 11290, + 11291, + 11292, + 11293, + 11294, + 11295, + 11296, + 11297, + 11298, + 11299, + 11300, + 11301, + 11302, + 11303, + 11304, + 11305, + 11306, + 11307, + 11308, + 11309, + 11310, + 11311, + 11312, + 11313, + 11314, + 11315, + 11345, + 11346, + 11350, + 11373, + 11374, + 11387, + 11393, + 11394, + 11395, + 11396, + 11407, + 11409, + 11410, + 11411, + 11412, + 11436, + 11611, + 11612, + 11619, + 11624, + 11629, + 11632, + 11633, + 11654, + 11655, + 11656, + 11657, + 11658, + 11659, + 11660, + 11661, + 11662, + 11663, + 11669, + 11670, + 11672, + 11673, + 11683, + 11684, + 11685, + 11698, + 11700, + 11701, + 11709, + 11714, + 11717, + 11720, + 11723, + 11726, + 11738, + 11741, + 11742, + 11743, + 11751, + 11753, + 11754, + 11756, + 11758, + 11760, + 11763, + 11771, + 11793, + 11794, + 11795, + 11796, + 11801, + 11811, + 11814, + 11815, + 11819, + 11820, + 11821, + 11829, + 11830, + 11831, + 11832, + 11834, + 11835, + 11840, + 11850, + 11851, + 11853, + 11854, + 11859, + 11861, + 11914, + 11943, + 11948, + 11949, + 11954, + 11962, + 11967, + 11980, + 11990, + 11994, + 11995, + 11996, + 11997, + 11998, + 12000, + 12001, + 12002, + 12004, + 12005, + 12006, + 12007, + 12008, + 12009, + 12010, + 12019, + 12047, + 12066, + 12067, + 12071, + 12073, + 12087, + 12088, + 12090, + 12091, + 12093, + 12096, + 12097, + 12106, + 12113, + 12118, + 12119, + 12120, + 12121, + 12122, + 12123, + 12124, + 12125, + 12126, + 12128, + 12132, + 12133, + 12134, + 12135, + 12136, + 12142, + 12143, + 12145, + 12146, + 12147, + 12148, + 12152, + 12157, + 12158, + 12160, + 12162, + 12163, + 12165, + 12167, + 12168, + 12171, + 12172, + 12173, + 12179, + 12182, + 12188, + 12189, + 12192, + 12193, + 12194, + 12195, + 12196, + 12198, + 12202, + 12204, + 12205, + 12210, + 12213, + 12217, + 12218, + 12219, + 12220, + 12221, + 12227, + 12228, + 12235, + 12236, + 12237, + 12238, + 12239, + 12244, + 12245, + 12250, + 12252, + 12260, + 12262, + 12286, + 12288, + 12292, + 12294, + 12296, + 12298, + 12305, + 12312, + 12319, + 12320, + 12333, + 12337, + 12338, + 12343, + 12345, + 12348, + 12350, + 12355, + 12357, + 12362, + 12363, + 12388, + 12396, + 12399, + 12403, + 12405, + 12411, + 12414, + 12415, + 12416, + 12418, + 12422, + 12427, + 12433, + 12434, + 12435, + 12443, + 12444, + 12461, + 12462, + 12463, + 12466, + 12467, + 12468, + 12470, + 12520, + 12527, + 12528, + 12529, + 12530, + 12537, + 12539, + 12631, + 12634, + 12637, + 12644, + 12646, + 12680, + 12692, + 12693, + 12695, + 12708, + 12712, + 12745, + 12754, + 12920, + 12923, + 12924, + 12930, + 12931, + 12933, + 12956, + 13043, + 13062, + 13063, + 13064, + 13065, + 13073, + 13081, + 13090, + 13097, + 13098, + 13099, + 13112, + 13120, + 13121, + 13122, + 13178, + 13187, + 13188, + 13189, + 13190, + 13191, + 13192, + 13193, + 13194, + 13195, + 13196, + 13197, + 13198, + 13199, + 13200, + 13210, + 13211, + 13212, + 13213, + 13214, + 13215, + 13216, + 13217, + 13218, + 13219, + 13220, + 13221, + 13222, + 13223, + 13224, + 13225, + 13226, + 13227, + 13228, + 13229, + 13242, + 13243, + 13245, + 13246, + 13359, + 13360, + 13379, + 13384, + 13389, + 13390, + 13417, + 13438, + 13484, + 13495, + 13496, + 13519, + 13528, + 13529, + 13530, + 13531, + 13532, + 13533, + 13534, + 13535, + 13536, + 13537, + 13538, + 13539, + 13540, + 13546, + 13547, + 13550, + 13551, + 13552, + 13553, + 13557, + 13598, + 13623, + 13624, + 13640, + 13703, + 13713, + 13740, + 13741, + 13742, + 13746, + 13747, + 13750, + 13775, + 13776, + 13777, + 13778, + 13779, + 13780, + 13781, + 13782, + 13783, + 13784, + 13785, + 13786, + 13787, + 13788, + 13789, + 13790, + 13791, + 13792, + 13793, + 13794, + 13795, + 13796, + 13797, + 13798, + 13806, + 13807, + 13808, + 13809, + 13810, + 13811, + 13812, + 13813, + 13814, + 13815, + 13816, + 13817, + 13818, + 13819, + 13820, + 13821, + 13822, + 13823, + 13824, + 13825, + 13826, + 13827, + 13828, + 13829, + 13830, + 13831, + 13845, + 13846, + 13847, + 13848, + 13849, + 13850, + 13851, + 13852, + 13853, + 13854, + 13855, + 13856, + 13857, + 13858, + 13859, + 13860, + 13861, + 13862, + 13863, + 13864, + 13865, + 13866, + 13867, + 13868, + 13869, + 13870, + 13871, + 13876, + 13877, + 13878, + 13879, + 13899, + 13900, + 13901, + 13902, + 13903, + 13904, + 13905, + 13906, + 13907, + 13908, + 13909, + 13910, + 13911, + 13912, + 13913, + 13914, + 13915, + 13916, + 13917, + 13918, + 13919, + 13920, + 13921, + 13922, + 13923, + 13924, + 14066, + 14067, + 14068, + 14069, + 14070, + 14071, + 14072, + 14073, + 14074, + 14075, + 14076, + 14077, + 14078, + 14079, + 14080, + 14081, + 14082, + 14083, + 14084, + 14085, + 14086, + 14087, + 14088, + 14089, + 14090, + 14091, + 14148, + 14477, + 14478, + 14479, + 14480, + 14570, + 14572, + 14573, + 14574, + 14575, + 14576, + 14577, + 14578, + 14703, + 14704, + 14705, + 14706, + 14719, + 14781, + 14843, + 14867, + 14914, + 14915, + 15097, + 15098, + 15099, + 15114, + 15176, + 15180, + 15190, + 15207, + 15214, + 15217, + 15219, + 15223, + 15231, + 15235, + 15242, + 15249, + 15251, + 15253, + 15254, + 15257, + 15258, + 15259, + 15262, + 15267, + 15370, + 15382, + 15408, + 15456, + 15458, + 15459, + 15460, + 15462, + 15463, + 15470, + 15471, + 15472, + 15473, + 15474, + 15475, + 15476, + 15477, + 15478, + 15479, + 15480, + 15490, + 15491, + 15492, + 15493, + 15494, + 15495, + 15496, + 15497, + 15498, + 15499, + 15500, + 15596, + 15610, + 15614, + 15618, + 15619, + 15620, + 15621, + 15671, + 15673, + 15674, + 15675, + 15676, + 15677, + 15678, + 15751, + 15752, + 15753, + 15754, + 15755, + 15761, + 15762, + 15763, + 15764, + 15765, + 15772, + 15773, + 15774, + 15775, + 15776, + 15782, + 15783, + 15784, + 15785, + 15786, + 15789, + 16071, + 16073, + 16074, + 16075, + 16077, + 16078, + 16085, + 16086, + 16087, + 16088, + 16089, + 16090, + 16091, + 16092, + 16093, + 16094, + 16095, + 16105, + 16106, + 16107, + 16108, + 16109, + 16110, + 16111, + 16112, + 16113, + 16114, + 16115, + 16236, + 16237, + 16240, + 16252, + 16256, + 16260, + 16261, + 16262, + 16263, + 16383, + 16595, + 16597, + 16598, + 16599, + 16600, + 16604, + 16605, + 16607, + 16609, + 16612, + 16613, + 16614, + 16619, + 16621, + 16624, + 16626, + 16628, + 16630, + 16631, + 16632, + 16633, + 16634, + 16637, + 16639, + 16642, + 16643, + 16644, + 16645, + 16646, + 16649, + 16655, + 17325, + 17347, + 17351, + 17760, + 17761, + 17768, + 17781, + 17800, + 17808, + 17821, + 17829, + 17835, + 17849, + 17850, + 17851, + 17852, + 17854, + 17855, + 17856, + 17857, + 17859, + 17860, + 17861, + 17862, + 17864, + 17865, + 17866, + 17867, + 17869, + 17870, + 17871, + 17872, + 17896, + 17898, + 17900, + 18000, + 18013, + 18020, + 18022, + 18149, + 18150, + 18155, + 18163, + 18174, + 18183, + 18188, + 18199, + 18202, + 18205, + 18208, + 18209, + 18210, + 18213, + 18214, + 18215, + 18221, + 18228, + 18229, + 18230, + 18236, + 18250, + 18261, + 18263, + 18264, + 18265, + 18270, + 18283, + 18284, + 18286, + 18287, + 18291, + 18295, + 18296, + 18297, + 18298, + 18299, + 18300, + 18316, + 18390, + 18391, + 18394, + 18395, + 18398, + 18399, + 18400, + 18401, + 18402, + 18403, + 18404, + 18407, + 18410, + 18412, + 18420, + 18421, + 18425, + 18435, + 18436, + 18522, + 18527, + 18571, + 18585, + 18599, + 18608, + 18609, + 18610, + 18611, + 18633, + 18634, + 18635, + 18636, + 18637, + 18639, + 18643, + 18647, + 18648, + 18654, + 18655, + 18657, + 18658, + 18667, + 18681, + 18682, + 18683, + 18684, + 18686, + 18690, + 18693, + 18694, + 18698, + 18699, + 18702, + 18722, + 18725, + 18753, + 18754, + 18755, + 18756, + 18757, + 18760, + 18763, + 18765, + 18768, + 18770, + 18781, + 18784, + 18785, + 18786, + 18787, + 18788, + 18789, + 18790, + 18791, + 18802, + 18803, + 18804, + 18805, + 18806, + 18807, + 18808, + 18809, + 18820, + 18828, + 18838, + 18869, + 18902, + 18903, + 18905, + 18908, + 18926, + 18927, + 18929, + 19513, + 19514, + 19515, + 19522, + 19527, + 19528, + 19530, + 19533, + 19594, + 19596, + 19597, + 19618, + 19637, + 19640, + 19643, + 19653, + 19663, + 19666, + 21140, + 21160, + 21165, + 21195, + 21231, + 21236, + 21265, + 21266, + 21267, + 21268, + 21270, + 21271, + 21310, + 21526, + 21527, + 21576, + 21577, + 21578, + 21580, + 21581, + 21582, + 21584, + 21586, + 21588, + 21781, + 21782, + 21783, + 21784, + 21786, + 21787, + 21788, + 21789, + 21825, + 21866, + 21867, + 21908, + 21914, + 21915, + 21916, + 21918, + 21919, + 21921, + 21922, + 21974, + 21980, + 21981, + 21985, + 21990, + 21993, + 21999, + 22000, + 22004, + 22008, + 22009, + 22139, + 22313, + 22398, + 22399, + 22402, + 22403, + 22405, + 22407, + 22410, + 22419, + 22420, + 22421, + 22427, + 22464, + 22472, + 22476, + 22514, + 22515, + 22516, + 22518, + 22528, + 22529, + 22530, + 22533, + 22534, + 22557, + 22560, + 22578, + 22581, + 22583, + 22585, + 22589, + 22599, + 22600, + 22637, + 22638, + 22648, + 22649, + 22650, + 22674, + 22685, + 22703, + 22733, + 22734, + 22736, + 22737, + 22774, + 22775, + 22776, + 22777, + 22778, + 22779, + 22780, + 22781, + 22782, + 22783, + 22784, + 22785, + 22786, + 22826, + 22864, + 22865, + 22893, + 22901, + 22906, + 22915, + 23196, + 23198, + 23200, + 23202, + 23203, + 23204, + 23206, + 23207, + 23211, + 23213, + 23217, + 23222, + 23225, + 23229, + 23233, + 23239, + 23246, + 23248, + 23282, + 23284, + 23285, + 23388, + 23391, + 23396, + 23397, + 23398, + 23400, + 23402, + 23403, + 23404, + 23405, + 23406, + 23407, + 23412, + 23415, + 23430, + 23432, + 23437, + 23456, + 23476, + 23483, + 23511, + 23513, + 23514, + 23537, + 23553, + 23565, + 23566, + 23567, + 23568, + 23569, + 23570, + 23571, + 23572, + 23574, + 23576, + 23577, + 23613, + 23615, + 23623, + 23636, + 23637, + 23638, + 23639, + 23641, + 23643, + 23644, + 23647, + 23648, + 23651, + 23652, + 23662, + 23663, + 23664, + 23715, + 23717, + 23729, + 23730, + 23731, + 23741, + 23749, + 23751, + 23787, + 23788, + 23789, + 23790, + 23791, + 23792, + 23793, + 23794, + 23799, + 23800, + 23811, + 23813, + 23817, + 23822, + 23825, + 23829, + 23833, + 23837, + 23841, + 23847, + 23862, + 23864, + 23866, + 23869, + 24009, + 24019, + 24038, + 24039, + 24047, + 24052, + 24055, + 24057, + 24058, + 24059, + 24098, + 24136, + 24188, + 24195, + 24197, + 24198, + 24199, + 24200, + 24201, + 24202, + 24203, + 24204, + 24205, + 24206, + 24207, + 24208, + 24209, + 24210, + 24211, + 24212, + 24213, + 24214, + 24215, + 24216, + 24217, + 24218, + 24219, + 24220, + 24221, + 24222, + 24223, + 24224, + 24225, + 24226, + 24227, + 24228, + 24229, + 24230, + 24231, + 24232, + 24233, + 24234, + 24235, + 24236, + 24237, + 24238, + 24239, + 24240, + 24241, + 24242, + 24243, + 24244, + 24245, + 24246, + 24247, + 24248, + 24249, + 24250, + 24251, + 24252, + 24253, + 24254, + 24255, + 24256, + 24257, + 24258, + 24259, + 24260, + 24261, + 24262, + 24263, + 24264, + 24265, + 24281, + 24282, + 24283, + 24285, + 24286, + 24288, + 24289, + 24290, + 24291, + 24293, + 24294, + 24295, + 24299, + 24300, + 24310, + 24311, + 24313, + 24317, + 24322, + 24324, + 24325, + 24331, + 24332, + 24334, + 24335, + 24336, + 24341, + 24343, + 24344, + 24346, + 24347, + 24349, + 24351, + 24356, + 24362, + 24363, + 24364, + 24367, + 24371, + 24380, + 24381, + 24384, + 24386, + 24394, + 24395, + 24396, + 24397, + 24398, + 24399, + 24405, + 24411, + 24412, + 24414, + 24415, + 24416, + 24417, + 24418, + 24419, + 24425, + 24426, + 24428, + 24444, + 24445, + 24446, + 24447, + 24448, + 24449, + 24450, + 24451, + 24452, + 24453, + 24454, + 24455, + 24456, + 24517, + 24519, + 24532, + 24534, + 24538, + 24539, + 24542, + 24545, + 24546, + 24548, + 24550, + 24551, + 24556, + 24563, + 24571, + 24572, + 24573, + 24580, + 24583, + 24589, + 24590, + 24591, + 24592, + 24593, + 24594, + 24595, + 24596, + 24597, + 24627, + 24628, + 24629, + 24632, + 24633, + 24634, + 24637, + 24638, + 24668, + 24669, + 24670, + 24691, + 24706, + 24707, + 24708, + 24748, + 24749, + 24755, + 24756, + 24760, + 24765, + 24768, + 24769, + 24774, + 24796, + 24935, + 24936, + 24989, + 24990, + 24991, + 24992, + 24997, + 24998, + 25023, + 25026, + 25033, + 25034, + 25035, + 25036, + 25038, + 25039, + 25040, + 25052, + 25062, + 25072, + 25082, + 25084, + 25085, + 25086, + 25093, + 25175, + 25180, + 25181, + 25182, + 25184, + 25191, + 25193, + 25196, + 25197, + 25201, + 25210, + 25211, + 25213, + 25214, + 25237, + 25238, + 25239, + 25240, + 25241, + 25242, + 25243, + 25244, + 25257, + 25258, + 25259, + 25260, + 25293, + 25296, + 25297, + 25298, + 25301, + 25347, + 25360, + 25381, + 25383, + 25391, + 25394, + 25396, + 25407, + 25408, + 25421, + 25428, + 25439, + 25440, + 25441, + 25442, + 25443, + 25444, + 25445, + 25446, + 25447, + 25448, + 25449, + 25472, + 25481, + 25493, + 25494, + 25495, + 25496, + 25511, + 25512, + 25513, + 25514, + 25521, + 25522, + 25523, + 25528, + 25537, + 25539, + 25540, + 25541, + 25547, + 25548, + 25549, + 25550, + 25552, + 25553, + 25571, + 25573, + 25574, + 25588, + 25589, + 25590, + 25591, + 25592, + 25594, + 25596, + 25599, + 25635, + 25639, + 25640, + 25643, + 25663, + 25664, + 25666, + 25667, + 25668, + 25669, + 25670, + 25672, + 25673, + 25674, + 25676, + 25678, + 25679, + 25680, + 25681, + 25682, + 25683, + 25684, + 25685, + 25686, + 25688, + 25691, + 25692, + 25693, + 25697, + 25699, + 25701, + 25702, + 25704, + 25706, + 25707, + 25708, + 25709, + 25710, + 25711, + 25712, + 25713, + 25722, + 25723, + 25724, + 25726, + 25732, + 25733, + 25736, + 25754, + 25755, + 25756, + 25757, + 25762, + 25768, + 25773, + 25776, + 25777, + 25820, + 25826, + 25833, + 25834, + 25839, + 25846, + 25863, + 25866, + 25868, + 25921, + 25922, + 25923, + 25926, + 25931, + 25943, + 25944, + 25948, + 25949, + 25956, + 25959, + 25998, + 26000, + 26008, + 26011, + 26024, + 26027, + 26036, + 26037, + 26038, + 26044, + 26045, + 26046, + 26057, + 26061, + 26064, + 26108, + 26134, + 26148, + 26149, + 26159, + 26160, + 26208, + 26223, + 26239, + 26240, + 26241, + 26242, + 26243, + 26244, + 26245, + 26246, + 26259, + 26269, + 26273, + 26274, + 26327, + 26340, + 26341, + 26344, + 26345, + 26347, + 26348, + 26350, + 26351, + 26352, + 26353, + 26354, + 26355, + 26373, + 26378, + 26388, + 26395, + 26396, + 26397, + 26398, + 26400, + 26401, + 26402, + 26413, + 26414, + 26415, + 26434, + 26435, + 26442, + 26495, + 26496, + 26497, + 26506, + 26511, + 26512, + 26516, + 26533, + 26544, + 26546, + 26547, + 26554, + 26564, + 26571, + 26572, + 26573, + 26585, + 26605, + 26634, + 26837, + 26866, + 26867, + 26927, + 27075, + 27140, + 27141, + 27142, + 27143, + 27144, + 27145, + 27146, + 27147, + 27148, + 27149, + 27150, + 27152, + 27158, + 27161, + 27162, + 27171, + 27174, + 27175, + 27177, + 27179, + 27181, + 27186, + 27189, + 27192, + 27193, + 27195, + 27200, + 27203, + 27205, + 27219, + 27220, + 27221, + 27222, + 27223, + 27224, + 27225, + 27226, + 27228, + 27229, + 27230, + 27231, + 27233, + 27234, + 27235, + 27237, + 27241, + 27246, + 27248, + 27250, + 27350, + 27351, + 27352, + 27353, + 27354, + 27355, + 27356, + 27357, + 27358, + 27359, + 27360, + 27361, + 27362, + 27363, + 27364, + 27365, + 27367, + 27368, + 27369, + 27370, + 27371, + 27372, + 27373, + 27387, + 27399, + 27417, + 27425, + 27453, + 27458, + 27459, + 27486, + 27499, + 27506, + 27527, + 27533, + 27541, + 27542, + 27543, + 27551, + 27555, + 27556, + 27561, + 27572, + 27576, + 27579, + 27581, + 27586, + 27588, + 27590, + 27592, + 27594, + 27597, + 27598, + 27605, + 27607, + 27614, + 27616, + 27620, + 27621, + 27636, + 27637, + 27641, + 27647, + 27677, + 27682, + 27686, + 27723, + 27725, + 27741, + 27748, + 27753, + 27754, + 27756, + 27781, + 27808, + 27817, + 27819, + 27825, + 27830, + 27831, + 27839, + 27852, + 27877, + 27880, + 27881, + 27882, + 27883, + 27884, + 27885, + 27886, + 27887, + 27888, + 27892, + 27893, + 27897, + 27898, + 27899, + 27908, + 27920, + 27926, + 27927, + 27938, + 27940, + 27945, + 27960, + 27974, + 27976, + 27977, + 27978, + 27986, + 28004, + 28027, + 28028, + 28033, + 28034, + 28035, + 28040, + 28058, + 28059, + 28064, + 28066, + 28071, + 28075, + 28080, + 28084, + 28088, + 28092, + 28096, + 28098, + 28106, + 28107, + 28108, + 28109, + 28110, + 28111, + 28112, + 28120, + 28121, + 28157, + 28176, + 28177, + 28195, + 28242, + 28244, + 28245, + 28246, + 28247, + 28248, + 28249, + 28250, + 28251, + 28252, + 28253, + 28254, + 28255, + 28256, + 28258, + 28259, + 28260, + 28261, + 28263, + 28264, + 28265, + 28266, + 28268, + 28269, + 28270, + 28271, + 28272, + 28273, + 28274, + 28275, + 28276, + 28278, + 28360, + 28361, + 28362, + 28368, + 28371, + 28591, + 28647, + 28668, + 28704, + 28789, + 28792, + 28803, + 28805, + 28811, + 28815, + 28818, + 28845, + 28846, + 28847, + 28850, + 28851, + 28859, + 28861, + 28864, + 28875, + 28884, + 28889, + 28893, + 28898, + 28899, + 28907, + 28908, + 28909, + 28910, + 28911, + 28912, + 28913, + 28914, + 28915, + 28916, + 28922, + 28923, + 28925, + 28926, + 28927, + 28928, + 28929, + 28930, + 28934, + 28937, + 28945, + 28981, + 28994, + 28995, + 29024, + 29026, + 29052, + 29053, + 29054, + 29127, + 29129, + 29135, + 29137, + 29138, + 29139, + 29142, + 29144, + 29148, + 29152, + 29154, + 29156, + 29169, + 29182, + 29183, + 29184, + 29185, + 29187, + 29188, + 29190, + 29202, + 29203, + 29204, + 29205, + 29206, + 29207, + 29210, + 29229, + 29236, + 29246, + 29247, + 29266, + 29276, + 29287, + 29304, + 29310, + 29314, + 29316, + 29317, + 29320, + 29325, + 29327, + 29329, + 29336, + 29340, + 29343, + 29346, + 29347, + 29365, + 29373, + 29375, + 29377, + 29405, + 29406, + 29410, + 29437, + 29438, + 29439, + 29440, + 29441, + 29442, + 29443, + 29444, + 29445, + 29446, + 29509, + 29511, + 29532, + 29533, + 29561, + 29585, + 29586, + 29599, + 29600, + 29601, + 29605, + 29608, + 29609, + 29611, + 29612, + 29613, + 29614, + 29615, + 29637, + 29638, + 29639, + 29641, + 29643, + 29645, + 29646, + 29647, + 29648, + 29649, + 29650, + 29651, + 29652, + 29653, + 29654, + 29655, + 29656, + 29657, + 29659, + 29661, + 29663, + 29667, + 29673, + 29693, + 29695, + 29698, + 29699, + 29700, + 29701, + 29702, + 29703, + 29704, + 29705, + 29706, + 29707, + 29708, + 29709, + 29710, + 29721, + 29722, + 29723, + 29724, + 29725, + 29726, + 29727, + 29748, + 29768, + 29769, + 29779, + 29786, + 29787, + 29789, + 29795, + 29796, + 29805, + 29807, + 29808, + 29809, + 29810, + 29811, + 29816, + 29824, + 29825, + 29826, + 29827, + 29828, + 29829, + 29830, + 29831, + 29832, + 29833, + 29834, + 29835, + 29836, + 29842, + 29843, + 29850, + 29851, + 29853, + 29854, + 29855, + 29856, + 29857, + 29858, + 29859, + 29860, + 29861, + 29862, + 29863, + 29864, + 29865, + 29866, + 29867, + 29868, + 29869, + 29870, + 29873, + 29874, + 29875, + 29876, + 29877, + 29878, + 29879, + 29880, + 29881, + 29882, + 29883, + 29884, + 29885, + 29886, + 29887, + 29888, + 29889, + 29890, + 29891, + 29892, + 29893, + 29894, + 29895, + 29898, + 29899, + 29900, + 29904, + 29905, + 29906, + 29907, + 29908, + 29909, + 29912, + 29913, + 29914, + 29915, + 29916, + 29918, + 29919, + 29920, + 29921, + 29924, + 29929, + 29939, + 29940, + 29941, + 29942, + 29944, + 29972, + 29978, + 29979, + 29980, + 29991, + 29993, + 29994, + 29995, + 29996, + 29997, + 29998, + 29999, + 30001, + 30003, + 30004, + 30005, + 30007, + 30008, + 30009, + 30010, + 30011, + 30012, + 30013, + 30014, + 30015, + 30016, + 30017, + 30018, + 30019, + 30027, + 30032, + 30036, + 30037, + 30055, + 30057, + 30058, + 30059, + 30060, + 30061, + 30062, + 30063, + 30064, + 30065, + 30066, + 30069, + 30071, + 30073, + 30076, + 30078, + 30079, + 30080, + 30081, + 30082, + 30083, + 30084, + 30094, + 30095, + 30116, + 30117, + 30118, + 30119, + 30120, + 30131, + 30132, + 30133, + 30134, + 30135, + 30136, + 30137, + 30138, + 30139, + 30140, + 30141, + 30142, + 30143, + 30144, + 30145, + 30146, + 30147, + 30148, + 30149, + 30150, + 30151, + 30152, + 30153, + 30154, + 30155, + 30156, + 30157, + 30158, + 30159, + 30160, + 30161, + 30162, + 30163, + 30164, + 30165, + 30166, + 30167, + 30168, + 30169, + 30170, + 30171, + 30172, + 30173, + 30174, + 30178, + 30179, + 30180, + 30181, + 30182, + 30183, + 30184, + 30185, + 30186, + 30189, + 30194, + 30199, + 30202, + 30203, + 30204, + 30205, + 30207, + 30211, + 30213, + 30215, + 30217, + 30218, + 30220, + 30221, + 30222, + 30224, + 30225, + 30226, + 30233, + 30254, + 30272, + 30273, + 30298, + 30300, + 30302, + 30322, + 30340, + 30343, + 30345, + 30348, + 30349, + 30353, + 30354, + 30356, + 30362, + 30363, + 30364, + 30365, + 30366, + 30367, + 30368, + 30369, + 30370, + 30371, + 30372, + 30373, + 30374, + 30375, + 30376, + 30377, + 30380, + 30381, + 30400, + 30406, + 30408, + 30414, + 30418, + 30421, + 30431, + 30433, + 30437, + 30443, + 30445, + 30457, + 30464, + 30492, + 30494, + 30498, + 30507, + 30511, + 30578, + 30580, + 30604, + 30623, + 30625, + 30627, + 30629, + 30630, + 30632, + 30642, + 30720, + 30769, + 30770, + 30771, + 30845, + 30884, + 30885, + 30886, + 30891, + 30892, + 30899, + 30900, + 30902, + 30905, + 30906, + 30908, + 30911, + 30922, + 30924, + 30926, + 30930, + 30931, + 30934, + 30937, + 30975, + 30979, + 30982, + 30992, + 30994, + 30995, + 30996, + 30997, + 30998, + 30999, + 31000, + 31001, + 31002, + 31003, + 31004, + 31005, + 31206, + 31207, + 31208, + 31209, + 31212, + 31296, + 31363, + 31364, + 31382, + 31383, + 31410, + 31413, + 31464, + 31465, + 31466, + 31467, + 31675, + 31678, + 31684, + 31685, + 31695, + 31696, + 31697, + 31698, + 31715, + 31737, + 31744, + 31747, + 31748, + 31749, + 31762, + 31784, + 31790, + 31791, + 31792, + 31793, + 31794, + 31795, + 31796, + 31797, + 31798, + 31799, + 31803, + 31804, + 31806, + 31807, + 31809, + 31814, + 31823, + 31824, + 31825, + 31826, + 31827, + 31828, + 31829, + 31830, + 31831, + 31834, + 31838, + 31842, + 31845, + 31847, + 31857, + 31861, + 31866, + 31869, + 31870, + 31872, + 31873, + 31874, + 31878, + 31881, + 31883, + 31892, + 31896, + 31898, + 31905, + 31916, + 31919, + 31920, + 31921, + 31922, + 31931, + 31932, + 31933, + 31939, + 31940, + 31942, + 31944, + 31945, + 31948, + 31949, + 31951, + 31952, + 31954, + 31955, + 31969, + 31973, + 31974, + 31976, + 32007, + 32008, + 32013, + 32014, + 32015, + 32019, + 32020, + 32025, + 32026, + 32031, + 32038, + 32043, + 32046, + 32048, + 32050, + 32053, + 32059, + 32088, + 32093, + 32098, + 32101, + 32104, + 32107, + 32110, + 32120, + 32121, + 32122, + 32123, + 32125, + 32127, + 32128, + 32129, + 32130, + 32270, + 32271, + 32272, + 32273, + 32274, + 32275, + 32299, + 32300, + 32303, + 32304, + 32305, + 32307, + 32308, + 32837, + 32840, + 32896, + 32897, + 32898, + 32899, + 32900, + 32901, + 32902, + 32903, + 32904, + 32905, + 32906, + 32907, + 32908, + 32909, + 32910, + 32911, + 32949, + 32951, + 32962, + 32969, + 32972, + 32988, + 32990, + 33038, + 33043, + 33046, + 33070, + 33071, + 33083, + 33103, + 33105, + 33110, + 33111, + 33113, + 33115, + 33116, + 33117, + 33118, + 33119, + 33120, + 33121, + 33122, + 33124, + 33126, + 33127, + 33128, + 33129, + 33131, + 33132, + 33133, + 33134, + 33135, + 33136, + 33138, + 33139, + 33140, + 33141, + 33142, + 33144, + 33146, + 33147, + 33148, + 33149, + 33150, + 33151, + 33152, + 33153, + 33154, + 33155, + 33156, + 33157, + 33158, + 33161, + 33162, + 33166, + 33167, + 33168, + 33170, + 33171, + 33172, + 33177, + 33179, + 33182, + 33183, + 33184, + 33185, + 33186, + 33203, + 33204, + 33205, + 33329, + 33348, + 33349, + 33350, + 33351, + 33357, + 33581, + 33585, + 33595, + 33597, + 33603, + 33642, + 33644, + 33672, + 33680, + 33683, + 33730, + 33732, + 33740, + 33748, + 33753, + 33760, + 33761, + 33785, + 33787, + 33794, + 33796, + 33797, + 33800, + 33806, + 33810, + 33817, + 33820, + 33830, + 33832, + 33877, + 33879, + 33881, + 33886, + 33892, + 33896, + 33897, + 33899, + 33942, + 33999, + 34000, + 34022, + 34143, + 34160, + 34303, + 34311, + 34345, + 34346, + 34347, + 34348, + 34349, + 34350, + 34354, + 34386, + 34651, + 34740, + 34924, + 34928, + 34930, + 34932, + 34935, + 34936, + 34937, + 34938, + 34939, + 34940, + 34953, + 34972, + 34975, + 34976, + 34980, + 34981, + 35050, + 35052, + 35053, + 35055, + 35062, + 35063, + 35064, + 35065, + 35066, + 35069, + 35075, + 35076, + 35090, + 35091, + 35127, + 35203, + 35226, + 35325, + 35340, + 35341, + 35342, + 35343, + 35344, + 35345, + 35366, + 35367, + 35381, + 35382, + 35391, + 35392, + 35406, + 35488, + 35532, + 35533, + 35534, + 35555, + 35559, + 35617, + 35619, + 35625, + 35632, + 35648, + 35656, + 35659, + 35660, + 35663, + 35667, + 35669, + 35679, + 35682, + 35683, + 35691, + 35696, + 35697, + 35702, + 35711, + 35718, + 35734, + 35740, + 35741, + 35745, + 35748, + 35749, + 35750, + 35751, + 35757, + 35758, + 35759, + 35764, + 35765, + 35767, + 35785, + 35788, + 35792, + 35822, + 35823, + 35835, + 35837, + 35838, + 35841, + 35851, + 35852, + 35861, + 35867, + 35871, + 35876, + 35879, + 35881, + 35890, + 35897, + 35939, + 35940, + 35961, + 35964, + 35974, + 35990, + 35991, + 35992, + 35993, + 35994, + 35995, + 35996, + 35997, + 35998, + 36002, + 36003, + 36005, + 36006, + 36007, + 36008, + 36009, + 36015, + 36017, + 36018, + 36021, + 36024, + 36035, + 36037, + 36040, + 36051, + 36080, + 36082, + 36102, + 36103, + 36104, + 36105, + 36106, + 36158, + 36159, + 36171, + 36173, + 36175, + 36198, + 36211, + 36212, + 36270, + 36274, + 36327, + 36378, + 36379, + 36380, + 36381, + 36382, + 36383, + 36384, + 36385, + 36386, + 36388, + 36389, + 36392, + 36399, + 36407, + 36411, + 36413, + 36451, + 36452, + 36454, + 36455, + 36456, + 36459, + 36606, + 36610, + 36611, + 36614, + 36617, + 36618, + 36736, + 36740, + 36789, + 36862, + 36914, + 36915, + 36916, + 36917, + 36918, + 36919, + 36924, + 36925, + 36926, + 36927, + 36928, + 36930, + 36938, + 36939, + 36940, + 36944, + 36946, + 36947, + 36948, + 36949, + 36950, + 36951, + 36952, + 36953, + 36954, + 36955, + 36956, + 36957, + 36958, + 37025, + 37055, + 37108, + 37109, + 37110, + 37118, + 37123, + 37125, + 37127, + 37130, + 37131, + 37161, + 37162, + 37182, + 37185, + 37186, + 37189, + 37190, + 37224, + 37227, + 37229, + 37233, + 37239, + 37240, + 37241, + 37242, + 37243, + 37244, + 37245, + 37246, + 37247, + 37248, + 37249, + 37250, + 37251, + 37252, + 37253, + 37254, + 37255, + 37256, + 37257, + 37258, + 37259, + 37260, + 37261, + 37262, + 37263, + 37264, + 37265, + 37266, + 37267, + 37268, + 37269, + 37270, + 37271, + 37272, + 37273, + 37274, + 37275, + 37276, + 37277, + 37278, + 37279, + 37280, + 37281, + 37282, + 37283, + 37284, + 37285, + 37286, + 37287, + 37288, + 37289, + 37290, + 37291, + 37292, + 37293, + 37294, + 37295, + 37296, + 37297, + 37298, + 37299, + 37300, + 37301, + 37302, + 37303, + 37304, + 37305, + 37314, + 37315, + 37317, + 37321, + 37331, + 37332, + 37333, + 37334, + 37335, + 37336, + 37355, + 37356, + 37357, + 37365, + 37366, + 37367, + 37368, + 37377, + 37401, + 37402, + 37403, + 37406, + 37407, + 37411, + 37417, + 37419, + 37420, + 37421, + 37422, + 37423, + 37424, + 37425, + 37431, + 37432, + 37433, + 37434, + 37437, + 37438, + 37439, + 37440, + 37441, + 37442, + 37448, + 37449, + 37450, + 37484, + 37487 + ] + } + }, + "snowflake_types.json": { + "snowflake": { + "statements": 267, + "refused": [ + 192, + 193, + 194, + 224 + ] + } + }, + "sqlglot.json": { + "duckdb": { + "statements": 3017, + "refused": [ + 993, + 1012, + 1013, + 1014, + 1015, + 1016, + 1017, + 1018, + 1260, + 2248, + 2545, + 2553, + 2711, + 2713, + 4915, + 4987, + 5026 + ] + }, + "snowflake": { + "statements": 3351, + "refused": [ + 0, + 1, + 2, + 4, + 6, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 191, + 192, + 193, + 194, + 195, + 196, + 202, + 203, + 211, + 212, + 465, + 502, + 506, + 507, + 533, + 534, + 535, + 536, + 537, + 554, + 556, + 557, + 558, + 559, + 565, + 566, + 614, + 615, + 624, + 625, + 626, + 627, + 644, + 645, + 646, + 708, + 783, + 784, + 807, + 820, + 821, + 822, + 823, + 824, + 825, + 826, + 827, + 828, + 829, + 851, + 852, + 858, + 864, + 875, + 892, + 893, + 894, + 898, + 899, + 927, + 928, + 950, + 977, + 978, + 979, + 980, + 981, + 982, + 983, + 984, + 985, + 986, + 987, + 988, + 989, + 990, + 991, + 992, + 993, + 994, + 995, + 996, + 997, + 998, + 999, + 1012, + 1013, + 1014, + 1015, + 1016, + 1017, + 1018, + 1028, + 1035, + 1039, + 1062, + 1067, + 1083, + 1106, + 1119, + 1128, + 1130, + 1133, + 1135, + 1144, + 1146, + 1151, + 1199, + 1200, + 1202, + 1283, + 1284, + 1291, + 1292, + 1293, + 1796, + 1798, + 1829, + 1902, + 1905, + 1908, + 1911, + 1914, + 1917, + 1919, + 1921, + 1981, + 2013, + 2015, + 2017, + 2019, + 2021, + 2023, + 2025, + 2027, + 2029, + 2031, + 2033, + 2035, + 2037, + 2039, + 2041, + 2043, + 2045, + 2047, + 2049, + 2051, + 2053, + 2055, + 2078, + 2096, + 2097, + 2110, + 2185, + 2187, + 2191, + 2197, + 2199, + 2205, + 2206, + 2208, + 2209, + 2210, + 2233, + 2238, + 2239, + 2240, + 2241, + 2246, + 2247, + 2249, + 2250, + 2251, + 2252, + 2253, + 2254, + 2256, + 2257, + 2258, + 2259, + 2261, + 2262, + 2263, + 2264, + 2266, + 2267, + 2269, + 2270, + 2271, + 2272, + 2273, + 2274, + 2275, + 2276, + 2277, + 2296, + 2297, + 2299, + 2300, + 2310, + 2312, + 2314, + 2316, + 2318, + 2320, + 2322, + 2323, + 2324, + 2325, + 2327, + 2328, + 2330, + 2331, + 2333, + 2335, + 2336, + 2337, + 2338, + 2340, + 2341, + 2342, + 2344, + 2346, + 2347, + 2349, + 2350, + 2352, + 2354, + 2356, + 2357, + 2359, + 2360, + 2361, + 2362, + 2363, + 2365, + 2366, + 2367, + 2368, + 2369, + 2370, + 2371, + 2372, + 2373, + 2374, + 2375, + 2376, + 2377, + 2378, + 2379, + 2380, + 2381, + 2382, + 2384, + 2385, + 2386, + 2388, + 2389, + 2391, + 2392, + 2394, + 2395, + 2396, + 2397, + 2399, + 2400, + 2402, + 2403, + 2405, + 2406, + 2407, + 2409, + 2411, + 2413, + 2415, + 2417, + 2427, + 2429, + 2430, + 2432, + 2434, + 2435, + 2437, + 2439, + 2441, + 2442, + 2443, + 2445, + 2446, + 2448, + 2450, + 2451, + 2452, + 2453, + 2454, + 2455, + 2456, + 2457, + 2474, + 2475, + 2477, + 2478, + 2480, + 2481, + 2484, + 2485, + 2486, + 2487, + 2489, + 2491, + 2493, + 2494, + 2496, + 2498, + 2500, + 2502, + 2504, + 2508, + 2509, + 2510, + 2511, + 2512, + 2514, + 2515, + 2516, + 2518, + 2519, + 2521, + 2522, + 2523, + 2536, + 2537, + 2541, + 2554, + 2568, + 2569, + 2573, + 2574, + 2575, + 2577, + 2578, + 2580, + 2584, + 2604, + 2606, + 2608, + 2611, + 2612, + 2613, + 2619, + 2620, + 2621, + 2622, + 2623, + 2624, + 2625, + 2626, + 2627, + 2628, + 2629, + 2631, + 2635, + 2636, + 2637, + 2646, + 2647, + 2649, + 2651, + 2652, + 2653, + 2654, + 2655, + 2656, + 2657, + 2658, + 2659, + 2661, + 2662, + 2664, + 2665, + 2667, + 2668, + 2670, + 2671, + 2673, + 2674, + 2676, + 2677, + 2679, + 2680, + 2681, + 2682, + 2683, + 2684, + 2685, + 2686, + 2687, + 2688, + 2689, + 2690, + 2691, + 2692, + 2693, + 2694, + 2695, + 2696, + 2697, + 2698, + 2699, + 2701, + 2702, + 2703, + 2704, + 2705, + 2706, + 2709, + 2715, + 2717, + 2718, + 2747, + 2749, + 2754, + 2756, + 2758, + 2763, + 2765, + 2771, + 2792, + 2795, + 2799, + 2811, + 2812, + 2855, + 2857, + 2859, + 2861, + 2889, + 2890, + 2892, + 2897, + 2898, + 2901, + 2902, + 2907, + 2912, + 2939, + 2940, + 3056, + 3057, + 3059, + 3060, + 3069, + 3070, + 3071, + 3072, + 3081, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136, + 3137, + 3138, + 3139, + 3140, + 3141, + 3142, + 3143, + 3144, + 3145, + 3146, + 3147, + 3148, + 3149, + 3150, + 3201, + 3202, + 3219, + 3220, + 3227, + 3228, + 3237, + 3238, + 3239, + 3264, + 3265, + 3270, + 3272, + 3273, + 3274, + 3275, + 3276, + 3277, + 3278, + 3279, + 3280, + 3281, + 3288, + 3289, + 3290, + 3291, + 3296, + 3297, + 3298, + 3299, + 3317, + 3318, + 3319, + 3320, + 3323, + 3348, + 3349, + 3350, + 3351, + 3352, + 3393, + 3394, + 3395, + 3396, + 3413, + 3452, + 3453, + 3454, + 3455, + 3456, + 3457, + 3458, + 3459, + 3503, + 3504, + 3505, + 3506, + 3507, + 3508, + 3509, + 3510, + 3511, + 3512, + 3513, + 3514, + 3518, + 3604, + 3747, + 3751, + 3752, + 3753, + 3754, + 3770, + 3776, + 3777, + 3778, + 3779, + 3780, + 3781, + 3782, + 3783, + 3784, + 3785, + 3790, + 3792, + 3793, + 3794, + 3795, + 3796, + 3797, + 3798, + 3799, + 3800, + 3801, + 3802, + 3803, + 3804, + 3805, + 3806, + 3807, + 3813, + 3814, + 3815, + 3816, + 3817, + 3818, + 3819, + 3820, + 3821, + 3822, + 3823, + 3824, + 3825, + 3826, + 3827, + 3828, + 3829, + 3830, + 3831, + 3832, + 3856, + 3857, + 3858, + 3859, + 3860, + 3861, + 3862, + 3863, + 3864, + 3868, + 3879, + 3903, + 3938, + 3955, + 3957, + 3959, + 3960, + 3961, + 3963, + 3964, + 4021, + 4022, + 4023, + 4024, + 4060, + 4062, + 4069, + 4098, + 4099, + 4100, + 4102, + 4103, + 4104, + 4106, + 4107, + 4109, + 4110, + 4112, + 4113, + 4115, + 4116, + 4118, + 4119, + 4141, + 4143, + 4144, + 4146, + 4147, + 4148, + 4164, + 4178, + 4180, + 4181, + 4183, + 4188, + 4189, + 4190, + 4191, + 4192, + 4194, + 4207, + 4217, + 4237, + 4240, + 4242, + 4346, + 4348, + 4349, + 4351, + 4353, + 4455, + 4456, + 4459, + 4461, + 4463, + 4465, + 4467, + 4488, + 4523, + 4525, + 4541, + 4542, + 4543, + 4545, + 4547, + 4549, + 4551, + 4553, + 4555, + 4557, + 4559, + 4561, + 4563, + 4564, + 4565, + 4567, + 4569, + 4570, + 4572, + 4573, + 4575, + 4577, + 4579, + 4581, + 4582, + 4583, + 4606, + 4607, + 4608, + 4610, + 4611, + 4612, + 4613, + 4615, + 4616, + 4618, + 4619, + 4621, + 4622, + 4624, + 4627, + 4628, + 4629, + 4631, + 4661, + 4669, + 4700, + 4704, + 4706, + 4707, + 4708, + 4709, + 4712, + 4714, + 4716, + 4717, + 4718, + 4720, + 4722, + 4724, + 4725, + 4742, + 4744, + 4745, + 4747, + 4748, + 4750, + 4752, + 4753, + 4755, + 4757, + 4761, + 4762, + 4763, + 4764, + 4771, + 4772, + 4773, + 4774, + 4817, + 4818, + 4819, + 4830, + 4898, + 4918, + 4920, + 4966, + 4977, + 4979, + 4980, + 4981, + 4988, + 4989, + 4996, + 4997, + 4998, + 5029, + 5031 + ] + } + }, + "sqlglot_postgres.json": { + "postgres": { + "statements": 568, + "refused": [ + 124 + ] + } + }, + "sqlglot_sqlite.json": { + "sqlite": { + "statements": 149, + "refused": [] + } + }, + "sqllogictest.json": { + "sqlite": { + "statements": 20392, + "refused": [] + } + } + } +} diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs index e56610976..1387b5bab 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs @@ -49,7 +49,6 @@ use sqlquack_syntax::expressions::typed_functions::models::ValueFunc; use sqlquack_syntax::expressions::types::models::DataType; use sqlquack_syntax::expressions::types::models::Tuple; use sqlquack_syntax::tokens::models::SourceSpan; -use sqlquack_syntax::tokens::models::Span; use sqlquack_syntax::tokens::models::TokenType; use sqlquack_syntax::tokens::types::Result; @@ -268,7 +267,7 @@ impl Parser { } let comments = self.call_comments(saved); let typed = self.parse_typed_function_lowered(name, upper_name, quoted)?; - let mut expr = match Self::call_name(&typed) { + let expr = match Self::call_name(&typed) { Some(kept) if kept.eq_ignore_ascii_case(name) => typed, _ if sqlquack_syntax::functions::names::main::faithful_keyword_syntax_name::faithful_keyword_syntax_name(upper_name) && self.tokens_use_keyword_syntax(saved) => @@ -290,9 +289,6 @@ impl Parser { } } }; - if self.config.record_call_spans { - self.record_call_span(&mut expr, saved); - } self.faithful_calls .insert(saved, (expr.clone(), self.current)); Ok(expr) @@ -351,30 +347,6 @@ impl Parser { } } - /// SQLQuack: records, on a call node, the source span from its name to its last token. - pub(crate) fn record_call_span(&self, expr: &mut Expression, arguments_start: usize) { - let Some(name_token) = arguments_start - .checked_sub(2) - .and_then(|index| self.tokens.get(index)) - else { - return; - }; - if self.tokens[arguments_start - 1].token_type != TokenType::LParen { - return; - } - let span = Span { - start: name_token.span.start, - end: self.tokens[self.current - 1].span.end, - line: 0, - column: 0, - }; - match expr { - Expression::Function(function) => function.span = SourceSpan(Some(span)), - Expression::AggregateFunction(function) => function.span = SourceSpan(Some(span)), - _ => {} - } - } - /// SQLQuack: whether arguments from `start` use top-level keywords, as `EXTRACT(YEAR FROM d)`. pub(crate) fn tokens_use_keyword_syntax(&self, start: usize) -> bool { let mut depth = 0_usize; diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/snowflake_context.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/snowflake_context.rs index 22193e1fe..cce6bb042 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/snowflake_context.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/snowflake_context.rs @@ -1,4 +1,4 @@ -//! Snowflake's statement-context rules, which the exact check's polyglot-matching modes skip. +//! Snowflake's statement-context rules, which the lowered mode skips. use crate::parser::models::Parser; use sqlquack_syntax::expressions::tree::models::Expression; @@ -9,7 +9,6 @@ impl Parser { pub(crate) fn applies_snowflake_context_rules(&self) -> bool { self.config.dialect == Some(sqlquack_syntax::tokens::models::DialectType::Snowflake) && self.config.faithful_calls - && !self.config.record_call_spans } /// Refuses a backtick-quoted identifier, which Snowflake does not tokenize, in its faithful modes. diff --git a/crates/sqlquack-syntax-snowflake/src/parser/models.rs b/crates/sqlquack-syntax-snowflake/src/parser/models.rs index e8fa6ff91..38a4cfe92 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/models.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/models.rs @@ -142,7 +142,6 @@ impl Dialect { dialect: Some(self.dialect_type), complexity_guard, faithful_calls: mode != CallMode::Lowered, - record_call_spans: mode == CallMode::FaithfulWithSpans, record_node_spans: mode == CallMode::FaithfulLocated, ..Default::default() }; @@ -440,8 +439,6 @@ pub(crate) struct ParserConfig { pub complexity_guard: ComplexityGuardOptions, /// SQLQuack: in Snowflake, keep calls as written; only keyword-syntax calls become typed nodes. pub faithful_calls: bool, - /// SQLQuack: record the source span of every call parsed faithfully (for the exact check). - pub record_call_spans: bool, /// SQLQuack: record each expression and statement node's source range (for locations). pub record_node_spans: bool, /// Whether a Snowflake CREATE SEMANTIC VIEW is being checked, where a window may PARTITION BY EXCLUDING. diff --git a/crates/sqlquack-syntax-snowflake/tests/parser/test_scripting.rs b/crates/sqlquack-syntax-snowflake/tests/parser/test_scripting.rs index c89ce7875..3f12ce90d 100644 --- a/crates/sqlquack-syntax-snowflake/tests/parser/test_scripting.rs +++ b/crates/sqlquack-syntax-snowflake/tests/parser/test_scripting.rs @@ -8,9 +8,8 @@ use crate::test_types::{ ScriptingVerdictTestCase, ScriptingWalkTestCase, }; -const MODES: [CallMode; 4] = [ +const MODES: [CallMode; 3] = [ CallMode::Faithful, - CallMode::FaithfulWithSpans, CallMode::FaithfulLocated, CallMode::Lowered, ]; diff --git a/crates/sqlquack-syntax/src/tokens/models.rs b/crates/sqlquack-syntax/src/tokens/models.rs index 8d22115ce..5d32e1d1b 100644 --- a/crates/sqlquack-syntax/src/tokens/models.rs +++ b/crates/sqlquack-syntax/src/tokens/models.rs @@ -46,9 +46,7 @@ pub enum Dialect { pub enum CallMode { /// Every Snowflake call keeps its name and arguments as written; SQLQuack's mode. Faithful, - /// As [`CallMode::Faithful`] with call spans, for the exact check; skips statement-context rules. - FaithfulWithSpans, - /// polyglot's call lowering, for the exact check only; skips statement-context rules. + /// polyglot's call lowering, used by the parser's own tests; skips statement-context rules. Lowered, /// As [`CallMode::Faithful`], recording each node's source range (for refusal locations). FaithfulLocated, diff --git a/crates/sqlquack/THIRD_PARTY_NOTICES b/crates/sqlquack/THIRD_PARTY_NOTICES index de5b44d4b..01a5f876e 100644 --- a/crates/sqlquack/THIRD_PARTY_NOTICES +++ b/crates/sqlquack/THIRD_PARTY_NOTICES @@ -11,7 +11,7 @@ answer from Snowflake itself. sqlglot's expected DuckDB output is kept in the `s field as reference side data only: it is never asserted, does not count toward verification and is not part of the case hash. -The exact-check corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test +The corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test fixtures at v30.14.0 (tests/fixtures/identity.sql, tests/test_parser.py and tests/dialects/), as extracted by polyglot's fixture tooling. polyglot, below, is itself inspired by sqlglot and ships sqlglot's licence. corpora/polyglot-check/sqlglot_postgres.json holds the SQL of @@ -60,7 +60,7 @@ polyglot SQLQuack's tokenizer, parser and syntax tree are a modified fork of polyglot (https://github.com/tobilg/polyglot), as forked at https://github.com/sqlstack-io/polyglot. The derived code is in crates/sqlquack-syntax-snowflake, which now holds most of it, -crates/sqlquack-syntax and crates/sqlquack-derive. The exact-check corpus +crates/sqlquack-syntax and crates/sqlquack-derive. The corpus corpora/polyglot-check/polyglot_tests.json holds the SQL inputs of polyglot's tests. SQLQuack's DuckDB result-type rules, in crates/sqlquack-semantics/src/duckdb/profile/_helpers/typing.rs, are ported from polyglot's engine type rules, and the fixture diff --git a/docs/architecture.md b/docs/architecture.md index e030e594e..f1a05538b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,7 +1,7 @@ # Architecture -The tokenizer, parser and syntax tree, and how they move from polyglot into SQLQuack, are -designed in [syntax.md](syntax.md). +The tokenizer, parser and syntax tree, and how they came from polyglot into SQLQuack, are +described in [syntax.md](syntax.md). ## Principle @@ -28,7 +28,7 @@ a parameter of every entry point; Snowflake is the only one implemented. | Crate | Role | |---|---| | `sqlquack-syntax` | The shared syntax tree, lossless tokens, spans, complexity guards and parse diagnostics that every dialect parser builds on. It names no dialect. Derived from polyglot; see [syntax.md](syntax.md). | -| `sqlquack-syntax-snowflake` | The Snowflake parser, with strict and tolerant modes, building the shared tree. It also reads DuckDB, PostgreSQL and SQLite as polyglot does. | +| `sqlquack-syntax-snowflake` | The Snowflake parser, with strict and tolerant modes, building the shared tree. It also reads DuckDB, PostgreSQL and SQLite. | | `sqlquack-derive` | The derive that generates the tree's child traversal. | | `sqlquack-syntax-bigquery`, `-clickhouse`, `-databricks`, `-duckdb`, `-postgres`, `-sqlite`, `-sqlserver` | Native parsers, one per dialect, each following its engine's own grammar and building the shared `sqlquack-syntax` tree. Each README lists what it covers and its gaps. | | `sqlquack-parse` | The one way the other crates reach a parser. The Snowflake parser's entry points are always there; each native parser is a module behind the feature of its dialect's name. | @@ -49,8 +49,7 @@ Development tools live under `tools/`. No shipped crate depends on them: | Crate | Role | |---|---| -| `sqlquack-polyglot-check` | Proves `sqlquack-syntax` parses exactly as polyglot does, on every corpus. | -| `sqlquack-coverage-check` | The syntax catalogues of every dialect root under `coverage/` checked against the parsers (binary `syntax-coverage`), the translation-results measure, the nesting-depth probe and the instruction-count guard. See [syntax-coverage.md](syntax-coverage.md). | +| `sqlquack-coverage-check` | The syntax catalogues of every dialect root under `coverage/` checked against the parsers (binary `syntax-coverage`), the translation-results measure, the nesting-depth probe, the instruction-count guard and the corpora's parse verdicts (`corpus-verdicts`). See [syntax-coverage.md](syntax-coverage.md). | | `sqlquack-format-check` | Proves that formatting DuckDB SQL leaves the output columns DuckDB binds it to unchanged, on the format corpus. | Fensu layer targets keep the rules out of every crate but the command line ones and @@ -70,8 +69,8 @@ matching prebuilt `libduckdb`, so binaries are run with `cargo run`, which sets The recorder, `sqlquack-record`, is a separate Python package under `recorder/`. Rust and Python share files only. -Polyglot is pinned to one revision with `default-features = false`. Its `transpile` feature stays -off: none of its own conversions are used. +No crate depends on polyglot. Real Snowflake results, not polyglot's behaviour, are the bar for the +Snowflake parser. ## Constructs @@ -81,18 +80,18 @@ A construct is a key `:`: - `fn:`: a call, by its upper-case name as written; the tree keeps every Snowflake call as written (see `docs/syntax.md`, step 5), so `fn:IFNULL` and `fn:COALESCE` are distinct; calls to project functions are one construct, `fn:`; -- `cast:`: the base target type of a cast, as polyglot prints it in the source dialect; +- `cast:`: the base target type of a cast, by its name in the source dialect (`cast_type_name`); - `join:`: a join kind; - `clause:`: a query clause or set-operation form, such as `clause:qualify` or `clause:union_all`; - `syntax:`: a spelling recorded separately, such as `syntax:double_colon_cast`. -Clause classification destructures polyglot's query structs field by field, so a polyglot upgrade -that adds a clause fails to compile until the clause is classified. +Clause classification destructures the tree's query structs field by field, so a change that adds a +clause fails to compile until the clause is classified. Each construct carries a source position. Source SQL is parsed located (`CallMode::FaithfulLocated`, see `docs/syntax.md`), so operators, casts, calls, predicates, aliases, subqueries, queries, joins, -CTEs and statements carry their own source range besides the names polyglot positions; any other +CTEs and statements carry their own source range besides the names the parser positions; any other node takes the first position inside it, or else its parent's. ## Refusals diff --git a/docs/syntax-notes.md b/docs/syntax-notes.md index c689c5f01..9332eac36 100644 --- a/docs/syntax-notes.md +++ b/docs/syntax-notes.md @@ -195,9 +195,8 @@ expression. SQLQuack's tokenizer, parser and syntax tree, for Snowflake and DuckDB. -The modules below the crate root are derived from polyglot (see `THIRD_PARTY_NOTICES`) and kept -close to it, so a dev-only check can prove that parse results are identical. Only Snowflake and -DuckDB are reachable through this API. +The modules below the crate root are derived from polyglot (see `THIRD_PARTY_NOTICES`). Only +Snowflake and DuckDB are reachable through this API. Every module is linted as the rest of the workspace. diff --git a/docs/syntax.md b/docs/syntax.md index 9ffde882f..29f3a0ada 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -1,12 +1,13 @@ # Syntax foundation: tokenizer, parser and tree Status: P1 Part A. Steps 1 to 5 are done: the tree keeps every Snowflake call as written, the -Snowflake print-and-reparse fidelity check is retired, and polyglot is a dependency of the -development-only check crate alone. +Snowflake print-and-reparse fidelity check is retired, and no crate depends on polyglot. The parser +is no longer compared with polyglot: real Snowflake results (the referee and the recorded receipts) +are the bar for the Snowflake parser, and the copied code diverges from polyglot as it is fixed. -SQLQuack owns its whole stack. This note fixes how the tokenizer, parser and syntax tree get into -SQLQuack, how we prove they behave exactly like the code they come from, and the order in which -they then become ours. Snowflake and DuckDB are the first analysis dialects. +SQLQuack owns its whole stack. This note records how the tokenizer, parser and syntax tree got into +SQLQuack, how their parse verdicts are held, and the order in which they became ours. Snowflake and +DuckDB are the first analysis dialects. ## Requirements @@ -20,7 +21,7 @@ they then become ours. Snowflake and DuckDB are the first analysis dialects. what it can statement by statement, keeps unparsed ranges as such, and never panics. 3. **A faithful tree.** Functions stay as written, a name plus arguments, with no cross-dialect canonicalisation at parse time. Meaning comes from SQLQuack's typing and rules. -4. **Speed and memory** at least as good as polyglot's, within sqlbuild's CI budgets. +4. **Speed and memory** within sqlbuild's CI budgets. ## Crate layout @@ -28,7 +29,6 @@ they then become ours. Snowflake and DuckDB are the first analysis dialects. |---|---|---| | `sqlquack-syntax` | the tokenizer (significant tokens and trivia), the parser (strict and tolerant), the tree, the Snowflake and DuckDB dialect settings | never | | `sqlquack-derive` | the proc macro that generates child traversal for tree nodes, copied from `polyglot-sql-ast-derive` | never | -| `sqlquack-polyglot-check` | dev-only tool and tests: the exact comparison with polyglot, corpus runs and parse timings; `publish = false`, a fensu tooling crate | yes, through the workspace pin | | `sqlquack-core` | parsing entry points re-pointed at `sqlquack-syntax`, plus classification, refusals and provenance as today | removed in step 5 | | `sqlquack-blocks` | building blocks, the DuckDB printer and DuckDB's own check of the printed SQL (step 4) | never | @@ -46,16 +46,12 @@ The tree's node types are about 12,000 lines, more than fensu's 2,000-line limit `typed_functions`, `types`, `tables`, `columns`, `schema_objects` and `statements`. Each area's `main/` holds a function the parser calls that builds or reshapes that area's nodes. -No runtime crate depends on `sqlquack-polyglot-check`. `make polyglot-guard`, part of `check-ci`, -reads Cargo's resolved graph and fails if a workspace crate outside an allowed set reaches polyglot -through normal (not dev or build) dependencies. Since step 5 the allowed set -(`scripts/polyglot_guard/constants.py`) is the check crate alone, so the `sqlquack` binary contains -no polyglot at all. +No crate depends on polyglot. ## Where the code comes from The tokenizer, parser and tree are copied from **sqlstack-io/polyglot `7d5fe2c`**, the commit -SQLQuack's polyglot pin moved to first, so the exact check compares like with like. It includes the +SQLQuack's polyglot pin had moved to first. It includes the fork's typed-marker lowering (`__polyglot_typed`, inert when the names are absent) and upstream 0.13.4's parser fixes. @@ -66,7 +62,7 @@ and `stacker` enabled: `tokens.rs`, `parser.rs`, `expressions.rs`, `error.rs`, ` transpiler, optimizer, lineage, validation, bindings and other dialects' modules stay behind. `lib.rs` is SQLQuack's own and exposes only Snowflake and DuckDB. -The copied files are byte-for-byte polyglot's except for these edits: +At the copy, the files were byte-for-byte polyglot's except for these edits: - the derive crate's name; - `mod` declarations of dialect modules that were not copied (they are never compiled, and @@ -76,10 +72,11 @@ The copied files are byte-for-byte polyglot's except for these edits: - in-file tests that need the generator or transpiler, or name another dialect, removed. Polyglot's other feature gates stay as written and are never enabled. `parser.rs` still holds -branches for other dialects; they stay, unreachable, until the strip step deletes them under the -exact check. Until the strip and split steps the copied modules allowed polyglot's lint level and -dead code on their declarations in `lib.rs`, and fensu skipped both copied crates; since the split -fensu checks them like the rest of the workspace. +branches for other dialects; they stay, unreachable, until the strip step deletes them. Until the +strip and split steps the copied modules allowed polyglot's lint level and dead code on their +declarations in `lib.rs`, and fensu skipped both copied crates; since the split fensu checks them +like the rest of the workspace. + **Licensing.** `THIRD_PARTY_NOTICES` holds polyglot's MIT text and copyright line next to sqlglot's, and says that parts of the tokenizer and parser are derived from polyglot. There are no @@ -90,29 +87,29 @@ commit. There are two options. -- **A. Copy the tree nearly as-is first, then change it in checked steps (recommended).** The - copied `Expression` keeps polyglot's variants, fields and serde shape, so the exact check is a - plain equality of the two serialised trees. Today's rules already work on the serialised tree - (`serde_json::Value` dispatched by node kind), so step 3 swaps the parser underneath them without - rewriting them. +- **A. Copy the tree nearly as-is first, then change it in checked steps (recommended).** The copied + `Expression` keeps polyglot's variants, fields and serde shape, so the exact check (since removed) + was a plain equality of the two serialised trees. Today's rules already work on the serialised + tree (`serde_json::Value` dispatched by node kind), so step 3 swaps the parser underneath them + without rewriting them. + - *Cost:* for a while we carry polyglot's warts: canonicalised function nodes, comment wrappers, uneven field names and about 600 variants. Each later step removes some of them under the check. - **B. Define our own tree now and convert polyglot's into it in one place.** The tree is clean - from day one. But the exact check then also tests a converter of about 600 variants, written - before we know which shapes the rules need, and all the rules must be rewritten in step 3. + from day one. But the exact check would then also have tested a converter of about 600 variants, + written before we knew which shapes the rules need, and all the rules must be rewritten in step 3. With A, every change after the copy is a small, checked diff. With B, the first diff is large and the check is weaker. A is also the route the plan already lays out: copy, strip, split, make faithful. -**After the copy, the tree diverges on purpose in this order:** +**After the copy, the tree diverged on purpose in this order:** -1. Spans move from Unicode character offsets (polyglot's) to byte offsets. The check compares - spans after converting units. -2. Functions become faithful calls (step 5, below). The check compares through an explicit, - reviewed list of expected differences, one entry per spelling. -3. Further cleanup (field names, comment wrappers) follows the same pattern. +1. Spans moved from Unicode character offsets (polyglot's) to byte offsets. +2. Functions became faithful calls (step 5, below). +3. Further cleanup (field names, comment wrappers) follows, along with fixes to statements polyglot + parses wrongly or not at all; real Snowflake results decide each one. **Two mechanisms retired with the faithful tree (step 5):** - the six proven parse-fidelity equivalences (RLIKE, `:` paths, POSITION, MOD, TEXT, FROM VALUES), @@ -131,9 +128,10 @@ layer around it produces the lossless stream: it walks the input and emits trivi - Each token carries its kind and its byte range; its text is a slice of the input. - Polyglot's string-literal text is processed (escapes resolved); the slice keeps the original. - The invariant "tokens joined back equal the input, byte for byte" is checked on every corpus - and by fuzzing (`fuzz/`, a cargo-fuzz target outside the workspace). CI's `fuzz` job runs the - tolerant-mode target on nightly for 90 seconds, seeded with the exact check's statements, and - keeps any failing input as an artifact. + statement by `make corpus-verdicts` and by fuzzing (`fuzz/`, a cargo-fuzz target outside the + workspace). CI's `fuzz` job runs the tolerant-mode target on nightly for 90 seconds, seeded with + the corpora's statements, and keeps any failing input as an artifact. + - Gap text that is neither whitespace nor a comment, or overlapping tokens, is an error. **Parse modes.** `parse_strict` refuses on any parse error or any byte the lossless tokens do not @@ -141,107 +139,50 @@ cover. Once the tree carries byte spans, it will also check that they cover ever token exactly once. `parse_tolerant` always returns the tokens, splits statements at top-level semicolons, parses each on its own, and keeps the ranges it cannot parse; it never panics. -## The exact check against polyglot - -`sqlquack-polyglot-check` parses each statement with both polyglot `7d5fe2c` and -`sqlquack-syntax`, in the same dialect, and compares: - -1. whether each succeeded; -2. for errors: the kind, the message and the position (the error's full debug form); -3. for trees: the serde serialisations, which must be equal. - -It also asserts the lossless-token invariant, and times both parsers on the same inputs -(`--timing-rounds`: medians over whole-corpus passes, alternating which parser goes first). It -runs as an integration test on the committed corpora, as a CI job on DuckShop, and as a command -for local corpora (`--counts-only` prints no SQL, trees or file names). - -During steps 2 to 4 the bar is zero differences. From step 5 the check runs both of SQLQuack's -call modes: the lowered mode (`CallMode::Lowered`, polyglot's lowering, kept for the check only) -must still give zero differences from polyglot, and the faithful mode must differ from the lowered -one only as `corpora/polyglot-check/expected_differences.toml` allows: - -- the faithful parse records each kept call's source span (`CallMode::FaithfulWithSpans`); -- top-down, each kept call that the lowered mode parses differently from its own source text is - replaced by that lowered parse, and the pair (written name, polyglot's node) is recorded; - calls the lowered mode also keeps are descended into; -- the result must equal the lowered tree exactly, and every recorded pair must be listed; -- the only other accepted difference is comment placement: a kept call holds its trailing comment - itself, where polyglot's caller places a lowered node's comment, sometimes twice. The trees must - be equal without comments, and the faithful tree must keep every distinct comment (the lossless - tokens hold each comment exactly once). - -Inside a lowered call the check compares the whole call through polyglot's own parse of its text, -not argument by argument; arguments are parsed by the same code in both modes. - -SQLQuack's pipeline parses in a third mode, `CallMode::FaithfulLocated`: the faithful parse that -also records each node's source range, so refusals and runtime raises can say where they come -from. Ranges live in `span` fields of type `SourceSpan` on the node payloads that carry one -(operators, casts, CASE, predicates, aliases, subqueries, queries, joins and CTEs, besides the -names and calls that had spans already); the parser fills them at the expression levels, select -items, table expressions and statements. A `SourceSpan` is left out of serialised trees when -absent and is ignored by equality, and a located parse never changes the spans the parser keeps -on names, so the located tree equals the faithful one. The check proves it on every corpus: each -entry's located parse, without `span` fields, must equal its faithful parse. - -SQLQuack also fixes valid Snowflake that polyglot rejects. Each fix is a `[[fix]]` entry in the -same file: an id, a marker the statement contains (whitespace removed) and polyglot's errors on it. -The check counts a statement as fixed, not different, only when SQLQuack parses it, polyglot's -error is one listed, and the marker appears; `crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs` pins the tree -each fix produces. The first fix: `v[3]:a`, a key path on an array element. - -The opposite case, a statement SQLQuack rejects and polyglot parses, is a `[[rejection]]` entry: -an id, the exact statement, its dialects and text of SQLQuack's error. The check counts it as -rejected, not different, only for that statement in a listed dialect with that error. The only -rejection: `__polyglot_typed(amount, 'NOT A TYPE (')`, a generated typed-expression marker whose -type text is malformed, errors instead of becoming an ordinary call. - -A statement both parsers parse, where polyglot drops the quoting of a name SQLQuack keeps quoted, -is a `[[kept_quoting]]` entry: an id, a marker the statement contains (whitespace removed) and a -reason. The check counts it as kept quoting, not different, only when the marker appears and the -two trees differ in nothing but identifiers that are unquoted in polyglot's tree and quoted in -SQLQuack's. The entries: a star's `EXCLUDE ("note")` and a `COMMENT ON` name, whose quotes make -the name case-sensitive. - -A statement both parsers parse, where SQLQuack keeps a wrapper node polyglot drops, is a -`[[kept_node]]` entry: an id, a marker, the node and a reason. The check counts it as a kept node, -not different, only when the marker appears and the trees are equal once every -`{ "": { "this": … } }` in SQLQuack's tree is replaced by its operand. The only entry: unary -plus (`pos`), which Snowflake documents and polyglot drops. - -A statement both parsers parse, where SQLQuack keeps a field as written that polyglot drops or -infers, is a `[[kept_field]]` entry: an id, a marker, the field and a reason. The check counts it as -a kept field, not different, only when the marker appears, the trees differ, and they are equal once -every `` key is removed from both. The entries: the role list of `USE SECONDARY ROLES`, which -polyglot joins into one name; the container kind of `SHOW ... IN a.b`, which polyglot infers as -SCHEMA; and the cast target of `TIMESTAMP WITHOUT TIME ZONE`, which Snowflake documents as -TIMESTAMP_NTZ and polyglot reads as bare TIMESTAMP. - -A clause whose `ALL` SQLQuack keeps apart from its keys, where polyglot reads `ALL` as a key named -ALL, is a `[[kept_all]]` entry: an id, a marker, the clause and a reason. The check counts it with the -kept fields only when the marker appears, the trees differ, and they are equal once SQLQuack's -clause with no keys and an `all` is rewritten as polyglot's one key `ALL` with the same DESC and -NULLS options. The only entry: `ORDER BY ALL`. - -The faithful parse also keeps a quoted variant path key that holds `.`, `[`, `]`, `"` or `:` in -brackets (`v:a."x.y"` is the path `a["x.y"]`), where polyglot joins it with dots (`a.x.y`), which -reads as two keys. The check compares variant paths written flat, so this bracketing is the only -difference it allows there; `crates/sqlquack-syntax-snowflake/tests/parser/test_faithful.rs` pins the paths. +## Parse verdicts + +`coverage/syntax/corpus-verdicts.json` records SQLQuack's verdict on every statement of the corpora +under `corpora/polyglot-check`: per file and dialect, how many statements it holds and the positions +of those SQLQuack refuses. An entry for `both` dialects is parsed as Snowflake and as DuckDB. `make +corpus-verdicts`, part of `check-ci`, parses them all again and fails on any change, naming each +statement whose verdict moved. It also fails on any statement whose lossless tokens do not rebuild +it, or whose located parse differs from its faithful parse once spans are removed. A change that +moves a verdict on purpose records it with `make corpus-verdicts-baseline` and commits the file in +the same pull request. Trees are not recorded: `crates/sqlquack-syntax-snowflake/tests/parser/` pins +the trees that matter, and the recorded cases replay every conversion against real Snowflake +results. + + +SQLQuack's pipeline parses in `CallMode::FaithfulLocated`: the faithful parse that also records +each node's source range, so refusals and runtime raises can say where they come from. Ranges live +in `span` fields of type `SourceSpan` on the node payloads that carry one (operators, casts, CASE, +predicates, aliases, subqueries, queries, joins and CTEs, besides the names and calls that had spans +already); the parser fills them at the expression levels, select items, table expressions and +statements. A `SourceSpan` is left out of serialised trees when absent and is ignored by equality, +and a located parse never changes the spans the parser keeps on names, so the located tree equals +the faithful one; `make corpus-verdicts` checks it on every corpus statement. + +Where polyglot parses valid Snowflake or DuckDB wrongly or not at all, SQLQuack fixes it, with a +test in `crates/sqlquack-syntax-snowflake/tests/parser/test_fixes.rs` pinning the tree each fix +produces, such as `v[3]:a`, a key path on an array element. The faithful tree also keeps what +polyglot drops or infers: the quoting of a star's `EXCLUDE ("note")` and of `COMMENT ON` names, +unary plus, the role list of `USE SECONDARY ROLES`, the container kind of `SHOW ... IN a.b`, +`TIMESTAMP WITHOUT TIME ZONE` as TIMESTAMP_NTZ, `ORDER BY ALL` apart from its keys, and a quoted +variant path key that holds `.`, `[`, `]`, `"` or `:` in brackets (`v:a."x.y"` is the path +`a["x.y"]`), which `crates/sqlquack-syntax-snowflake/tests/parser/test_faithful.rs` pins. ## Corpora -| Corpus | Dialect | Where it runs | -|---|---|---| -| SQLQuack's cases (source SQL and converted DuckDB) | Snowflake, DuckDB | CI, `cargo test` | -| sqlglot v30.14.0's identity, parser and dialect fixtures: every Snowflake and DuckDB statement, generic ones in both (`corpora/polyglot-check/sqlglot.json`) | Snowflake, DuckDB | CI, `cargo test` | -| The SQL inputs of polyglot's own tests, whatever they were written for, in both dialects (`corpora/polyglot-check/polyglot_tests.json`) | Snowflake, DuckDB | CI, `cargo test` | -| sqlglot v30.14.0's PostgreSQL dialect tests (`corpora/polyglot-check/sqlglot_postgres.json`) | PostgreSQL | CI, `cargo test` | -| sqlglot v30.14.0's SQLite dialect tests (`corpora/polyglot-check/sqlglot_sqlite.json`) | SQLite | CI, `cargo test` | -| sqllogictest's evidence statements and a seeded sample of the rest (`corpora/polyglot-check/sqllogictest.json`, MIT) | SQLite | CI, `cargo test` | -| All 5,719,877 distinct sqllogictest statements SQLite runs | SQLite | locally only; counts in the change notes | -| PostgreSQL 17.2's regression SQL (`src/test/regress/sql`), split into statements (`corpora/polyglot-check/postgres_regress.json`, PostgreSQL License) | PostgreSQL | CI, `cargo test` | -| DuckShop (`benchmarks/duckshop` in sqlbuild), compiled with `sqb compile`: models, audits, tests and functions | Snowflake | CI job `polyglot-check`, at a pinned sqlbuild commit and release | -| DuckShop 10K (`benchmarks/duckshop-10k`, generated, then compiled) | Snowflake | the same job | -| The production project, compiled on a local copy | Snowflake | locally only; aggregate counts only, never committed | +The parse verdicts cover these corpora; the fuzzer is seeded from them. + +| Corpus | Dialect | +|---|---| +| sqlglot v30.14.0's identity, parser and dialect fixtures: every Snowflake and DuckDB statement, generic ones in both (`corpora/polyglot-check/sqlglot.json`) | Snowflake, DuckDB | +| The SQL inputs of polyglot's own tests, whatever they were written for, in both dialects (`corpora/polyglot-check/polyglot_tests.json`) | Snowflake, DuckDB | +| sqlglot v30.14.0's PostgreSQL dialect tests (`corpora/polyglot-check/sqlglot_postgres.json`) | PostgreSQL | +| sqlglot v30.14.0's SQLite dialect tests (`corpora/polyglot-check/sqlglot_sqlite.json`) | SQLite | +| sqllogictest's evidence statements and a seeded sample of the rest (`corpora/polyglot-check/sqllogictest.json`, MIT) | SQLite | +| PostgreSQL 17.2's regression SQL (`src/test/regress/sql`), split into statements (`corpora/polyglot-check/postgres_regress.json`, PostgreSQL License) | PostgreSQL | ## Tests ported from polyglot @@ -260,8 +201,8 @@ integration tests, through a small `compat` module standing in for polyglot's ro SQLQuack prints no Snowflake. Every SQL string in the tests that need the generator, in `identity_roundtrip`, `prefix_not_regression`, `set_operation_subquery_operand_regression`, `create_view_security_comment_regression` and `is_predicate_precedence`, and in the sqlglot -fixtures `sqlglot_identity`, `sqlglot_dialect_identity` and `sqlglot_parser` read, is instead an -exact-check corpus entry: SQLQuack must parse it exactly as polyglot does. +fixtures `sqlglot_identity`, `sqlglot_dialect_identity` and `sqlglot_parser` read, is instead a +corpus entry, whose parse verdict is recorded. **Leave behind:** - every transpile test (`sqlglot_transpile*`, `sqlglot_compat` transpile cases, @@ -303,9 +244,9 @@ when the analysis API needs them. the census once its completion pass bound thousands of statements. The engine is never closed, since a thread-local engine closing during process exit killed processes intermittently. The re-parse of emitted SQL and the unused `check_duckdb_print` are gone. - - Done: cast type names in classification are SQLQuack's own (`cast_type_name`); the exact - check proves them equal to polyglot's printed base names on every cast in every corpus, - and on every Snowflake type spelling in each cast form (`snowflake_types.json`). + - Done: cast type names in classification are SQLQuack's own (`cast_type_name`), proven equal + to polyglot's printed base names on every cast in every corpus, and on every Snowflake type + spelling in each cast form (`snowflake_types.json`), while polyglot was still compared. - The Snowflake print-and-reparse fidelity check, the last runtime use of polyglot, stayed until the tree was faithful, so no rule was added on a tree that still canonicalised; it retired in step 5. @@ -338,6 +279,8 @@ when the analysis API needs them. requires both spellings of every pair to record one result, and a rule that converts both spellings of a pair records it in its provenance (`RLIKE`, and `TEXT` and `STRING` cast targets). The census loses its print-back section; files the check refused are refused by other rules. - `make polyglot-guard` allows the check crate only. + The comparison with polyglot itself, and with it the polyglot dependency, has since been + removed. -After step 5, under the exact check: strip other dialects and split into fensu-clean modules. +After step 5: strip other dialects and split into fensu-clean modules, held by the parse verdicts, +the parser tests and the recorded cases. diff --git a/fensu.toml b/fensu.toml index fb6db2695..94d7e8741 100644 --- a/fensu.toml +++ b/fensu.toml @@ -10,7 +10,7 @@ select = ["FF", "XSQ005", "XSQ006"] analyzer = "rust" roots = ["crates", "tools"] tests = [] -tooling = ["tools/sqlquack-polyglot-check"] +tooling = [] rule_packs = ["rust"] rule_modules = [ "scripts.fensu_policy.rules.crate_layout", @@ -85,7 +85,6 @@ tests = [] tooling = [ "crates/sqlquack", "crates/sqlquack-convert", - "tools/sqlquack-polyglot-check", "crates/sqlquack-python", ] rule_packs = ["rust"] @@ -102,7 +101,6 @@ tests = [] tooling = [ "crates/sqlquack", "crates/sqlquack-convert", - "tools/sqlquack-polyglot-check", "crates/sqlquack-python", "crates/sqlquack-blocks", "crates/sqlquack-evidence", @@ -124,5 +122,4 @@ forbidden_packages = [ "duckdb", "libduckdb-sys", "arrow", - "polyglot-sql", ] diff --git a/python/THIRD_PARTY_NOTICES b/python/THIRD_PARTY_NOTICES index de5b44d4b..01a5f876e 100644 --- a/python/THIRD_PARTY_NOTICES +++ b/python/THIRD_PARTY_NOTICES @@ -11,7 +11,7 @@ answer from Snowflake itself. sqlglot's expected DuckDB output is kept in the `s field as reference side data only: it is never asserted, does not count toward verification and is not part of the case hash. -The exact-check corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test +The corpus corpora/polyglot-check/sqlglot.json holds the SQL of sqlglot's test fixtures at v30.14.0 (tests/fixtures/identity.sql, tests/test_parser.py and tests/dialects/), as extracted by polyglot's fixture tooling. polyglot, below, is itself inspired by sqlglot and ships sqlglot's licence. corpora/polyglot-check/sqlglot_postgres.json holds the SQL of @@ -60,7 +60,7 @@ polyglot SQLQuack's tokenizer, parser and syntax tree are a modified fork of polyglot (https://github.com/tobilg/polyglot), as forked at https://github.com/sqlstack-io/polyglot. The derived code is in crates/sqlquack-syntax-snowflake, which now holds most of it, -crates/sqlquack-syntax and crates/sqlquack-derive. The exact-check corpus +crates/sqlquack-syntax and crates/sqlquack-derive. The corpus corpora/polyglot-check/polyglot_tests.json holds the SQL inputs of polyglot's tests. SQLQuack's DuckDB result-type rules, in crates/sqlquack-semantics/src/duckdb/profile/_helpers/typing.rs, are ported from polyglot's engine type rules, and the fixture diff --git a/scripts/check_polyglot_guard.py b/scripts/check_polyglot_guard.py deleted file mode 100644 index 0493cb63e..000000000 --- a/scripts/check_polyglot_guard.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Direct wrapper for the polyglot runtime-dependency guard.""" - -from __future__ import annotations - -from scripts.polyglot_guard.main.check import run_polyglot_guard - - -def main(argv: list[str] | None = None) -> int: - """Fail when a crate other than the dev-only check reaches polyglot at runtime.""" - return run_polyglot_guard(argv) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/fensu/constants.py b/scripts/fensu/constants.py index fda3aebde..6266c7977 100644 --- a/scripts/fensu/constants.py +++ b/scripts/fensu/constants.py @@ -53,7 +53,6 @@ ("crates/sqlquack-diagnostics/src", "sqlquack_syntax", True), ("crates/sqlquack-convert/src", "sqlquack_syntax", True), ("crates/sqlquack-blocks/src", "sqlquack_syntax", True), - ("tools/sqlquack-polyglot-check/src", "sqlquack_syntax", True), ("crates/sqlquack/src", "sqlquack_syntax", True), ("fuzz/fuzz_targets", "sqlquack_syntax", True), ("crates/sqlquack-derive/src", "crate", False), diff --git a/scripts/fuzz_corpus/__init__.py b/scripts/fuzz_corpus/__init__.py index 0979f28fe..059658f1b 100644 --- a/scripts/fuzz_corpus/__init__.py +++ b/scripts/fuzz_corpus/__init__.py @@ -1 +1 @@ -"""Seeding the fuzz corpus with the exact check's statements.""" +"""Seeding the fuzz corpus with the corpora's statements.""" diff --git a/scripts/fuzz_corpus/constants.py b/scripts/fuzz_corpus/constants.py index 6053974d6..1239bb32f 100644 --- a/scripts/fuzz_corpus/constants.py +++ b/scripts/fuzz_corpus/constants.py @@ -1,4 +1,4 @@ -"""Where the exact check's statements are read from.""" +"""Where the corpora's statements are read from.""" from __future__ import annotations diff --git a/scripts/fuzz_corpus/main/seed.py b/scripts/fuzz_corpus/main/seed.py index c540fe1e8..19481b1d8 100644 --- a/scripts/fuzz_corpus/main/seed.py +++ b/scripts/fuzz_corpus/main/seed.py @@ -12,7 +12,7 @@ def run_seed(argv: list[str] | None = None) -> int: - """Write each exact-check statement to the corpus directory, named by its SHA-1.""" + """Write each corpus statement to the corpus directory, named by its SHA-1.""" parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repository", type=Path, default=Path.cwd()) parser.add_argument("--out", type=Path, required=True) diff --git a/scripts/polyglot_guard/__init__.py b/scripts/polyglot_guard/__init__.py deleted file mode 100644 index d158423ae..000000000 --- a/scripts/polyglot_guard/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Guard that keeps polyglot out of SQLQuack's runtime dependencies.""" diff --git a/scripts/polyglot_guard/_helpers/__init__.py b/scripts/polyglot_guard/_helpers/__init__.py deleted file mode 100644 index dd0df9b57..000000000 --- a/scripts/polyglot_guard/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Guard helpers.""" diff --git a/scripts/polyglot_guard/_helpers/metadata.py b/scripts/polyglot_guard/_helpers/metadata.py deleted file mode 100644 index ca64f3756..000000000 --- a/scripts/polyglot_guard/_helpers/metadata.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Normal-dependency reachability from Cargo's resolved graph.""" - -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -from scripts.polyglot_guard.types import Metadata, NodeDependency, ResolveNode - - -def cargo_metadata(*, repository: Path) -> Metadata: - """Cargo's resolved dependency graph for the workspace, from the lockfile.""" - output: str = subprocess.run( - ["cargo", "metadata", "--format-version", "1", "--locked"], - cwd=repository, - check=True, - capture_output=True, - text=True, - ).stdout - metadata: Metadata = json.loads(output) - return metadata - - -def dependents_of(*, metadata: Metadata, package: str) -> set[str]: - """Workspace members that reach `package` through normal dependencies, not dev or build.""" - names: dict[str, str] = {entry["id"]: entry["name"] for entry in metadata["packages"]} - edges: dict[str, list[str]] = { - node["id"]: _normal_dependencies(node=node) for node in metadata["resolve"]["nodes"] - } - reaching: set[str] = set() - for member in metadata["workspace_members"]: - if _reaches(start=member, package=package, edges=edges, names=names): - reaching.add(names[member]) - return reaching - - -def _normal_dependencies(*, node: ResolveNode) -> list[str]: - return [dependency["pkg"] for dependency in node["deps"] if _is_normal(dependency=dependency)] - - -def _is_normal(*, dependency: NodeDependency) -> bool: - return any(kind["kind"] is None for kind in dependency["dep_kinds"]) - - -def _reaches( - *, start: str, package: str, edges: dict[str, list[str]], names: dict[str, str] -) -> bool: - seen: set[str] = {start} - pending: list[str] = [start] - while pending: - for dependency in edges.get(pending.pop(), []): - if names[dependency] == package: - return True - if dependency not in seen: - seen.add(dependency) - pending.append(dependency) - return False diff --git a/scripts/polyglot_guard/constants.py b/scripts/polyglot_guard/constants.py deleted file mode 100644 index 376f3e1aa..000000000 --- a/scripts/polyglot_guard/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Which workspace crates may reach polyglot: only the check crate, as docs/syntax.md explains.""" - -from __future__ import annotations - -POLYGLOT_PACKAGE: str = "polyglot-sql" -CHECK_CRATE: str = "sqlquack-polyglot-check" -ALLOWED_DEPENDENTS: frozenset[str] = frozenset({CHECK_CRATE}) diff --git a/scripts/polyglot_guard/main/__init__.py b/scripts/polyglot_guard/main/__init__.py deleted file mode 100644 index 6e5e89b28..000000000 --- a/scripts/polyglot_guard/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Guard entry points.""" diff --git a/scripts/polyglot_guard/main/check.py b/scripts/polyglot_guard/main/check.py deleted file mode 100644 index 6d574d98b..000000000 --- a/scripts/polyglot_guard/main/check.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Polyglot guard entry point.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -from scripts.polyglot_guard._helpers.metadata import cargo_metadata, dependents_of -from scripts.polyglot_guard.constants import ALLOWED_DEPENDENTS, POLYGLOT_PACKAGE -from scripts.polyglot_guard.types import Metadata - - -def run_polyglot_guard(argv: list[str] | None = None) -> int: - """Fail when a workspace crate outside the allowed set reaches polyglot at runtime.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repository", type=Path, default=Path.cwd()) - parser.add_argument( - "--metadata", type=Path, help="cargo metadata JSON instead of running cargo" - ) - arguments: argparse.Namespace = parser.parse_args(argv) - metadata: Metadata = ( - json.loads(arguments.metadata.read_text()) - if arguments.metadata - else cargo_metadata(repository=arguments.repository) - ) - dependents: set[str] = dependents_of(metadata=metadata, package=POLYGLOT_PACKAGE) - unexpected: list[str] = sorted(dependents - ALLOWED_DEPENDENTS) - for crate in unexpected: - print(f"polyglot guard: {crate} depends on {POLYGLOT_PACKAGE} at runtime", file=sys.stderr) - print( - f"polyglot guard: {'FAILED' if unexpected else 'passed'}, " - f"{len(dependents)} crates reach {POLYGLOT_PACKAGE}: {', '.join(sorted(dependents))}", - file=sys.stderr, - ) - return 1 if unexpected else 0 diff --git a/scripts/polyglot_guard/types.py b/scripts/polyglot_guard/types.py deleted file mode 100644 index 1c8a7b9ee..000000000 --- a/scripts/polyglot_guard/types.py +++ /dev/null @@ -1,46 +0,0 @@ -"""The parts of `cargo metadata` output the guard reads.""" - -from __future__ import annotations - -from typing import TypedDict - - -class DependencyKind(TypedDict): - """`null` for a normal dependency, otherwise `dev` or `build`.""" - - kind: str | None - - -class NodeDependency(TypedDict): - """One resolved dependency of a package.""" - - pkg: str - dep_kinds: list[DependencyKind] - - -class ResolveNode(TypedDict): - """A package in the resolved graph.""" - - id: str - deps: list[NodeDependency] - - -class Resolve(TypedDict): - """The resolved dependency graph.""" - - nodes: list[ResolveNode] - - -class Package(TypedDict): - """A package and its name.""" - - id: str - name: str - - -class Metadata(TypedDict): - """`cargo metadata --format-version 1` output.""" - - packages: list[Package] - workspace_members: list[str] - resolve: Resolve diff --git a/scripts/run_rust_tests.sh b/scripts/run_rust_tests.sh index 99f7c942d..5875c6dba 100755 --- a/scripts/run_rust_tests.sh +++ b/scripts/run_rust_tests.sh @@ -15,7 +15,7 @@ if [ -n "${RUST_TEST_FEATURES:-}" ]; then features=(--features "$RUST_TEST_FEATU # The fewest test binaries the workspace builds; fewer means tests went missing, not that they pass. minimum=${RUST_TEST_MIN_BINARIES:-21} # Started first, so the longest binaries overlap the rest. -slowest="cli runs index" +slowest="cli index" work=$(mktemp -d) trap 'rm -rf "$work"' EXIT diff --git a/scripts/seed_fuzz_corpus.py b/scripts/seed_fuzz_corpus.py index 0138b2dba..6d43cbfa3 100644 --- a/scripts/seed_fuzz_corpus.py +++ b/scripts/seed_fuzz_corpus.py @@ -6,7 +6,7 @@ def main(argv: list[str] | None = None) -> int: - """Seed the fuzz corpus with the exact check's statements.""" + """Seed the fuzz corpus with the corpora's statements.""" return run_seed(argv) diff --git a/tests/unit/scripts/polyglot_guard/__init__.py b/tests/unit/scripts/polyglot_guard/__init__.py deleted file mode 100644 index cfbb18d66..000000000 --- a/tests/unit/scripts/polyglot_guard/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Polyglot guard tests.""" diff --git a/tests/unit/scripts/polyglot_guard/_test_types.py b/tests/unit/scripts/polyglot_guard/_test_types.py deleted file mode 100644 index 923154e01..000000000 --- a/tests/unit/scripts/polyglot_guard/_test_types.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Test types for the polyglot guard.""" - -from __future__ import annotations - -from dataclasses import dataclass - -type Edge = tuple[str, str, str | None] - - -@dataclass(frozen=True) -class GuardTestCase: - description: str - members: tuple[str, ...] - edges: tuple[Edge, ...] - expected_return_code: int - expected_fragment: str diff --git a/tests/unit/scripts/polyglot_guard/helpers.py b/tests/unit/scripts/polyglot_guard/helpers.py deleted file mode 100644 index 564c6fd69..000000000 --- a/tests/unit/scripts/polyglot_guard/helpers.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Synthetic cargo metadata for the polyglot guard.""" - -from __future__ import annotations - -import json -from collections import defaultdict -from pathlib import Path - -from tests.unit.scripts.polyglot_guard._test_types import Edge - - -def write_metadata(*, directory: Path, members: tuple[str, ...], edges: tuple[Edge, ...]) -> Path: - """A cargo metadata file with `members` as workspace members and `edges` as the graph.""" - dependencies: defaultdict[str, list[dict[str, object]]] = defaultdict(list) - names: set[str] = set(members) - for dependent, dependency, kind in edges: - dependencies[dependent].append({"pkg": dependency, "dep_kinds": [{"kind": kind}]}) - names.update((dependent, dependency)) - metadata: dict[str, object] = { - "packages": [{"id": name, "name": name} for name in sorted(names)], - "workspace_members": list(members), - "resolve": {"nodes": [{"id": name, "deps": dependencies[name]} for name in sorted(names)]}, - } - path: Path = directory / "metadata.json" - path.write_text(json.dumps(metadata)) - return path diff --git a/tests/unit/scripts/polyglot_guard/test_check.py b/tests/unit/scripts/polyglot_guard/test_check.py deleted file mode 100644 index a81550c7a..000000000 --- a/tests/unit/scripts/polyglot_guard/test_check.py +++ /dev/null @@ -1,63 +0,0 @@ -"""The polyglot guard against synthetic dependency graphs.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.polyglot_guard.main.check import run_polyglot_guard -from tests.unit.scripts.polyglot_guard._test_types import GuardTestCase -from tests.unit.scripts.polyglot_guard.helpers import write_metadata - - -@pytest.mark.parametrize( - "test_case", - ( - GuardTestCase( - description="the check crate may depend on polyglot", - members=("sqlquack-polyglot-check",), - edges=(("sqlquack-polyglot-check", "polyglot-sql", None),), - expected_return_code=0, - expected_fragment="passed, 1 crates reach polyglot-sql", - ), - GuardTestCase( - description="a dev-dependency does not count", - members=("sqlquack-syntax",), - edges=(("sqlquack-syntax", "polyglot-sql", "dev"),), - expected_return_code=0, - expected_fragment="passed, 0 crates", - ), - GuardTestCase( - description="the syntax crate must never reach polyglot", - members=("sqlquack-syntax",), - edges=(("sqlquack-syntax", "polyglot-sql", None),), - expected_return_code=1, - expected_fragment="sqlquack-syntax depends on polyglot-sql", - ), - GuardTestCase( - description="the core crate no longer reaches polyglot", - members=("sqlquack-core",), - edges=(("sqlquack-core", "polyglot-sql", None),), - expected_return_code=1, - expected_fragment="sqlquack-core depends on polyglot-sql", - ), - GuardTestCase( - description="a transitive runtime path counts", - members=("sqlquack-policy",), - edges=(("sqlquack-policy", "middle", None), ("middle", "polyglot-sql", None)), - expected_return_code=1, - expected_fragment="sqlquack-policy depends on polyglot-sql", - ), - ), - ids=lambda case: case.description, -) -def test_given_dependency_graph_when_guarding_then_only_allowed_crates_reach_polyglot( - test_case: GuardTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - metadata: Path = write_metadata( - directory=tmp_path, members=test_case.members, edges=test_case.edges - ) - return_code: int = run_polyglot_guard(["--metadata", str(metadata)]) - assert return_code == test_case.expected_return_code - assert test_case.expected_fragment in capsys.readouterr().err diff --git a/tools/sqlquack-coverage-check/Cargo.toml b/tools/sqlquack-coverage-check/Cargo.toml index 7c838f62c..e46e102c9 100644 --- a/tools/sqlquack-coverage-check/Cargo.toml +++ b/tools/sqlquack-coverage-check/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqlquack-coverage-check" version.workspace = true -description = "Development check of each dialect's documented syntax coverage, the translation results, the nesting-depth probe and the instruction-count guard; never a runtime dependency." +description = "Development check of each dialect's documented syntax coverage, the translation results, the nesting-depth probe, the instruction-count guard and the corpora's parse verdicts; never a runtime dependency." edition.workspace = true license.workspace = true repository.workspace = true diff --git a/tools/sqlquack-coverage-check/src/cli/_helpers/mod.rs b/tools/sqlquack-coverage-check/src/cli/_helpers/mod.rs index a9cacfc2f..fff28d215 100644 --- a/tools/sqlquack-coverage-check/src/cli/_helpers/mod.rs +++ b/tools/sqlquack-coverage-check/src/cli/_helpers/mod.rs @@ -5,4 +5,5 @@ pub(crate) mod local; pub(crate) mod output; pub(crate) mod results_command; pub(crate) mod rustc_version; +pub(crate) mod verdicts_command; pub(crate) mod worker; diff --git a/tools/sqlquack-coverage-check/src/cli/_helpers/verdicts_command.rs b/tools/sqlquack-coverage-check/src/cli/_helpers/verdicts_command.rs new file mode 100644 index 000000000..bb98abceb --- /dev/null +++ b/tools/sqlquack-coverage-check/src/cli/_helpers/verdicts_command.rs @@ -0,0 +1,81 @@ +//! `corpus-verdicts`: the corpora's parse verdicts against the recorded ones. + +use std::path::Path; +use std::process::ExitCode; + +use crate::cli::_helpers::output::say; +use crate::verdicts::constants::SHOWN_CHANGES; +use crate::verdicts::main::changed_verdicts::changed_verdicts; +use crate::verdicts::main::current_verdicts::current_verdicts; +use crate::verdicts::models::{CorpusParse, Verdicts}; + +fn failed(problem: &str) -> ExitCode { + say(&format!("FAILED, {problem}")); + ExitCode::FAILURE +} + +fn write_verdicts(path: &Path, verdicts: &Verdicts) -> Result { + let text: String = serde_json::to_string_pretty(verdicts) + .map_err(|error| failed(&format!("the verdicts do not serialise: {error}")))?; + std::fs::write(path, text + "\n") + .map_err(|error| failed(&format!("{}: {error}", path.display())))?; + say(&format!("recorded the verdicts in {}", path.display())); + Ok(ExitCode::SUCCESS) +} + +pub(crate) fn corpus_verdicts( + corpora: &Path, + verdicts_path: &Path, + update: bool, +) -> Result { + say(&format!( + "parsing every statement under {}", + corpora.display() + )); + let CorpusParse { + verdicts: current, + broken, + } = current_verdicts(corpora).map_err(|problem| failed(&problem))?; + for problem in broken.iter().take(SHOWN_CHANGES) { + say(&format!("BROKEN {problem}")); + } + if !broken.is_empty() { + return Err(failed(&format!( + "{} statements break the lossless-token or located-parse invariant", + broken.len() + ))); + } + let statements: usize = current + .corpora + .values() + .flat_map(|by_dialect| by_dialect.values()) + .map(|verdicts| verdicts.statements) + .sum(); + if update { + return write_verdicts(verdicts_path, ¤t); + } + let recorded: Verdicts = std::fs::read_to_string(verdicts_path) + .map_err(|error| error.to_string()) + .and_then(|text| serde_json::from_str(&text).map_err(|error| error.to_string())) + .map_err(|problem| failed(&format!("{}: {problem}", verdicts_path.display())))?; + let changes: Vec = changed_verdicts(&recorded, ¤t); + for change in changes.iter().take(SHOWN_CHANGES) { + say(&format!("CHANGED {change}")); + } + match (changes.is_empty(), recorded == current) { + (true, true) => { + say(&format!( + "corpus verdicts: passed, {statements} verdicts as recorded" + )); + Ok(ExitCode::SUCCESS) + } + (true, false) => Err(failed(&format!( + "{} differs from the current verdicts outside them; record it with `make corpus-verdicts-baseline`", + verdicts_path.display() + ))), + (false, _) => Err(failed(&format!( + "{} parse verdicts changed; review them and record them with `make corpus-verdicts-baseline`", + changes.len() + ))), + } +} diff --git a/tools/sqlquack-coverage-check/src/cli/main/run_cli.rs b/tools/sqlquack-coverage-check/src/cli/main/run_cli.rs index a191ac736..304518753 100644 --- a/tools/sqlquack-coverage-check/src/cli/main/run_cli.rs +++ b/tools/sqlquack-coverage-check/src/cli/main/run_cli.rs @@ -11,6 +11,7 @@ use crate::cli::_helpers::commands::{ use crate::cli::_helpers::depth_command::depth_probe; use crate::cli::_helpers::instructions_command::{instruction_counts, instruction_workload}; use crate::cli::_helpers::results_command::results; +use crate::cli::_helpers::verdicts_command::corpus_verdicts; use crate::cli::models::{Arguments, Command}; /// Runs one subcommand; progress and problems go to stderr, results to stdout. @@ -58,6 +59,11 @@ pub fn run_cli() -> ExitCode { } => depth_probe(&profile, &allowlist, jobs, json), Command::InstructionCounts { baseline, update } => instruction_counts(&baseline, update), Command::InstructionWorkload { workload } => instruction_workload(workload), + Command::CorpusVerdicts { + corpora, + verdicts, + update, + } => corpus_verdicts(&corpora, &verdicts, update), }; result.unwrap_or_else(|code| code) } diff --git a/tools/sqlquack-coverage-check/src/cli/models.rs b/tools/sqlquack-coverage-check/src/cli/models.rs index 7dd3ac8f8..5c9061c27 100644 --- a/tools/sqlquack-coverage-check/src/cli/models.rs +++ b/tools/sqlquack-coverage-check/src/cli/models.rs @@ -9,6 +9,7 @@ use crate::formats::constants::COVERAGE_ROOT; use crate::formats::models::LocalParser; use crate::instructions::constants::BASELINE_FILE; use crate::instructions::models::Workload; +use crate::verdicts::constants::{CORPORA_DIR, VERDICTS_FILE}; #[derive(Debug, Parser)] #[command( @@ -150,6 +151,18 @@ pub enum Command { #[arg(long, value_enum)] workload: Workload, }, + /// SQLQuack's parse verdict on every corpus statement against the recorded ones; fails on any change. + CorpusVerdicts { + /// The directory of vendored corpus files. + #[arg(long, default_value = CORPORA_DIR)] + corpora: PathBuf, + /// The recorded verdicts. + #[arg(long, default_value = VERDICTS_FILE)] + verdicts: PathBuf, + /// Record the current verdicts instead of comparing with them. + #[arg(long)] + update: bool, + }, /// What conversion makes of every case Snowflake parsed, by form and family. Translation { #[command(flatten)] diff --git a/tools/sqlquack-coverage-check/src/lib.rs b/tools/sqlquack-coverage-check/src/lib.rs index 08183f2ce..6a415769b 100644 --- a/tools/sqlquack-coverage-check/src/lib.rs +++ b/tools/sqlquack-coverage-check/src/lib.rs @@ -1,4 +1,4 @@ -//! Development check of each dialect's documented syntax coverage, the translation results, the nesting-depth probe and the instruction-count guard; never a runtime dependency. +//! Development check of each dialect's documented syntax coverage, the translation results, the nesting-depth probe, the instruction-count guard and the corpora's parse verdicts; never a runtime dependency. #![forbid(unsafe_code)] @@ -10,3 +10,4 @@ pub mod instructions; pub mod report; pub mod results; pub mod translation; +pub mod verdicts; diff --git a/tools/sqlquack-coverage-check/src/verdicts/_helpers/corpus.rs b/tools/sqlquack-coverage-check/src/verdicts/_helpers/corpus.rs new file mode 100644 index 000000000..f3fc63862 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/_helpers/corpus.rs @@ -0,0 +1,76 @@ +//! Reading the corpus files and taking each file's verdicts. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use sqlquack_syntax::Dialect; + +use crate::verdicts::_helpers::invariants::check_statement; +use crate::verdicts::constants::CORPUS_EXTENSION; +use crate::verdicts::models::{CorpusFile, DialectVerdicts, StatementCheck}; + +/// The dialects an entry is parsed in, by the name it gives. +fn dialects(name: &str) -> Result<&'static [(&'static str, Dialect)], String> { + match name { + "snowflake" => Ok(&[("snowflake", Dialect::Snowflake)]), + "duckdb" => Ok(&[("duckdb", Dialect::DuckDB)]), + "postgres" => Ok(&[("postgres", Dialect::PostgreSQL)]), + "sqlite" => Ok(&[("sqlite", Dialect::SQLite)]), + "both" => Ok(&[ + ("snowflake", Dialect::Snowflake), + ("duckdb", Dialect::DuckDB), + ]), + other => Err(format!("unknown corpus dialect {other}")), + } +} + +/// The corpus file at `path`. +pub(crate) fn read_corpus(path: &Path) -> Result { + std::fs::read_to_string(path) + .map_err(|error| error.to_string()) + .and_then(|text| serde_json::from_str(&text).map_err(|error| error.to_string())) + .map_err(|problem| format!("{}: {problem}", path.display())) +} + +/// The `.json` corpus files in `corpora`, sorted. +pub(crate) fn corpus_files(corpora: &Path) -> Result, String> { + let listing = + std::fs::read_dir(corpora).map_err(|error| format!("{}: {error}", corpora.display()))?; + let mut files: Vec = Vec::new(); + for entry in listing { + let path: PathBuf = entry + .map_err(|error| format!("{}: {error}", corpora.display()))? + .path(); + if path.extension() == Some(std::ffi::OsStr::new(CORPUS_EXTENSION)) { + files.push(path); + } + } + files.sort(); + Ok(files) +} + +/// One corpus file's verdicts by dialect, and its statements that break an invariant. +pub(crate) fn file_verdicts( + file: &CorpusFile, + name: &str, +) -> Result<(BTreeMap, Vec), String> { + let mut by_dialect: BTreeMap = BTreeMap::new(); + let mut broken: Vec = Vec::new(); + for (position, entry) in file.entries.iter().enumerate() { + let parsed_in = + dialects(&entry.dialect).map_err(|problem| format!("{name} #{position}: {problem}"))?; + for (dialect_name, dialect) in parsed_in { + let check: StatementCheck = check_statement(&entry.sql, *dialect); + let verdicts: &mut DialectVerdicts = + by_dialect.entry((*dialect_name).to_owned()).or_default(); + verdicts.statements += 1; + if check.refused { + verdicts.refused.push(position); + } + for problem in check.problems { + broken.push(format!("{name} {dialect_name} #{position}: {problem}")); + } + } + } + Ok((by_dialect, broken)) +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/_helpers/invariants.rs b/tools/sqlquack-coverage-check/src/verdicts/_helpers/invariants.rs new file mode 100644 index 000000000..abc1f0d9d --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/_helpers/invariants.rs @@ -0,0 +1,93 @@ +//! Checks every corpus statement must pass whatever its verdict: lossless tokens, located parses. + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use serde_json::Value; +use sqlquack_syntax::{CallMode, Dialect, Expression, ParseOptions, TokenizeFailure}; +use sqlquack_syntax_snowflake::{parse_statements, parse_statements_with, tokenize}; + +use crate::verdicts::constants::SPAN_FIELD; +use crate::verdicts::models::StatementCheck; + +/// Why the lossless tokens of `sql` do not rebuild it byte for byte, if they do not. +fn lossless_failure(sql: &str, dialect: Dialect, parsed: bool) -> Option { + let tokens = match catch_unwind(|| tokenize(sql, dialect)) { + Ok(Ok(tokens)) => tokens, + Ok(Err(TokenizeFailure::Tokenizer(_))) if !parsed => return None, + Ok(Err(failure)) => return Some(failure.to_string()), + Err(_) => return Some("the tokenizer panicked".to_owned()), + }; + let mut cursor: usize = 0; + for token in &tokens { + if token.start != cursor || token.end <= token.start { + return Some(format!( + "token {}..{} does not continue at byte {cursor}", + token.start, token.end + )); + } + cursor = token.end; + } + match cursor == sql.len() { + true => None, + false => Some(format!("tokens end at byte {cursor} of {}", sql.len())), + } +} + +fn without_spans(value: Value) -> Value { + match value { + Value::Object(fields) => Value::Object( + fields + .into_iter() + .filter(|(field, _)| field != SPAN_FIELD) + .map(|(field, child)| (field, without_spans(child))) + .collect(), + ), + Value::Array(items) => Value::Array(items.into_iter().map(without_spans).collect()), + other => other, + } +} + +/// A parse result as comparable JSON without spans, or the error's debug form. +fn comparable(parsed: Result, impl std::fmt::Debug>) -> Result { + parsed + .map_err(|error| format!("{error:?}")) + .and_then(|trees| serde_json::to_value(trees).map_err(|error| error.to_string())) + .map(without_spans) +} + +/// Why the located parse of `sql` differs from `faithful` beyond spans, if it does. +fn located_difference( + sql: &str, + dialect: Dialect, + faithful: Result, impl std::fmt::Debug>, +) -> Option { + let located = catch_unwind(AssertUnwindSafe(|| { + parse_statements_with( + sql, + dialect, + &ParseOptions::default(), + CallMode::FaithfulLocated, + ) + })); + let Ok(located) = located else { + return Some("the located parse panicked".to_owned()); + }; + match comparable(faithful) == comparable(located) { + true => None, + false => Some("the located parse differs from the faithful parse".to_owned()), + } +} + +/// Whether SQLQuack refuses `sql` in `dialect`, and which invariants it breaks. +pub(crate) fn check_statement(sql: &str, dialect: Dialect) -> StatementCheck { + let faithful = parse_statements(sql, dialect, &ParseOptions::default()); + let (refused, lossless) = match &faithful { + Ok(_) => (false, lossless_failure(sql, dialect, true)), + Err(_) => (true, lossless_failure(sql, dialect, false)), + }; + let problems: Vec = [lossless, located_difference(sql, dialect, faithful)] + .into_iter() + .flatten() + .collect(); + StatementCheck { refused, problems } +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/_helpers/mod.rs b/tools/sqlquack-coverage-check/src/verdicts/_helpers/mod.rs new file mode 100644 index 000000000..fac5e8d40 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod corpus; +pub(crate) mod invariants; diff --git a/tools/sqlquack-coverage-check/src/verdicts/constants.rs b/tools/sqlquack-coverage-check/src/verdicts/constants.rs new file mode 100644 index 000000000..fa5bce9f5 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/constants.rs @@ -0,0 +1,14 @@ +//! Where the corpora and their recorded verdicts are, and how many changes a failure shows. + +/// The vendored corpora, relative to the repository. +pub const CORPORA_DIR: &str = "corpora/polyglot-check"; +/// The extension of a corpus file. +pub const CORPUS_EXTENSION: &str = "json"; +/// The recorded verdicts, relative to the repository. +pub const VERDICTS_FILE: &str = "coverage/syntax/corpus-verdicts.json"; +/// What the recorded file says about itself. +pub const ABOUT: &str = "SQLQuack's parse verdict on every statement of the corpora under corpora/polyglot-check: per file and dialect, how many statements it holds and the positions of those SQLQuack refuses. Any change, and any statement whose lossless tokens do not rebuild it or whose located parse differs from its faithful parse, fails `make corpus-verdicts`; record a reviewed change with `make corpus-verdicts-baseline`."; +/// The field located parses add to a node, ignored when comparing with the faithful parse. +pub const SPAN_FIELD: &str = "span"; +/// Changed verdicts printed before the rest are counted. +pub const SHOWN_CHANGES: usize = 20; diff --git a/tools/sqlquack-coverage-check/src/verdicts/main/changed_verdicts.rs b/tools/sqlquack-coverage-check/src/verdicts/main/changed_verdicts.rs new file mode 100644 index 000000000..9ec2fddcd --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/main/changed_verdicts.rs @@ -0,0 +1,53 @@ +//! `changed_verdicts`: the statements whose verdict moved since it was recorded. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::verdicts::models::{DialectVerdicts, Verdicts}; + +/// One dialect's changes in one corpus file, as readable lines. +fn dialect_changes( + place: &str, + recorded: &DialectVerdicts, + current: &DialectVerdicts, +) -> Vec { + if recorded.statements != current.statements { + return vec![format!( + "{place}: {} statements, recorded {}", + current.statements, recorded.statements + )]; + } + let was: BTreeSet = recorded.refused.iter().copied().collect(); + let is: BTreeSet = current.refused.iter().copied().collect(); + let refused = is + .difference(&was) + .map(|position| format!("{place} #{position}: now refused, recorded as parsed")); + let parsed = was + .difference(&is) + .map(|position| format!("{place} #{position}: now parsed, recorded as refused")); + refused.chain(parsed).collect() +} + +/// Every statement whose verdict differs between `recorded` and `current`, as readable lines. +pub fn changed_verdicts(recorded: &Verdicts, current: &Verdicts) -> Vec { + let empty: BTreeMap = BTreeMap::new(); + let none: DialectVerdicts = DialectVerdicts::default(); + let names: BTreeSet<&String> = recorded + .corpora + .keys() + .chain(current.corpora.keys()) + .collect(); + let mut lines: Vec = Vec::new(); + for name in names { + let before = recorded.corpora.get(name).unwrap_or(&empty); + let after = current.corpora.get(name).unwrap_or(&empty); + let dialects: BTreeSet<&String> = before.keys().chain(after.keys()).collect(); + for dialect in dialects { + lines.extend(dialect_changes( + &format!("{name} {dialect}"), + before.get(dialect).unwrap_or(&none), + after.get(dialect).unwrap_or(&none), + )); + } + } + lines +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/main/current_verdicts.rs b/tools/sqlquack-coverage-check/src/verdicts/main/current_verdicts.rs new file mode 100644 index 000000000..bd7760d5d --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/main/current_verdicts.rs @@ -0,0 +1,29 @@ +//! `current_verdicts`: SQLQuack's verdict on every statement of the corpora, and broken invariants. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::verdicts::_helpers::corpus::{corpus_files, file_verdicts, read_corpus}; +use crate::verdicts::constants::ABOUT; +use crate::verdicts::models::{CorpusParse, Verdicts}; + +/// The verdicts of every `.json` corpus file in `corpora`, and the statements that break an invariant. +pub fn current_verdicts(corpora: &Path) -> Result { + let mut parse: CorpusParse = CorpusParse { + verdicts: Verdicts { + about: ABOUT.to_owned(), + corpora: BTreeMap::new(), + }, + broken: Vec::new(), + }; + for path in corpus_files(corpora)? { + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let (verdicts, broken) = file_verdicts(&read_corpus(&path)?, &name)?; + parse.verdicts.corpora.insert(name, verdicts); + parse.broken.extend(broken); + } + Ok(parse) +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/main/mod.rs b/tools/sqlquack-coverage-check/src/verdicts/main/mod.rs new file mode 100644 index 000000000..41519330f --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/main/mod.rs @@ -0,0 +1,4 @@ +//! Entry points. + +pub mod changed_verdicts; +pub mod current_verdicts; diff --git a/tools/sqlquack-coverage-check/src/verdicts/mod.rs b/tools/sqlquack-coverage-check/src/verdicts/mod.rs new file mode 100644 index 000000000..e5a437d62 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/mod.rs @@ -0,0 +1,9 @@ +//! The parse verdicts: whether SQLQuack parses each statement of the vendored corpora, as recorded. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/tools/sqlquack-coverage-check/src/verdicts/models.rs b/tools/sqlquack-coverage-check/src/verdicts/models.rs new file mode 100644 index 000000000..ec5a4dc4b --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/models.rs @@ -0,0 +1,57 @@ +//! The recorded verdicts and the corpus files they are taken from. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// One corpus file's statements in one dialect: how many, and which SQLQuack refuses. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct DialectVerdicts { + /// The statements parsed in the dialect. + pub statements: usize, + /// The positions, in the file's `entries`, of the statements SQLQuack refuses. + pub refused: Vec, +} + +/// The recorded verdicts: by corpus file name, then by dialect. +#[derive(Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub struct Verdicts { + /// What the file records and how to record it again. + pub about: String, + /// The verdicts of each corpus file. + pub corpora: BTreeMap>, +} + +/// The corpora as parsed now: their verdicts, and the statements that break an invariant. +#[derive(Debug, Default)] +pub struct CorpusParse { + /// The verdict on every statement. + pub verdicts: Verdicts, + /// ` #: ` for each statement that breaks an invariant. + pub broken: Vec, +} + +/// One statement in one dialect: whether SQLQuack refuses it, and the invariants it breaks. +#[derive(Debug, Default)] +pub struct StatementCheck { + /// Whether the faithful parse fails. + pub refused: bool, + /// What each broken invariant says. + pub problems: Vec, +} + +/// A vendored corpus file. +#[derive(Debug, Deserialize)] +pub struct CorpusFile { + /// Its statements. + pub entries: Vec, +} + +/// A statement of a corpus file. +#[derive(Debug, Deserialize)] +pub struct CorpusEntry { + /// `snowflake`, `duckdb`, `postgres`, `sqlite`, or `both` for Snowflake and DuckDB. + pub dialect: String, + /// The SQL. + pub sql: String, +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/tests.rs b/tools/sqlquack-coverage-check/src/verdicts/tests.rs new file mode 100644 index 000000000..b452e9063 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_changed_verdicts.rs"] +mod test_changed_verdicts; +#[path = "tests/test_current_verdicts.rs"] +mod test_current_verdicts; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/tools/sqlquack-coverage-check/src/verdicts/tests/helpers.rs b/tools/sqlquack-coverage-check/src/verdicts/tests/helpers.rs new file mode 100644 index 000000000..2dcc6bf09 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/tests/helpers.rs @@ -0,0 +1,28 @@ +use std::collections::BTreeMap; + +use crate::verdicts::models::{DialectVerdicts, Verdicts}; + +/// One corpus file's verdicts, from (dialect, statements, refused positions). +pub(crate) fn by_dialect( + file: Vec<(&'static str, usize, Vec)>, +) -> BTreeMap { + file.into_iter() + .map(|(dialect, statements, refused)| { + ( + dialect.to_owned(), + DialectVerdicts { + statements, + refused, + }, + ) + }) + .collect() +} + +/// Verdicts holding one corpus file, `c.json`. +pub(crate) fn one_file(file: Vec<(&'static str, usize, Vec)>) -> Verdicts { + Verdicts { + about: String::new(), + corpora: BTreeMap::from([("c.json".to_owned(), by_dialect(file))]), + } +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/tests/test_changed_verdicts.rs b/tools/sqlquack-coverage-check/src/verdicts/tests/test_changed_verdicts.rs new file mode 100644 index 000000000..88f24e854 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/tests/test_changed_verdicts.rs @@ -0,0 +1,43 @@ +use crate::verdicts::main::changed_verdicts::changed_verdicts; +use crate::verdicts::tests::helpers::one_file; +use crate::verdicts::tests::test_types::ChangedTestCase; + +#[test] +fn given_recorded_and_current_verdicts_when_compared_then_each_moved_statement_is_named() { + let test_cases = [ + ChangedTestCase { + description: "the same verdicts", + recorded: vec![("duckdb", 3, vec![1])], + current: vec![("duckdb", 3, vec![1])], + expected_lines: vec![], + }, + ChangedTestCase { + description: "one statement refused now and another parsed now", + recorded: vec![("snowflake", 4, vec![0, 2])], + current: vec![("snowflake", 4, vec![2, 3])], + expected_lines: vec![ + "c.json snowflake #3: now refused, recorded as parsed", + "c.json snowflake #0: now parsed, recorded as refused", + ], + }, + ChangedTestCase { + description: "a different statement count is one line, without positions", + recorded: vec![("sqlite", 4, vec![0])], + current: vec![("sqlite", 5, vec![1])], + expected_lines: vec!["c.json sqlite: 5 statements, recorded 4"], + }, + ChangedTestCase { + description: "a dialect only one side has", + recorded: vec![("postgres", 2, vec![])], + current: vec![("duckdb", 1, vec![0])], + expected_lines: vec![ + "c.json duckdb: 1 statements, recorded 0", + "c.json postgres: 0 statements, recorded 2", + ], + }, + ]; + for test_case in test_cases { + let lines = changed_verdicts(&one_file(test_case.recorded), &one_file(test_case.current)); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + } +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/tests/test_current_verdicts.rs b/tools/sqlquack-coverage-check/src/verdicts/tests/test_current_verdicts.rs new file mode 100644 index 000000000..66a4de61a --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/tests/test_current_verdicts.rs @@ -0,0 +1,45 @@ +use serde_json::json; + +use crate::verdicts::main::current_verdicts::current_verdicts; +use crate::verdicts::tests::helpers::one_file; +use crate::verdicts::tests::test_types::CurrentTestCase; + +#[test] +fn given_a_corpus_file_when_parsed_then_each_dialect_counts_statements_and_refusals() { + let test_cases = [ + CurrentTestCase { + description: "a Snowflake-only and a DuckDB-only statement each count in their dialect", + entries: vec![ + ("snowflake", "SELECT v:a FROM t"), + ("duckdb", "SELECT FROM WHERE"), + ], + expected_verdicts: vec![("duckdb", 1, vec![1]), ("snowflake", 1, vec![])], + }, + CurrentTestCase { + description: "an entry for both dialects is parsed as each, refused in only one", + entries: vec![("both", "SELECT 1 ** 2"), ("both", "SELECT (")], + expected_verdicts: vec![("duckdb", 2, vec![1]), ("snowflake", 2, vec![0, 1])], + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary directory"); + let entries: Vec = test_case + .entries + .iter() + .map(|(dialect, sql)| json!({ "dialect": dialect, "sql": sql })) + .collect(); + std::fs::write( + directory.path().join("c.json"), + json!({ "entries": entries }).to_string(), + ) + .expect("corpus written"); + std::fs::write(directory.path().join("notes.txt"), "not a corpus").expect("note written"); + let parse = current_verdicts(directory.path()).expect("verdicts"); + assert_eq!( + (parse.verdicts.corpora, parse.broken), + (one_file(test_case.expected_verdicts).corpora, vec![]), + "{}", + test_case.description + ); + } +} diff --git a/tools/sqlquack-coverage-check/src/verdicts/tests/test_types.rs b/tools/sqlquack-coverage-check/src/verdicts/tests/test_types.rs new file mode 100644 index 000000000..a6637d922 --- /dev/null +++ b/tools/sqlquack-coverage-check/src/verdicts/tests/test_types.rs @@ -0,0 +1,14 @@ +/// Recorded and current verdicts of one corpus file, as (dialect, statements, refused positions). +pub(crate) struct ChangedTestCase { + pub(crate) description: &'static str, + pub(crate) recorded: Vec<(&'static str, usize, Vec)>, + pub(crate) current: Vec<(&'static str, usize, Vec)>, + pub(crate) expected_lines: Vec<&'static str>, +} + +/// A corpus file's entries, as (dialect, SQL), and the verdicts expected for it. +pub(crate) struct CurrentTestCase { + pub(crate) description: &'static str, + pub(crate) entries: Vec<(&'static str, &'static str)>, + pub(crate) expected_verdicts: Vec<(&'static str, usize, Vec)>, +} diff --git a/tools/sqlquack-polyglot-check/Cargo.toml b/tools/sqlquack-polyglot-check/Cargo.toml deleted file mode 100644 index a02976d67..000000000 --- a/tools/sqlquack-polyglot-check/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "sqlquack-polyglot-check" -version.workspace = true -description = "Development check that SQLQuack's parser gives exactly polyglot's results; never a runtime dependency." -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -rust-version.workspace = true - -[[bin]] -name = "sqlquack-polyglot-check" -path = "src/bin/sqlquack-polyglot-check.rs" - -[dependencies] -clap.workspace = true -polyglot-sql.workspace = true -serde.workspace = true -serde_json.workspace = true -sqlquack.workspace = true -sqlquack-diagnostics.workspace = true -sqlquack-evidence.workspace = true -sqlquack-policy.workspace = true -sqlquack-convert = { workspace = true, features = ["engine", "snowflake"] } -sqlquack-syntax.workspace = true -sqlquack-syntax-snowflake.workspace = true -toml.workspace = true -walkdir.workspace = true - -[lints] -workspace = true diff --git a/tools/sqlquack-polyglot-check/src/bin/sqlquack-polyglot-check.rs b/tools/sqlquack-polyglot-check/src/bin/sqlquack-polyglot-check.rs deleted file mode 100644 index 52a89187c..000000000 --- a/tools/sqlquack-polyglot-check/src/bin/sqlquack-polyglot-check.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! `sqlquack-polyglot-check`: compares SQLQuack's parser with polyglot's on corpora. - -use std::process::ExitCode; - -use sqlquack_polyglot_check::cli::main::run_check::run_check; - -fn main() -> ExitCode { - run_check() -} diff --git a/tools/sqlquack-polyglot-check/src/cli/_helpers/corpora.rs b/tools/sqlquack-polyglot-check/src/cli/_helpers/corpora.rs deleted file mode 100644 index 5563fb42c..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/_helpers/corpora.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! The corpora a command line names. - -use sqlquack_syntax::Dialect; - -use crate::cli::models::{CorpusArgument, DialectArgument}; -use crate::corpora::main::case_corpus::case_corpus; -use crate::corpora::main::directory_corpus::directory_corpus; -use crate::corpora::main::load_corpus::load_corpus; -use crate::corpora::models::CorpusEntry; - -/// A corpus name and its entries, or why it could not be loaded. -pub(crate) type NamedCorpus = (String, Result, String>); - -/// Each corpus `argument` names. -pub(crate) fn named_corpora(argument: &CorpusArgument) -> Vec { - match argument { - CorpusArgument::Files { paths } => paths - .iter() - .map(|path| (path.display().to_string(), load_corpus(path))) - .collect(), - CorpusArgument::Directory { path, dialect } => { - let dialect = match dialect { - DialectArgument::Snowflake => Dialect::Snowflake, - DialectArgument::Duckdb => Dialect::DuckDB, - DialectArgument::Postgres => Dialect::PostgreSQL, - DialectArgument::Sqlite => Dialect::SQLite, - }; - vec![(path.display().to_string(), directory_corpus(path, dialect))] - } - CorpusArgument::Cases { cases, evidence } => { - vec![(cases.display().to_string(), case_corpus(cases, evidence))] - } - } -} diff --git a/tools/sqlquack-polyglot-check/src/cli/_helpers/mod.rs b/tools/sqlquack-polyglot-check/src/cli/_helpers/mod.rs deleted file mode 100644 index 126e1c84a..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/_helpers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod corpora; -pub(crate) mod report; diff --git a/tools/sqlquack-polyglot-check/src/cli/_helpers/report.rs b/tools/sqlquack-polyglot-check/src/cli/_helpers/report.rs deleted file mode 100644 index ce6a02368..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/_helpers/report.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Run summaries as printed text. - -use std::fmt::Write; - -use crate::cli::constants::{MILLISECONDS, SHOWN_DETAIL, SHOWN_FINDINGS}; -use crate::runs::models::{ParserTimes, RunSummary}; - -/// The first findings, unless only counts may be shown, then a line of counts and timings. -pub(crate) fn report(label: &str, summary: &RunSummary, counts_only: bool) -> String { - let mut text = String::new(); - if !counts_only { - for finding in summary - .differences - .iter() - .chain(&summary.tokens_lost) - .chain(&summary.type_names) - .chain(&summary.unlisted) - .chain(&summary.unexplained) - .chain(&summary.located) - .take(SHOWN_FINDINGS) - { - let detail: String = finding.detail.chars().take(SHOWN_DETAIL).collect(); - let _ = writeln!(text, "{}\n{detail}\n", finding.from); - } - } - let _ = writeln!( - text, - "{label}: {} entries, {} same trees, {} same errors, {} fixed, {} rejected, {} kept quoting, {} kept nodes, {} kept fields, {} structured, {} differences, {} without lossless \ - tokens, {} cast type names unlike polyglot's, {} faithful calls kept, {} unlisted, {} \ - unexplained, {} located parses unlike faithful; parse time polyglot {:.1} ms, sqlquack \ - {:.1} ms", - summary.entries, - summary.same_trees, - summary.same_errors, - summary.fixed.values().sum::(), - summary.rejected.values().sum::(), - summary.kept_quoting.values().sum::(), - summary.kept_nodes.values().sum::(), - summary.kept_fields.values().sum::(), - summary.structured.values().sum::(), - summary.differences.len(), - summary.tokens_lost.len(), - summary.type_names.len(), - summary.spellings.values().sum::(), - summary.unlisted.len(), - summary.unexplained.len(), - summary.located.len(), - summary.polyglot.as_secs_f64() * MILLISECONDS, - summary.sqlquack.as_secs_f64() * MILLISECONDS, - ); - text -} - -/// A line of median parse times and their ratio. -pub(crate) fn timing(label: &str, rounds: usize, times: ParserTimes) -> String { - let polyglot = times.polyglot.as_secs_f64() * MILLISECONDS; - let sqlquack = times.sqlquack.as_secs_f64() * MILLISECONDS; - format!( - "{label}: median of {rounds} rounds, polyglot {polyglot:.1} ms, sqlquack {sqlquack:.1} ms \ - ({:.3}x)", - sqlquack / polyglot.max(f64::MIN_POSITIVE) - ) -} - -/// One line per kept spelling: name, polyglot's node and the number of calls. -pub(crate) fn spelling_counts(summary: &RunSummary) -> String { - let mut text = String::new(); - for (spelling, count) in &summary.spellings { - let _ = writeln!( - text, - "spelling\t{}\t{}\t{count}", - spelling.name, spelling.polyglot - ); - } - text -} diff --git a/tools/sqlquack-polyglot-check/src/cli/constants.rs b/tools/sqlquack-polyglot-check/src/cli/constants.rs deleted file mode 100644 index 6d232c155..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/constants.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Output limits. - -/// Findings printed in full before the rest are only counted. -pub const SHOWN_FINDINGS: usize = 10; -/// Characters of a finding's detail printed. -pub const SHOWN_DETAIL: usize = 2000; -/// Milliseconds in a second, for printed parse times. -pub const MILLISECONDS: f64 = 1000.0; diff --git a/tools/sqlquack-polyglot-check/src/cli/main/mod.rs b/tools/sqlquack-polyglot-check/src/cli/main/mod.rs deleted file mode 100644 index 586f49b95..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/main/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Entry points. - -pub mod run_check; diff --git a/tools/sqlquack-polyglot-check/src/cli/main/run_check.rs b/tools/sqlquack-polyglot-check/src/cli/main/run_check.rs deleted file mode 100644 index bf1ae20e3..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/main/run_check.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! `run_check`: the whole command. - -use std::io::Write; -use std::process::ExitCode; - -use clap::Parser; - -use crate::cli::_helpers::corpora::named_corpora; -use crate::cli::_helpers::report::{report, spelling_counts, timing}; -use crate::cli::models::Arguments; -use crate::expected::main::load_expected::load_expected; -use crate::runs::main::run_corpus::run_corpus; -use crate::runs::main::time_parsers::time_parsers; - -/// Compares every named corpus; succeeds only when every entry agrees and keeps its tokens. -pub fn run_check() -> ExitCode { - let arguments = Arguments::parse(); - std::panic::set_hook(Box::new(|_| {})); - let mut stdout = std::io::stdout().lock(); - let mut stderr = std::io::stderr().lock(); - let expected = match load_expected(&arguments.expected) { - Ok(expected) => expected, - Err(error) => { - let _ = writeln!(stderr, "sqlquack-polyglot-check: failed: {error}"); - return ExitCode::FAILURE; - } - }; - let mut exact = true; - for (name, entries) in named_corpora(&arguments.corpus) { - let label = match arguments.counts_only { - true => "corpus".to_owned(), - false => name, - }; - let _ = writeln!(stderr, "sqlquack-polyglot-check: comparing {label}…"); - let entries = match entries { - Ok(entries) => entries, - Err(error) => { - let _ = writeln!(stderr, "sqlquack-polyglot-check: failed: {error}"); - return ExitCode::FAILURE; - } - }; - let summary = run_corpus(&entries, &expected); - let _ = write!( - stdout, - "{}", - report(&label, &summary, arguments.counts_only) - ); - exact &= summary.exact(); - if arguments.spellings && !arguments.counts_only { - let _ = write!(stdout, "{}", spelling_counts(&summary)); - } - if arguments.timing_rounds > 0 { - let _ = writeln!(stderr, "sqlquack-polyglot-check: timing {label}…"); - let times = time_parsers(&entries, arguments.timing_rounds); - let _ = writeln!(stdout, "{}", timing(&label, arguments.timing_rounds, times)); - } - } - let (message, code) = match exact { - true => ("done, every entry identical", ExitCode::SUCCESS), - false => ("failed, see the differences above", ExitCode::FAILURE), - }; - let _ = writeln!(stderr, "sqlquack-polyglot-check: {message}"); - code -} diff --git a/tools/sqlquack-polyglot-check/src/cli/mod.rs b/tools/sqlquack-polyglot-check/src/cli/mod.rs deleted file mode 100644 index 9c2af69e8..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The `sqlquack-polyglot-check` command line. - -pub(crate) mod _helpers; -pub mod constants; -pub mod main; -pub mod models; diff --git a/tools/sqlquack-polyglot-check/src/cli/models.rs b/tools/sqlquack-polyglot-check/src/cli/models.rs deleted file mode 100644 index 6fde68247..000000000 --- a/tools/sqlquack-polyglot-check/src/cli/models.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Command-line arguments. - -use std::path::PathBuf; - -use clap::{Parser, Subcommand, ValueEnum}; -use sqlquack_policy::policy::constants::EVIDENCE_FILE; - -/// `sqlquack-polyglot-check` arguments. -#[derive(Debug, Parser)] -#[command(about = "Check that SQLQuack's parser gives exactly polyglot's results")] -pub struct Arguments { - /// What to compare. - #[command(subcommand)] - pub corpus: CorpusArgument, - /// Print counts and timings only, never SQL, trees or file names (for private corpora). - #[arg(long, global = true)] - pub counts_only: bool, - /// After comparing, time each parser over the whole corpus this many times and print medians. - #[arg(long, global = true, default_value_t = 0)] - pub timing_rounds: usize, - /// The reviewed list of call spellings SQLQuack keeps where polyglot lowers them. - #[arg( - long, - global = true, - default_value = "corpora/polyglot-check/expected_differences.toml" - )] - pub expected: PathBuf, - /// Also print each kept spelling with the node polyglot lowers it to and its call count. - #[arg(long, global = true)] - pub spellings: bool, -} - -/// The corpus to compare. -#[derive(Debug, Subcommand)] -pub enum CorpusArgument { - /// Vendored corpus files. - Files { - /// The corpus JSON files. - paths: Vec, - }, - /// Every `.sql` file below a directory, one entry per file. - Directory { - /// The directory. - path: PathBuf, - /// The dialect its files are written in. - #[arg(long, value_enum)] - dialect: DialectArgument, - }, - /// SQLQuack's cases: source SQL and converted DuckDB SQL. - Cases { - /// The case directory. - #[arg(long, default_value = "cases")] - cases: PathBuf, - /// The evidence policy, which declares the cases' session profiles. - #[arg(long, default_value = EVIDENCE_FILE)] - evidence: PathBuf, - }, -} - -/// A dialect on the command line. -#[derive(Clone, Copy, Debug, ValueEnum)] -pub enum DialectArgument { - /// Snowflake. - Snowflake, - /// DuckDB. - Duckdb, - /// PostgreSQL. - Postgres, - /// SQLite. - Sqlite, -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/_helpers/faithful.rs b/tools/sqlquack-polyglot-check/src/comparison/_helpers/faithful.rs deleted file mode 100644 index d44cba415..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/_helpers/faithful.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! Faithful calls put back into polyglot's lowered form, one call at a time. - -use serde_json::{Map, Value}; -use sqlquack_syntax::{CallMode, Dialect, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements_with; - -use crate::comparison::constants::{ - AGGREGATE_KEY, ANNOTATED_KEY, COMMENTS_SUFFIX, ERROR_KIND, FUNCTION_KEY, JSON_EXTRACT_KEY, - SELECT_PREFIX, -}; -use crate::comparison::models::Spelling; - -/// The tree with each lowered faithful call replaced by polyglot's parse of its text, and why. -pub(crate) fn relower(value: Value, sql: &[char]) -> (Value, Vec) { - let mut spellings: Vec = Vec::new(); - let value = match call_text(&value, sql) { - Some((name, start, text)) => { - let value = without_span(value); - match lowered_expression(&text, start, sql) { - Some(lowered) if same_call(&value, &lowered) => value, - Some(lowered) => { - spellings.push(Spelling { - name, - polyglot: node_kind(&lowered), - }); - return (lowered, spellings); - } - None => { - spellings.push(Spelling { - name, - polyglot: ERROR_KIND.to_owned(), - }); - value - } - } - } - None => value, - }; - let value = match value { - Value::Object(object) => { - let mut children: Map = Map::new(); - for (key, child) in object { - let (child, found) = relower(child, sql); - spellings.extend(found); - children.insert(key, child); - } - Value::Object(children) - } - Value::Array(items) => { - let mut children: Vec = Vec::with_capacity(items.len()); - for child in items { - let (child, found) = relower(child, sql); - spellings.extend(found); - children.push(child); - } - Value::Array(children) - } - other => other, - }; - (value, spellings) -} - -/// The upper-case name, start offset and source text of a faithful call with a recorded span. -fn call_text(value: &Value, sql: &[char]) -> Option<(String, usize, String)> { - let payload = call_payload(value)?; - let span = payload.get("span")?; - let start = offset(span.get("start")?)?; - let end = offset(span.get("end")?)?; - let name = payload.get("name")?.as_str()?.to_ascii_uppercase(); - Some((name, start, sql.get(start..end)?.iter().collect())) -} - -/// A serialised character offset; one past any text when it does not fit. -fn offset(value: &Value) -> Option { - let number = value.as_u64()?; - Some(usize::try_from(number).unwrap_or(usize::MAX)) -} - -/// The function or aggregate payload of a call node. -fn call_payload(value: &Value) -> Option<&Value> { - let object = value.as_object()?; - object - .get(FUNCTION_KEY) - .or_else(|| object.get(AGGREGATE_KEY)) -} - -/// The call node without its recorded span. -fn without_span(mut value: Value) -> Value { - let Some(object) = value.as_object_mut() else { - return value; - }; - let key = match object.contains_key(FUNCTION_KEY) { - true => FUNCTION_KEY, - false => AGGREGATE_KEY, - }; - if let Some(Value::Object(payload)) = object.get_mut(key) { - payload.remove("span"); - } - value -} - -/// polyglot's lowered parse of `text`, found at `start` in `sql`, with spans placed in `sql`. -fn lowered_expression(text: &str, start: usize, sql: &[char]) -> Option { - let prefixed = format!("{SELECT_PREFIX}{text}"); - let parsed = parse_statements_with( - &prefixed, - Dialect::Snowflake, - &ParseOptions::default(), - CallMode::Lowered, - ); - let Ok(statements) = parsed else { - return None; - }; - let Ok(statement) = serde_json::to_value(statements.first()?) else { - return None; - }; - let expression = statement.get("select")?.get("expressions")?.get(0)?.clone(); - Some(moved_spans(expression, start, sql)) -} - -/// The tree with every span moved from the `SELECT `-prefixed text to the call's place in `sql`. -fn moved_spans(value: Value, start: usize, sql: &[char]) -> Value { - match value { - Value::Object(object) => Value::Object( - object - .into_iter() - .map(|(key, child)| match (key.as_str(), child) { - ("span", Value::Object(span)) => { - (key, Value::Object(moved_span(span, start, sql))) - } - (_, child) => (key, moved_spans(child, start, sql)), - }) - .collect(), - ), - Value::Array(items) => Value::Array( - items - .into_iter() - .map(|child| moved_spans(child, start, sql)) - .collect(), - ), - other => other, - } -} - -/// One span with shifted offsets and the line and column after its last character. -fn moved_span(mut span: Map, start: usize, sql: &[char]) -> Map { - let read = |key: &str| span.get(key).and_then(offset); - let (Some(from), Some(to)) = (read("start"), read("end")) else { - return span; - }; - let from = from + start - SELECT_PREFIX.len(); - let to = to + start - SELECT_PREFIX.len(); - let before = &sql[..to.min(sql.len())]; - let line_start = before - .iter() - .rposition(|character| *character == '\n') - .map_or(0, |index| index + 1); - let line = 1 + before - .iter() - .filter(|character| **character == '\n') - .count(); - span.insert("start".to_owned(), from.into()); - span.insert("end".to_owned(), to.into()); - span.insert("line".to_owned(), line.into()); - span.insert("column".to_owned(), (to - line_start + 1).into()); - span -} - -/// Whether `lowered` is the same kind of call node, with the same name and argument count. -fn same_call(faithful: &Value, lowered: &Value) -> bool { - let (Some(ours), Some(theirs)) = (call_payload(faithful), call_payload(lowered)) else { - return false; - }; - let name = |payload: &Value| { - payload - .get("name") - .and_then(Value::as_str) - .map(str::to_ascii_uppercase) - }; - let arity = |payload: &Value| payload.get("args").and_then(Value::as_array).map(Vec::len); - node_kind(faithful) == node_kind(lowered) - && name(ours) == name(theirs) - && arity(ours) == arity(theirs) -} - -/// The node's serialised variant name, such as `length` or `if_func`. -pub(crate) fn node_kind(value: &Value) -> String { - value - .as_object() - .and_then(|object| object.keys().next()) - .cloned() - .unwrap_or_default() -} - -/// The tree with every comment removed, and annotations that hold only comments unwrapped. -pub(crate) fn without_comments(value: &Value) -> Value { - match value { - Value::Object(object) => { - if let Some(Value::Object(annotated)) = object.get(ANNOTATED_KEY) - && annotated.get("type_assertion").is_none_or(Value::is_null) - && annotated.get("type_like").is_none_or(Value::is_null) - && let Some(this) = annotated.get("this") - { - return without_comments(this); - } - Value::Object( - object - .iter() - .filter(|(key, _)| !key.ends_with(COMMENTS_SUFFIX)) - .map(|(key, child)| (key.clone(), without_comments(child))) - .collect(), - ) - } - Value::Array(items) => Value::Array(items.iter().map(without_comments).collect()), - other => other.clone(), - } -} - -/// Every comment in the tree, in order. -pub(crate) fn comments(value: &Value) -> Vec { - match value { - Value::Object(object) => object - .iter() - .flat_map( - |(key, child)| match (key.ends_with(COMMENTS_SUFFIX), child) { - (true, Value::Array(items)) => items - .iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect(), - _ => comments(child), - }, - ) - .collect(), - Value::Array(items) => items.iter().flat_map(comments).collect(), - _ => Vec::new(), - } -} - -/// The tree with variant paths flat: the faithful `a["x.y"]` and polyglot's `a.x.y` agree. -pub(crate) fn with_flat_paths(value: &Value) -> Value { - match value { - Value::Object(object) => Value::Object( - object - .iter() - .map(|(key, child)| match (key.as_str(), child) { - (JSON_EXTRACT_KEY, Value::Object(extract)) => { - (key.clone(), Value::Object(flat_extract(extract))) - } - _ => (key.clone(), with_flat_paths(child)), - }) - .collect(), - ), - Value::Array(items) => Value::Array(items.iter().map(with_flat_paths).collect()), - other => other.clone(), - } -} - -fn flat_extract(extract: &Map) -> Map { - let mut flat: Map = extract - .iter() - .map(|(key, child)| (key.clone(), with_flat_paths(child))) - .collect(); - if let Some(Value::String(path)) = flat - .get_mut("expression") - .and_then(|expression| expression.get_mut("literal")) - .and_then(|literal| literal.get_mut("value")) - { - *path = flat_path(path); - } - flat -} - -/// `["k"]` segments written as `.k`, without a leading dot. -fn flat_path(path: &str) -> String { - let mut flat = String::with_capacity(path.len()); - let mut rest = path; - while let Some(start) = rest.find("[\"") { - flat.push_str(&rest[..start]); - let Some(end) = rest[start + 2..].find("\"]") else { - break; - }; - if !flat.is_empty() { - flat.push('.'); - } - flat.push_str(&rest[start + 2..start + 2 + end].replace("\\\"", "\"")); - rest = &rest[start + 2 + end + 2..]; - } - flat.push_str(rest); - flat -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/_helpers/mod.rs b/tools/sqlquack-polyglot-check/src/comparison/_helpers/mod.rs deleted file mode 100644 index 8842797f8..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/_helpers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod faithful; -pub(crate) mod outcomes; diff --git a/tools/sqlquack-polyglot-check/src/comparison/_helpers/outcomes.rs b/tools/sqlquack-polyglot-check/src/comparison/_helpers/outcomes.rs deleted file mode 100644 index b9f89f65e..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/_helpers/outcomes.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! Parse results rendered as comparable text, and the lossless-token check. - -use std::panic::{AssertUnwindSafe, catch_unwind}; - -use sqlquack_syntax::{Dialect, TokenizeFailure}; -use sqlquack_syntax_snowflake::tokenize; - -/// `ok:` and the serialised trees, `err:` and the error's debug form, or `panic`. -pub(crate) fn rendered( - parse: impl FnOnce() -> Result, -) -> String { - match catch_unwind(AssertUnwindSafe(parse)) { - Ok(Ok(trees)) => match serde_json::to_string(&trees) { - Ok(json) => format!("ok:{json}"), - Err(error) => format!("unserialisable:{error}"), - }, - Ok(Err(error)) => format!("err:{error:?}"), - Err(_) => "panic".to_owned(), - } -} - -/// Why the lossless tokens of `sql` do not reproduce it, if they do not. -pub(crate) fn lossless_failure(sql: &str, dialect: Dialect, parsed: bool) -> Option { - let tokens = match catch_unwind(|| tokenize(sql, dialect)) { - Ok(Ok(tokens)) => tokens, - Ok(Err(TokenizeFailure::Tokenizer(_))) if !parsed => return None, - Ok(Err(failure)) => return Some(failure.to_string()), - Err(_) => return Some("the tokenizer panicked".to_owned()), - }; - let mut cursor = 0; - for token in &tokens { - if token.start != cursor || token.end <= token.start { - return Some(format!( - "token {}..{} does not continue at byte {cursor}", - token.start, token.end - )); - } - cursor = token.end; - } - match cursor == sql.len() { - true => None, - false => Some(format!("tokens end at byte {cursor} of {}", sql.len())), - } -} - -/// `value` with each Snowflake `NOT NULL` element wrapper replaced by its inner type, which polyglot lacks. -fn without_not_null(value: serde_json::Value) -> serde_json::Value { - match value { - serde_json::Value::Object(mut object) - if object.get("data_type").and_then(serde_json::Value::as_str) == Some("not_null") => - { - object - .remove("inner") - .map(without_not_null) - .unwrap_or(serde_json::Value::Object(object)) - } - serde_json::Value::Object(object) => serde_json::Value::Object( - object - .into_iter() - .map(|(key, field)| (key, without_not_null(field))) - .collect(), - ), - serde_json::Value::Array(items) => { - serde_json::Value::Array(items.into_iter().map(without_not_null).collect()) - } - other => other, - } -} - -/// polyglot's Snowflake rendering of `data_type`, cut to its upper-case base name. -pub(crate) fn polyglot_base_type_name( - data_type: &sqlquack_syntax::DataType, - dialect: polyglot_sql::DialectType, -) -> String { - let converted = serde_json::to_value(data_type) - .map(without_not_null) - .and_then(serde_json::from_value::); - let theirs = match converted { - Ok(theirs) => theirs, - Err(_) => return "".to_owned(), - }; - match polyglot_sql::generate_data_type(&theirs, dialect) { - Ok(text) => text - .split(['(', '<', ' ']) - .next() - .unwrap_or_default() - .trim() - .to_ascii_uppercase(), - Err(_) => "".to_owned(), - } -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/constants.rs b/tools/sqlquack-polyglot-check/src/comparison/constants.rs deleted file mode 100644 index 790fd8658..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/constants.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Serialised tree keys the faithful comparison reads. - -/// A plain function call. -pub const FUNCTION_KEY: &str = "function"; -/// An aggregate function call. -pub const AGGREGATE_KEY: &str = "aggregate_function"; -/// The node kind recorded when polyglot rejects a call SQLQuack keeps. -pub const ERROR_KIND: &str = "error"; -/// Prefix that makes a call's text a statement. -pub const SELECT_PREFIX: &str = "SELECT "; -/// A node annotated with comments or a type. -pub const ANNOTATED_KEY: &str = "annotated"; -/// The serialised name of a variant path node. -pub const JSON_EXTRACT_KEY: &str = "j_s_o_n_extract"; -/// The suffix of every field holding comments. -pub const COMMENTS_SUFFIX: &str = "comments"; - -/// The field a located parse records a node's source range in. -pub const SPAN_FIELD: &str = "span"; diff --git a/tools/sqlquack-polyglot-check/src/comparison/main/compare_faithful.rs b/tools/sqlquack-polyglot-check/src/comparison/main/compare_faithful.rs deleted file mode 100644 index bb3f5918e..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/main/compare_faithful.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! `compare_faithful`: SQLQuack's faithful parse against its lowered parse. - -use serde_json::Value; -use sqlquack_syntax::{CallMode, Dialect, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements_with; - -use crate::comparison::_helpers::faithful::{comments, relower, with_flat_paths, without_comments}; -use crate::comparison::models::{FaithfulComparison, Spelling}; - -/// The spellings the faithful parse keeps, and any difference they do not explain. -pub fn compare_faithful(sql: &str, dialect: Dialect) -> FaithfulComparison { - let options = ParseOptions::default(); - let lowered = parse_statements_with(sql, dialect, &options, CallMode::Lowered); - let faithful = parse_statements_with(sql, dialect, &options, CallMode::FaithfulWithSpans); - let (spellings, unexplained): (Vec, Option) = match (faithful, lowered) { - (Ok(faithful), lowered) => { - let chars: Vec = sql.chars().collect(); - let kept = serde_json::to_value(&faithful).unwrap_or_default(); - let (relowered, spellings) = relower(kept.clone(), &chars); - let rejected = spellings.iter().any(Spelling::is_rejection); - let unexplained = match lowered { - Ok(lowered) if !rejected => { - let lowered = serde_json::to_value(&lowered).unwrap_or_default(); - let (flat_relowered, flat_lowered) = - (with_flat_paths(&relowered), with_flat_paths(&lowered)); - (flat_relowered != flat_lowered - && !only_comments_moved(&flat_relowered, &flat_lowered, &kept)) - .then(|| format!("faithful, put back: {relowered}\nlowered: {lowered}")) - } - Ok(_) => Some("a call polyglot rejects alone parses in context".to_owned()), - Err(_) if rejected => None, - Err(error) => Some(format!("only the lowered parse fails: {error}")), - }; - (spellings, unexplained) - } - (Err(faithful), Err(lowered)) => ( - Vec::new(), - (faithful.to_string() != lowered.to_string()) - .then(|| format!("faithful error: {faithful}\nlowered error: {lowered}")), - ), - (Err(error), Ok(_)) => ( - Vec::new(), - Some(format!("only the faithful parse fails: {error}")), - ), - }; - FaithfulComparison { - spellings, - unexplained, - } -} - -/// Whether only comment placement differs and the faithful tree keeps every distinct comment. -fn only_comments_moved(relowered: &Value, lowered: &Value, faithful: &Value) -> bool { - if without_comments(relowered) != without_comments(lowered) { - return false; - } - let kept: Vec = comments(faithful); - comments(lowered) - .iter() - .all(|comment| kept.contains(comment)) -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/main/compare_located.rs b/tools/sqlquack-polyglot-check/src/comparison/main/compare_located.rs deleted file mode 100644 index 132e2da9e..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/main/compare_located.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! `compare_located`: a located parse against the faithful parse it must equal, spans aside. - -use serde_json::Value; -use sqlquack_syntax::{CallMode, Dialect, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements_with; - -use crate::comparison::constants::SPAN_FIELD; - -/// How the located parse of `sql` differs from its faithful parse beyond spans, if it does. -pub fn compare_located(sql: &str, dialect: Dialect) -> Option { - let parse = |mode: CallMode| -> Result { - parse_statements_with(sql, dialect, &ParseOptions::default(), mode) - .map_err(|error| format!("{error:?}")) - .and_then(|trees| serde_json::to_value(trees).map_err(|error| error.to_string())) - .map(without_spans) - }; - let faithful = parse(CallMode::Faithful); - let located = parse(CallMode::FaithfulLocated); - match faithful == located { - true => None, - false => Some(format!("faithful: {faithful:?}\nlocated: {located:?}")), - } -} - -fn without_spans(value: Value) -> Value { - match value { - Value::Object(fields) => Value::Object( - fields - .into_iter() - .filter(|(field, _)| field != SPAN_FIELD) - .map(|(field, child)| (field, without_spans(child))) - .collect(), - ), - Value::Array(items) => Value::Array(items.into_iter().map(without_spans).collect()), - other => other, - } -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/main/compare_parse.rs b/tools/sqlquack-polyglot-check/src/comparison/main/compare_parse.rs deleted file mode 100644 index 66d300900..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/main/compare_parse.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! `compare_parse`: one statement parsed by both parsers. - -use std::time::Instant; - -use polyglot_sql::DialectType; -use sqlquack_syntax::{CallMode, Dialect, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements_with; - -use crate::comparison::_helpers::outcomes::{lossless_failure, rendered}; -use crate::comparison::models::{Comparison, Outcome}; - -/// Parses `sql` with polyglot and with SQLQuack lowering calls as polyglot does, and compares. -pub fn compare_parse(sql: &str, dialect: Dialect) -> Comparison { - let polyglot_dialect = match dialect { - Dialect::Snowflake => DialectType::Snowflake, - Dialect::DuckDB => DialectType::DuckDB, - Dialect::PostgreSQL => DialectType::PostgreSQL, - Dialect::SQLite => DialectType::SQLite, - Dialect::BigQuery => DialectType::BigQuery, - Dialect::ClickHouse => DialectType::ClickHouse, - Dialect::Databricks => DialectType::Databricks, - Dialect::SqlServer => DialectType::TSQL, - }; - let started = Instant::now(); - let theirs = rendered(|| polyglot_sql::parse(sql, polyglot_dialect)); - let polyglot = started.elapsed(); - let started = Instant::now(); - let ours = rendered(|| { - parse_statements_with(sql, dialect, &ParseOptions::default(), CallMode::Lowered) - }); - let sqlquack = started.elapsed(); - let parsed = ours.starts_with("ok:"); - let outcome = if theirs != ours { - Outcome::Differs { - polyglot: theirs, - sqlquack: ours, - } - } else if let Some(reason) = lossless_failure(sql, dialect, parsed) { - Outcome::TokensLost { reason } - } else if parsed { - Outcome::SameTree - } else { - Outcome::SameError - }; - Comparison { - outcome, - polyglot, - sqlquack, - } -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/main/compare_type_names.rs b/tools/sqlquack-polyglot-check/src/comparison/main/compare_type_names.rs deleted file mode 100644 index ae9b8aea1..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/main/compare_type_names.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! `compare_type_names`: SQLQuack's cast type names against polyglot's printed types. - -use polyglot_sql::DialectType; -use sqlquack_diagnostics::snowflake::classification::main::cast_type_name::cast_type_name; -use sqlquack_syntax::{DataType, Dialect, Expression, ExpressionWalk, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements; - -use crate::comparison::_helpers::outcomes::polyglot_base_type_name; - -/// Each cast in Snowflake `sql` whose classified name differs from polyglot's, as text. -pub fn compare_type_names(sql: &str, dialect: Dialect) -> Vec { - if dialect != Dialect::Snowflake { - return Vec::new(); - } - let Ok(statements) = parse_statements(sql, dialect, &ParseOptions::default()) else { - return Vec::new(); - }; - let mut mismatches: Vec = Vec::new(); - for statement in &statements { - for node in statement.dfs() { - let target: &DataType = match node { - Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { - &cast.to - } - _ => continue, - }; - let ours = cast_type_name(target); - let theirs = polyglot_base_type_name(target, DialectType::Snowflake); - if ours != theirs { - mismatches.push(format!("{target:?}: sqlquack {ours}, polyglot {theirs}")); - } - } - } - mismatches -} diff --git a/tools/sqlquack-polyglot-check/src/comparison/main/mod.rs b/tools/sqlquack-polyglot-check/src/comparison/main/mod.rs deleted file mode 100644 index 653339189..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/main/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Entry points. - -pub mod compare_faithful; -pub mod compare_located; -pub mod compare_parse; -pub mod compare_type_names; diff --git a/tools/sqlquack-polyglot-check/src/comparison/mod.rs b/tools/sqlquack-polyglot-check/src/comparison/mod.rs deleted file mode 100644 index 6d26a4fdc..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! One statement parsed by both polyglot and SQLQuack. - -pub(crate) mod _helpers; -pub mod constants; -pub mod main; -pub mod models; diff --git a/tools/sqlquack-polyglot-check/src/comparison/models.rs b/tools/sqlquack-polyglot-check/src/comparison/models.rs deleted file mode 100644 index e86453e2d..000000000 --- a/tools/sqlquack-polyglot-check/src/comparison/models.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! What a comparison found. - -use std::time::Duration; - -/// How SQLQuack's parse compares with polyglot's. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Outcome { - /// Both parsed it into trees that serialise identically. - SameTree, - /// Both refused it with identical errors. - SameError, - /// The results differ; each side rendered as text. - Differs { - /// polyglot's result. - polyglot: String, - /// SQLQuack's result. - sqlquack: String, - }, - /// The results agree, but SQLQuack's lossless tokens do not reproduce the input. - TokensLost { - /// Why. - reason: String, - }, -} - -/// The outcome and how long each parser took. -#[derive(Clone, Debug)] -pub struct Comparison { - /// How the results compare. - pub outcome: Outcome, - /// polyglot's parse time. - pub polyglot: Duration, - /// SQLQuack's parse time. - pub sqlquack: Duration, -} - -/// A call spelling SQLQuack keeps where polyglot lowers it. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct Spelling { - /// The call's upper-case name. - pub name: String, - /// The node polyglot lowers it to, or `error` when polyglot rejects it. - pub polyglot: String, -} - -impl Spelling { - /// Whether polyglot rejects the call rather than lowering it. - pub fn is_rejection(&self) -> bool { - self.polyglot == crate::comparison::constants::ERROR_KIND - } -} - -/// What the faithful parse keeps, and what nothing explains. -#[derive(Clone, Debug)] -pub struct FaithfulComparison { - /// The kept spellings, one per call. - pub spellings: Vec, - /// A difference from the lowered parse the kept spellings do not explain. - pub unexplained: Option, -} diff --git a/tools/sqlquack-polyglot-check/src/corpora/constants.rs b/tools/sqlquack-polyglot-check/src/corpora/constants.rs deleted file mode 100644 index a67b3a3e8..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Corpus file conventions. - -/// The extension of the SQL files a directory corpus reads. -pub const SQL_EXTENSION: &str = "sql"; diff --git a/tools/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs b/tools/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs deleted file mode 100644 index 9ac504be6..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! `case_corpus`: SQLQuack's cases, as Snowflake source SQL and converted DuckDB SQL. - -use std::path::Path; - -use sqlquack::analysis::models::PipelineConverter; -use sqlquack_convert::snowflake::session::main::conversion_options::conversion_options; -use sqlquack_evidence::cases::main::load_cases::load_cases; -use sqlquack_evidence::verify::models::Conversion; -use sqlquack_evidence::verify::types::CaseConverter; -use sqlquack_policy::policy::main::load_policy::load_policy; -use sqlquack_syntax::Dialect; - -use crate::corpora::models::CorpusEntry; - -/// Every case's source SQL, and the DuckDB SQL of every case that converts. -pub fn case_corpus(cases: &Path, policy: &Path) -> Result, String> { - let policy = load_policy(policy).map_err(|error| error.to_string())?; - let loaded = load_cases(cases, &policy).map_err(|error| error.to_string())?; - let converter = PipelineConverter { - options: conversion_options(&policy, false), - python: None, - }; - let mut entries = Vec::with_capacity(loaded.len() * 2); - for case in &loaded { - entries.push(CorpusEntry { - dialect: Dialect::Snowflake, - sql: case.source_sql.clone(), - from: case.reference.clone(), - }); - if let Conversion::Converted { sql, .. } = converter.convert(case) { - entries.push(CorpusEntry { - dialect: Dialect::DuckDB, - sql, - from: format!("{} (converted)", case.reference), - }); - } - } - Ok(entries) -} diff --git a/tools/sqlquack-polyglot-check/src/corpora/main/directory_corpus.rs b/tools/sqlquack-polyglot-check/src/corpora/main/directory_corpus.rs deleted file mode 100644 index 99a95655b..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/main/directory_corpus.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! `directory_corpus`: every `.sql` file below a directory. - -use std::path::Path; - -use sqlquack_syntax::Dialect; -use walkdir::WalkDir; - -use crate::corpora::constants::SQL_EXTENSION; -use crate::corpora::models::CorpusEntry; - -/// One entry per `.sql` file below `root`, in path order, parsed in `dialect`. -pub fn directory_corpus(root: &Path, dialect: Dialect) -> Result, String> { - let mut entries: Vec = Vec::new(); - for item in WalkDir::new(root).sort_by_file_name() { - let item = item.map_err(|error| format!("cannot walk {}: {error}", root.display()))?; - let path = item.path(); - if !item.file_type().is_file() || path.extension().is_none_or(|ext| ext != SQL_EXTENSION) { - continue; - } - let sql = std::fs::read_to_string(path) - .map_err(|error| format!("cannot read {}: {error}", path.display()))?; - let from = path - .strip_prefix(root) - .unwrap_or(path) - .display() - .to_string(); - entries.push(CorpusEntry { dialect, sql, from }); - } - Ok(entries) -} diff --git a/tools/sqlquack-polyglot-check/src/corpora/main/load_corpus.rs b/tools/sqlquack-polyglot-check/src/corpora/main/load_corpus.rs deleted file mode 100644 index 756e73082..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/main/load_corpus.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! `load_corpus`: the entries of a vendored corpus file. - -use std::path::Path; - -use sqlquack_syntax::Dialect; - -use crate::corpora::models::{CorpusEntry, CorpusFile}; - -/// The entries of the corpus file at `path`; an entry for both dialects becomes two. -pub fn load_corpus(path: &Path) -> Result, String> { - let text = std::fs::read_to_string(path) - .map_err(|error| format!("cannot read {}: {error}", path.display()))?; - let file: CorpusFile = serde_json::from_str(&text) - .map_err(|error| format!("cannot parse {}: {error}", path.display()))?; - let mut entries = Vec::with_capacity(file.entries.len() * 2); - for entry in file.entries { - let dialects: &[Dialect] = match entry.dialect.as_str() { - "snowflake" => &[Dialect::Snowflake], - "duckdb" => &[Dialect::DuckDB], - "postgres" => &[Dialect::PostgreSQL], - "sqlite" => &[Dialect::SQLite], - "both" => &[Dialect::Snowflake, Dialect::DuckDB], - other => return Err(format!("{}: unknown dialect {other}", path.display())), - }; - for dialect in dialects { - entries.push(CorpusEntry { - dialect: *dialect, - sql: entry.sql.clone(), - from: entry.from.clone(), - }); - } - } - Ok(entries) -} diff --git a/tools/sqlquack-polyglot-check/src/corpora/main/mod.rs b/tools/sqlquack-polyglot-check/src/corpora/main/mod.rs deleted file mode 100644 index 381d511b7..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/main/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Entry points. - -pub mod case_corpus; -pub mod directory_corpus; -pub mod load_corpus; diff --git a/tools/sqlquack-polyglot-check/src/corpora/mod.rs b/tools/sqlquack-polyglot-check/src/corpora/mod.rs deleted file mode 100644 index 815843964..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Statements to compare: vendored corpora, directories of SQL files and SQLQuack's cases. - -pub mod constants; -pub mod main; -pub mod models; diff --git a/tools/sqlquack-polyglot-check/src/corpora/models.rs b/tools/sqlquack-polyglot-check/src/corpora/models.rs deleted file mode 100644 index 384c7d93f..000000000 --- a/tools/sqlquack-polyglot-check/src/corpora/models.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Corpus entries. - -use serde::Deserialize; -use sqlquack_syntax::Dialect; - -/// One statement, or several, to parse in one dialect. -#[derive(Clone, Debug)] -pub struct CorpusEntry { - /// The dialect to parse in. - pub dialect: Dialect, - /// The SQL. - pub sql: String, - /// Where it comes from. - pub from: String, -} - -/// A vendored corpus file. -#[derive(Debug, Deserialize)] -pub struct CorpusFile { - /// Where the corpus comes from. - pub origin: String, - /// Its entries. - pub entries: Vec, -} - -/// An entry of a vendored corpus file. -#[derive(Debug, Deserialize)] -pub struct CorpusFileEntry { - /// `snowflake`, `duckdb` or `both`. - pub dialect: String, - /// The SQL. - pub sql: String, - /// Where it comes from. - pub from: String, -} diff --git a/tools/sqlquack-polyglot-check/src/expected/_helpers/mod.rs b/tools/sqlquack-polyglot-check/src/expected/_helpers/mod.rs deleted file mode 100644 index d78aaa9ab..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/_helpers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod nodes; -pub(crate) mod quoting; diff --git a/tools/sqlquack-polyglot-check/src/expected/_helpers/nodes.rs b/tools/sqlquack-polyglot-check/src/expected/_helpers/nodes.rs deleted file mode 100644 index 6a5dc5ec1..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/_helpers/nodes.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Trees that differ only by a wrapper node or a field SQLQuack keeps and polyglot drops or infers. - -use serde_json::Value; - -use crate::expected::constants::{ - ALL_KEY, ALL_TEXT, COMMAND_TREE, EXPRESSIONS_KEY, OK_PREFIX, RAW_TREE, THIS_KEY, VAR_KEY, -}; - -/// Whether both rendered parses are trees that are equal once SQLQuack's `node` wrappers go. -pub(crate) fn differ_only_by_kept_node(polyglot: &str, sqlquack: &str, node: &str) -> bool { - let (Some(theirs), Some(ours)) = ( - polyglot.strip_prefix(OK_PREFIX), - sqlquack.strip_prefix(OK_PREFIX), - ) else { - return false; - }; - match ( - serde_json::from_str::(theirs), - serde_json::from_str::(ours), - ) { - (Ok(theirs), Ok(ours)) => theirs != ours && without_node(ours, node) == theirs, - (Err(_), _) | (_, Err(_)) => false, - } -} - -/// `value` with every `{ node: { "this": inner } }` replaced by `inner`. -fn without_node(value: Value, node: &str) -> Value { - match value { - Value::Object(mut object) => { - let wrapped: bool = object.len() == 1 - && object - .get(node) - .and_then(Value::as_object) - .is_some_and(|inner| inner.len() == 1 && inner.contains_key(THIS_KEY)); - match object.remove(node).filter(|_| wrapped) { - Some(Value::Object(mut inner)) => { - without_node(inner.remove(THIS_KEY).unwrap_or(Value::Null), node) - } - Some(other) => other, - None => Value::Object( - object - .into_iter() - .map(|(key, value)| (key, without_node(value, node))) - .collect(), - ), - } - } - Value::Array(items) => Value::Array( - items - .into_iter() - .map(|value| without_node(value, node)) - .collect(), - ), - other => other, - } -} - -/// Whether both rendered parses are trees that differ, and are equal once every `field` key goes from each. -pub(crate) fn differ_only_by_kept_field(polyglot: &str, sqlquack: &str, field: &str) -> bool { - let (Some(theirs), Some(ours)) = ( - polyglot.strip_prefix(OK_PREFIX), - sqlquack.strip_prefix(OK_PREFIX), - ) else { - return false; - }; - match ( - serde_json::from_str::(theirs), - serde_json::from_str::(ours), - ) { - (Ok(theirs), Ok(ours)) => { - theirs != ours && without_field(theirs, field) == without_field(ours, field) - } - (Err(_), _) | (_, Err(_)) => false, - } -} - -/// `value` with every object key `field` removed. -fn without_field(value: Value, field: &str) -> Value { - match value { - Value::Object(object) => Value::Object( - object - .into_iter() - .filter(|(key, _)| key != field) - .map(|(key, value)| (key, without_field(value, field))) - .collect(), - ), - Value::Array(items) => Value::Array( - items - .into_iter() - .map(|value| without_field(value, field)) - .collect(), - ), - other => other, - } -} - -/// Whether polyglot's rendered tree is one opaque command or one statement kept as raw SQL. -pub(crate) fn is_opaque(polyglot: &str) -> bool { - [COMMAND_TREE, RAW_TREE] - .iter() - .any(|opaque| polyglot.starts_with(&format!("{OK_PREFIX}{opaque}"))) -} - -/// Whether both rendered parses are trees that differ, and are equal once SQLQuack's `clause` ALL is polyglot's key. -pub(crate) fn differ_only_by_kept_all(polyglot: &str, sqlquack: &str, clause: &str) -> bool { - let (Some(theirs), Some(ours)) = ( - polyglot.strip_prefix(OK_PREFIX), - sqlquack.strip_prefix(OK_PREFIX), - ) else { - return false; - }; - match ( - serde_json::from_str::(theirs), - serde_json::from_str::(ours), - ) { - (Ok(theirs), Ok(ours)) => theirs != ours && with_all_as_key(ours, clause) == theirs, - (Err(_), _) | (_, Err(_)) => false, - } -} - -/// `value` with each `clause` of no keys and an `all` rewritten as one key `ALL` carrying its options. -fn with_all_as_key(value: Value, clause: &str) -> Value { - match value { - Value::Object(object) => Value::Object( - object - .into_iter() - .map(|(key, value)| { - let value = match (key == clause, value) { - (true, Value::Object(fields)) => all_as_key(fields), - (_, value) => value, - }; - (key, with_all_as_key(value, clause)) - }) - .collect(), - ), - Value::Array(items) => Value::Array( - items - .into_iter() - .map(|value| with_all_as_key(value, clause)) - .collect(), - ), - other => other, - } -} - -/// A clause's fields, its `all` rewritten as one key `ALL` carrying its options when it has no keys. -fn all_as_key(mut fields: serde_json::Map) -> Value { - if fields.get(EXPRESSIONS_KEY) != Some(&Value::Array(Vec::new())) { - return Value::Object(fields); - } - match fields.remove(ALL_KEY) { - Some(Value::Object(mut options)) => { - options.insert( - THIS_KEY.to_owned(), - serde_json::json!({ VAR_KEY: { THIS_KEY: ALL_TEXT } }), - ); - fields.insert( - EXPRESSIONS_KEY.to_owned(), - Value::Array(vec![Value::Object(options)]), - ); - } - Some(other) => { - fields.insert(ALL_KEY.to_owned(), other); - } - None => {} - } - Value::Object(fields) -} diff --git a/tools/sqlquack-polyglot-check/src/expected/_helpers/quoting.rs b/tools/sqlquack-polyglot-check/src/expected/_helpers/quoting.rs deleted file mode 100644 index 956d6bd9a..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/_helpers/quoting.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Statement markers and trees that differ only by quoting polyglot drops. - -use serde_json::Value; - -use crate::expected::constants::{NAME_KEY, OK_PREFIX, QUOTED_KEY}; - -/// `sql` without whitespace, in upper case, as markers are matched. -pub(crate) fn compact(sql: &str) -> String { - sql.chars() - .filter(|character| !character.is_whitespace()) - .map(|character| character.to_ascii_uppercase()) - .collect() -} - -/// Whether both rendered parses are trees that differ only by names polyglot leaves unquoted. -pub(crate) fn differ_only_by_kept_quoting(polyglot: &str, sqlquack: &str) -> bool { - let (Some(theirs), Some(ours)) = ( - polyglot.strip_prefix(OK_PREFIX), - sqlquack.strip_prefix(OK_PREFIX), - ) else { - return false; - }; - match ( - serde_json::from_str::(theirs), - serde_json::from_str::(ours), - ) { - (Ok(theirs), Ok(ours)) => theirs != ours && only_kept_quoting(&theirs, &ours), - (Err(_), _) | (_, Err(_)) => false, - } -} - -/// Whether `theirs` equals `ours` except for names polyglot marks unquoted and SQLQuack quoted. -fn only_kept_quoting(theirs: &Value, ours: &Value) -> bool { - match (theirs, ours) { - (Value::Object(theirs), Value::Object(ours)) => { - let name = ours.contains_key(NAME_KEY) && ours.contains_key(QUOTED_KEY); - theirs.len() == ours.len() - && theirs.iter().all(|(key, their_value)| match ours.get(key) { - None => false, - Some(our_value) if their_value == our_value => true, - Some(our_value) => { - (name - && key == QUOTED_KEY - && *their_value == Value::Bool(false) - && *our_value == Value::Bool(true)) - || only_kept_quoting(their_value, our_value) - } - }) - } - (Value::Array(theirs), Value::Array(ours)) => { - theirs.len() == ours.len() - && theirs - .iter() - .zip(ours) - .all(|(their_value, our_value)| only_kept_quoting(their_value, our_value)) - } - _ => theirs == ours, - } -} diff --git a/tools/sqlquack-polyglot-check/src/expected/constants.rs b/tools/sqlquack-polyglot-check/src/expected/constants.rs deleted file mode 100644 index 608adb9ba..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/constants.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! How rendered parse results begin. - -/// polyglot's rendered error starts with this. -pub const ERROR_PREFIX: &str = "err:"; -/// SQLQuack's rendered tree starts with this. -pub const OK_PREFIX: &str = "ok:"; -/// How a rendered tree of one opaque command begins after the prefix. -pub const COMMAND_TREE: &str = "[{\"command\":"; -/// How a rendered tree of one statement polyglot keeps as raw SQL begins after the prefix. -pub const RAW_TREE: &str = "[{\"raw\":"; -/// The field naming an identifier in a serialised tree. -pub const NAME_KEY: &str = "name"; -/// The field holding the operand of a wrapper node such as unary plus. -pub const THIS_KEY: &str = "this"; -/// The field recording whether an identifier was quoted. -pub const QUOTED_KEY: &str = "quoted"; -/// The field of a clause SQLQuack sets for ALL, beside an empty `expressions`. -pub const ALL_KEY: &str = "all"; -/// The field listing a clause's keys. -pub const EXPRESSIONS_KEY: &str = "expressions"; -/// The text polyglot gives ALL when it reads ALL as a key. -pub const ALL_TEXT: &str = "ALL"; -/// The node polyglot reads a bare word such as ALL into. -pub const VAR_KEY: &str = "var"; diff --git a/tools/sqlquack-polyglot-check/src/expected/main/load_expected.rs b/tools/sqlquack-polyglot-check/src/expected/main/load_expected.rs deleted file mode 100644 index c3e7ff19c..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/main/load_expected.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `load_expected`: the reviewed expected-differences file. - -use std::path::Path; - -use crate::expected::models::{ExpectedDifferences, ExpectedFile}; - -/// The expected differences in `path`; each spelling may appear once. -pub fn load_expected(path: &Path) -> Result { - let text = std::fs::read_to_string(path) - .map_err(|error| format!("cannot read {}: {error}", path.display()))?; - let file: ExpectedFile = toml::from_str(&text) - .map_err(|error| format!("cannot parse {}: {error}", path.display()))?; - let mut expected = ExpectedDifferences { - fixes: file.fixes, - rejections: file.rejections, - kept_quotings: file.kept_quotings, - kept_nodes: file.kept_nodes, - kept_fields: file.kept_fields, - kept_alls: file.kept_alls, - structured: file.structured, - ..ExpectedDifferences::default() - }; - for spelling in file.spellings { - let nodes: std::collections::BTreeSet = spelling.polyglot.into_iter().collect(); - if expected - .allowed - .insert(spelling.name.clone(), nodes) - .is_some() - { - return Err(format!( - "{}: {} is listed twice", - path.display(), - spelling.name - )); - } - } - Ok(expected) -} diff --git a/tools/sqlquack-polyglot-check/src/expected/main/mod.rs b/tools/sqlquack-polyglot-check/src/expected/main/mod.rs deleted file mode 100644 index db8a2cfd2..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/main/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Entry points. - -pub mod load_expected; diff --git a/tools/sqlquack-polyglot-check/src/expected/mod.rs b/tools/sqlquack-polyglot-check/src/expected/mod.rs deleted file mode 100644 index ba59fd909..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The reviewed list of call spellings SQLQuack keeps where polyglot lowers them. - -pub(crate) mod _helpers; -pub mod constants; -pub mod main; -pub mod models; diff --git a/tools/sqlquack-polyglot-check/src/expected/models.rs b/tools/sqlquack-polyglot-check/src/expected/models.rs deleted file mode 100644 index 02ae6ebf0..000000000 --- a/tools/sqlquack-polyglot-check/src/expected/models.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! Expected differences. - -use crate::expected::_helpers::nodes::{ - differ_only_by_kept_all, differ_only_by_kept_field, differ_only_by_kept_node, is_opaque, -}; -use crate::expected::_helpers::quoting::{compact, differ_only_by_kept_quoting}; -use crate::expected::constants::{ERROR_PREFIX, OK_PREFIX}; -use std::collections::{BTreeMap, BTreeSet}; - -use serde::Deserialize; - -/// The expected-differences file. -#[derive(Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ExpectedFile { - /// One entry per spelling. - #[serde(default, rename = "spelling")] - pub spellings: Vec, - /// One entry per reviewed parser fix. - #[serde(default, rename = "fix")] - pub fixes: Vec, - /// One entry per reviewed statement SQLQuack rejects where polyglot parses it. - #[serde(default, rename = "rejection")] - pub rejections: Vec, - /// One entry per reviewed name whose quoting polyglot drops and SQLQuack keeps. - #[serde(default, rename = "kept_quoting")] - pub kept_quotings: Vec, - /// One entry per reviewed wrapper node SQLQuack keeps and polyglot drops. - #[serde(default, rename = "kept_node")] - pub kept_nodes: Vec, - /// One entry per reviewed field SQLQuack keeps as written where polyglot drops or infers it. - #[serde(default, rename = "kept_field")] - pub kept_fields: Vec, - /// One entry per reviewed clause whose ALL SQLQuack keeps apart where polyglot reads it as a key. - #[serde(default, rename = "kept_all")] - pub kept_alls: Vec, - /// One entry per reviewed statement polyglot keeps as a command and SQLQuack structures. - #[serde(default, rename = "structured")] - pub structured: Vec, -} - -/// A statement polyglot keeps as an opaque command or raw SQL and SQLQuack parses into a structured node. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StructuredStatement { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `BEGIN`. - pub marker: String, - /// The node SQLQuack's tree starts with, such as `scripting_block`. - pub sqlquack_node: String, - /// Why SQLQuack's structure is right; a test in sqlquack-syntax pins its tree. - pub note: String, -} - -/// A statement both parsers parse, where polyglot drops the quoting of names SQLQuack keeps quoted. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct KeptQuoting { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `EXCLUDE("`. - pub marker: String, - /// Why SQLQuack's quoting is right; the trees may differ in nothing else. - pub note: String, -} - -/// A statement both parsers parse, where SQLQuack keeps a clause's ALL apart and polyglot reads it as a key. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct KeptAll { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `ORDERBYALL`. - pub marker: String, - /// The clause holding the ALL, such as `order_by`. - pub clause: String, - /// Why SQLQuack keeps ALL apart; folded into polyglot's key the trees may differ in nothing. - pub note: String, -} - -/// A statement both parsers parse, where SQLQuack keeps a wrapper node, such as unary plus, polyglot drops. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct KeptNode { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `+`. - pub marker: String, - /// The wrapper node SQLQuack keeps, such as `pos`; its only field is `this`. - pub node: String, - /// Why SQLQuack keeps it; without its wrappers the trees may differ in nothing. - pub note: String, -} - -/// A statement both parsers parse, where SQLQuack keeps a field as written, such as a list polyglot joins into one name. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct KeptField { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `SECONDARYROLES`. - pub marker: String, - /// The field, such as `roles`; without it in either tree the trees may differ in nothing. - pub field: String, - /// Why SQLQuack's field is right; a test in sqlquack-syntax pins its tree. - pub note: String, -} - -/// A statement SQLQuack rejects where polyglot parses it or fails differently, recognised by its exact text and dialect. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ParserRejection { - /// A lower-case identifier. - pub id: String, - /// The exact statement. - pub sql: String, - /// The dialects it is rejected in, such as `DuckDB`. - pub dialects: Vec, - /// Text SQLQuack's error contains. - pub sqlquack_error: String, - /// Why SQLQuack's error is right; a test in sqlquack-syntax pins it. - pub note: String, -} - -/// Valid Snowflake polyglot rejects and SQLQuack parses, recognised by its text and polyglot's error. -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ParserFix { - /// A lower-case identifier. - pub id: String, - /// Text the statement contains, with whitespace removed and ignoring case, such as `]:`. - pub marker: String, - /// Polyglot's errors on it; its error contains one of them. - pub polyglot_errors: Vec, - /// Why SQLQuack's parse is right; a test in sqlquack-syntax pins its tree. - pub note: String, -} - -/// A call spelling SQLQuack keeps as written, and what polyglot makes of it. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ExpectedSpelling { - /// The upper-case call name. - pub name: String, - /// The nodes polyglot lowers it to, by arity; `error` where polyglot rejects the call. - pub polyglot: Vec, - /// Why keeping the call changes spelling only. - pub note: String, -} - -/// The allowed polyglot nodes for each spelling. -#[derive(Clone, Debug, Default)] -pub struct ExpectedDifferences { - /// Upper-case name to the polyglot nodes allowed for it. - pub allowed: BTreeMap>, - /// The reviewed parser fixes. - pub fixes: Vec, - /// The reviewed rejections. - pub rejections: Vec, - /// The reviewed names whose quoting polyglot drops. - pub kept_quotings: Vec, - /// The reviewed wrapper nodes SQLQuack keeps and polyglot drops. - pub kept_nodes: Vec, - /// The reviewed fields SQLQuack keeps as written where polyglot drops or infers them. - pub kept_fields: Vec, - /// The reviewed clauses whose ALL SQLQuack keeps apart where polyglot reads it as a key. - pub kept_alls: Vec, - /// The reviewed statements polyglot keeps as a command and SQLQuack structures. - pub structured: Vec, -} - -impl ParserFix { - /// Whether polyglot's rendered error is one this fix lists. - fn names_error(&self, polyglot: &str) -> bool { - self.polyglot_errors - .iter() - .any(|error| polyglot.contains(error.as_str())) - } -} - -impl ExpectedDifferences { - /// The fix explaining polyglot's error `polyglot` on `sql` where SQLQuack parsed it. - pub fn fix_for(&self, sql: &str, polyglot: &str) -> Option<&ParserFix> { - let compact = compact(sql); - self.fixes.iter().find(|fix| { - polyglot.starts_with(ERROR_PREFIX) - && fix.names_error(polyglot) - && compact.contains(&fix.marker.to_ascii_uppercase()) - }) - } - - /// The rejection explaining SQLQuack's error `sqlquack` on `sql` in `dialect`, whatever polyglot made of it. - pub fn rejection_for( - &self, - sql: &str, - dialect: &str, - sqlquack: &str, - ) -> Option<&ParserRejection> { - self.rejections.iter().find(|rejection| { - sqlquack.starts_with(ERROR_PREFIX) - && sqlquack.contains(&rejection.sqlquack_error) - && rejection.sql == sql - && rejection.dialects.contains(&dialect.to_owned()) - }) - } - - /// The entry explaining where polyglot drops the quoting of a name SQLQuack keeps quoted. - pub fn kept_quoting_for( - &self, - sql: &str, - polyglot: &str, - sqlquack: &str, - ) -> Option<&KeptQuoting> { - let compact = compact(sql); - self.kept_quotings - .iter() - .find(|entry| compact.contains(&entry.marker.to_ascii_uppercase())) - .filter(|_| differ_only_by_kept_quoting(polyglot, sqlquack)) - } - - /// The entry explaining where SQLQuack keeps a wrapper node polyglot drops. - pub fn kept_node_for(&self, sql: &str, polyglot: &str, sqlquack: &str) -> Option<&KeptNode> { - let compact = compact(sql); - self.kept_nodes.iter().find(|entry| { - compact.contains(&entry.marker.to_ascii_uppercase()) - && differ_only_by_kept_node(polyglot, sqlquack, &entry.node) - }) - } - - /// The entry explaining where SQLQuack keeps a field as written that polyglot drops or infers. - pub fn kept_field_for(&self, sql: &str, polyglot: &str, sqlquack: &str) -> Option<&KeptField> { - let compact = compact(sql); - self.kept_fields.iter().find(|entry| { - compact.contains(&entry.marker.to_ascii_uppercase()) - && differ_only_by_kept_field(polyglot, sqlquack, &entry.field) - }) - } - - /// The entry explaining where SQLQuack keeps a clause's ALL apart that polyglot reads as a key. - pub fn kept_all_for(&self, sql: &str, polyglot: &str, sqlquack: &str) -> Option<&KeptAll> { - let compact = compact(sql); - self.kept_alls.iter().find(|entry| { - compact.contains(&entry.marker.to_ascii_uppercase()) - && differ_only_by_kept_all(polyglot, sqlquack, &entry.clause) - }) - } - - /// The entry explaining where SQLQuack structures a statement polyglot keeps as a command. - pub fn structured_for( - &self, - sql: &str, - polyglot: &str, - sqlquack: &str, - ) -> Option<&StructuredStatement> { - let compact = compact(sql); - self.structured.iter().find(|entry| { - is_opaque(polyglot) - && sqlquack.starts_with(&format!("{OK_PREFIX}[{{\"{}\":", entry.sqlquack_node)) - && compact.contains(&entry.marker.to_ascii_uppercase()) - }) - } - - /// Whether keeping `name` where polyglot makes `polyglot` of it is listed. - pub fn allows(&self, name: &str, polyglot: &str) -> bool { - self.allowed - .get(name) - .is_some_and(|nodes| nodes.contains(polyglot)) - } -} diff --git a/tools/sqlquack-polyglot-check/src/lib.rs b/tools/sqlquack-polyglot-check/src/lib.rs deleted file mode 100644 index 6ee1b4854..000000000 --- a/tools/sqlquack-polyglot-check/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Development check that SQLQuack's parser gives exactly polyglot's results; never a runtime dependency. - -#![forbid(unsafe_code)] - -pub mod cli; -pub mod comparison; -pub mod corpora; -pub mod expected; -pub mod runs; diff --git a/tools/sqlquack-polyglot-check/src/runs/_helpers/mod.rs b/tools/sqlquack-polyglot-check/src/runs/_helpers/mod.rs deleted file mode 100644 index 4dce7b856..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/_helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod passes; diff --git a/tools/sqlquack-polyglot-check/src/runs/_helpers/passes.rs b/tools/sqlquack-polyglot-check/src/runs/_helpers/passes.rs deleted file mode 100644 index 34bf549b0..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/_helpers/passes.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Whole-corpus parse passes. - -use std::time::{Duration, Instant}; - -use polyglot_sql::DialectType; -use sqlquack_syntax::{Dialect, ParseOptions}; -use sqlquack_syntax_snowflake::parse_statements; - -use crate::corpora::models::CorpusEntry; - -/// polyglot's time to parse every entry once. -pub(crate) fn polyglot_pass(entries: &[CorpusEntry]) -> Duration { - let started = Instant::now(); - for entry in entries { - let dialect = match entry.dialect { - Dialect::Snowflake => DialectType::Snowflake, - Dialect::DuckDB => DialectType::DuckDB, - Dialect::PostgreSQL => DialectType::PostgreSQL, - Dialect::SQLite => DialectType::SQLite, - Dialect::BigQuery => DialectType::BigQuery, - Dialect::ClickHouse => DialectType::ClickHouse, - Dialect::Databricks => DialectType::Databricks, - Dialect::SqlServer => DialectType::TSQL, - }; - let _ = std::hint::black_box(polyglot_sql::parse(&entry.sql, dialect)); - } - started.elapsed() -} - -/// SQLQuack's time to parse every entry once. -pub(crate) fn sqlquack_pass(entries: &[CorpusEntry]) -> Duration { - let started = Instant::now(); - let options = ParseOptions::default(); - for entry in entries { - let _ = std::hint::black_box(parse_statements(&entry.sql, entry.dialect, &options)); - } - started.elapsed() -} - -/// The middle time, or zero when there are none. -pub(crate) fn median(mut times: Vec) -> Duration { - times.sort(); - times.get(times.len() / 2).copied().unwrap_or_default() -} diff --git a/tools/sqlquack-polyglot-check/src/runs/main/mod.rs b/tools/sqlquack-polyglot-check/src/runs/main/mod.rs deleted file mode 100644 index 61f69f955..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/main/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Entry points. - -pub mod run_corpus; -pub mod time_parsers; diff --git a/tools/sqlquack-polyglot-check/src/runs/main/run_corpus.rs b/tools/sqlquack-polyglot-check/src/runs/main/run_corpus.rs deleted file mode 100644 index 384fe3a78..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/main/run_corpus.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! `run_corpus`: every entry of a corpus compared. - -use crate::comparison::main::compare_faithful::compare_faithful; -use crate::comparison::main::compare_located::compare_located; -use crate::comparison::main::compare_parse::compare_parse; -use crate::comparison::main::compare_type_names::compare_type_names; -use crate::comparison::models::Outcome; -use crate::corpora::models::CorpusEntry; -use crate::expected::constants::OK_PREFIX; -use crate::expected::models::ExpectedDifferences; -use crate::runs::models::{Finding, RunSummary}; - -/// Compares every entry, with faithful calls differing only as `expected` allows. -pub fn run_corpus(entries: &[CorpusEntry], expected: &ExpectedDifferences) -> RunSummary { - let mut summary = RunSummary { - entries: entries.len(), - ..RunSummary::default() - }; - for entry in entries { - let comparison = compare_parse(&entry.sql, entry.dialect); - summary.polyglot += comparison.polyglot; - summary.sqlquack += comparison.sqlquack; - let from = format!("{} ({:?})", entry.from, entry.dialect); - let faithful = compare_faithful(&entry.sql, entry.dialect); - for spelling in faithful.spellings { - if !expected.allows(&spelling.name, &spelling.polyglot) { - summary.unlisted.push(Finding { - from: from.clone(), - detail: format!("unlisted: {} -> {}", spelling.name, spelling.polyglot), - }); - } - *summary.spellings.entry(spelling).or_default() += 1; - } - if let Some(detail) = faithful.unexplained { - summary.unexplained.push(Finding { - from: from.clone(), - detail, - }); - } - if let Some(detail) = compare_located(&entry.sql, entry.dialect) { - summary.located.push(Finding { - from: from.clone(), - detail, - }); - } - for detail in compare_type_names(&entry.sql, entry.dialect) { - summary.type_names.push(Finding { - from: from.clone(), - detail, - }); - } - match comparison.outcome { - Outcome::SameTree => summary.same_trees += 1, - Outcome::SameError => summary.same_errors += 1, - Outcome::Differs { polyglot, sqlquack } => { - let fix = expected - .fix_for(&entry.sql, &polyglot) - .filter(|_| sqlquack.starts_with(OK_PREFIX)); - let dialect = format!("{:?}", entry.dialect); - let rejection = expected.rejection_for(&entry.sql, &dialect, &sqlquack); - let kept_quoting = expected.kept_quoting_for(&entry.sql, &polyglot, &sqlquack); - let kept_node = expected.kept_node_for(&entry.sql, &polyglot, &sqlquack); - let kept_field = expected - .kept_field_for(&entry.sql, &polyglot, &sqlquack) - .map(|entry| entry.id.clone()) - .or_else(|| { - expected - .kept_all_for(&entry.sql, &polyglot, &sqlquack) - .map(|entry| entry.id.clone()) - }); - let structured = expected.structured_for(&entry.sql, &polyglot, &sqlquack); - match ( - fix, - rejection, - kept_quoting, - kept_node, - kept_field, - structured, - ) { - (Some(fix), ..) => *summary.fixed.entry(fix.id.clone()).or_default() += 1, - (None, Some(rejection), ..) => { - *summary.rejected.entry(rejection.id.clone()).or_default() += 1; - } - (None, None, Some(kept_quoting), ..) => { - *summary - .kept_quoting - .entry(kept_quoting.id.clone()) - .or_default() += 1; - } - (None, None, None, Some(kept_node), ..) => { - *summary.kept_nodes.entry(kept_node.id.clone()).or_default() += 1; - } - (None, None, None, None, Some(kept_field), _) => { - *summary.kept_fields.entry(kept_field).or_default() += 1; - } - (None, None, None, None, None, Some(structured)) => { - *summary.structured.entry(structured.id.clone()).or_default() += 1; - } - (None, None, None, None, None, None) => summary.differences.push(Finding { - from, - detail: format!("polyglot: {polyglot}\nsqlquack: {sqlquack}"), - }), - } - } - Outcome::TokensLost { reason } => summary.tokens_lost.push(Finding { - from, - detail: reason, - }), - } - } - summary -} diff --git a/tools/sqlquack-polyglot-check/src/runs/main/time_parsers.rs b/tools/sqlquack-polyglot-check/src/runs/main/time_parsers.rs deleted file mode 100644 index 2a9fa1694..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/main/time_parsers.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! `time_parsers`: each parser's median time over a whole corpus. - -use std::time::Duration; - -use crate::corpora::models::CorpusEntry; -use crate::runs::_helpers::passes::{median, polyglot_pass, sqlquack_pass}; -use crate::runs::models::ParserTimes; - -/// Median whole-corpus parse times over `rounds` rounds, alternating which parser runs first. -pub fn time_parsers(entries: &[CorpusEntry], rounds: usize) -> ParserTimes { - let mut polyglot: Vec = Vec::with_capacity(rounds); - let mut sqlquack: Vec = Vec::with_capacity(rounds); - for round in 0..rounds { - if round.is_multiple_of(2) { - polyglot.push(polyglot_pass(entries)); - sqlquack.push(sqlquack_pass(entries)); - } else { - sqlquack.push(sqlquack_pass(entries)); - polyglot.push(polyglot_pass(entries)); - } - } - ParserTimes { - polyglot: median(polyglot), - sqlquack: median(sqlquack), - } -} diff --git a/tools/sqlquack-polyglot-check/src/runs/mod.rs b/tools/sqlquack-polyglot-check/src/runs/mod.rs deleted file mode 100644 index ec5798fc4..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! A whole corpus compared, with counts and timings. - -pub(crate) mod _helpers; -pub mod main; -pub mod models; diff --git a/tools/sqlquack-polyglot-check/src/runs/models.rs b/tools/sqlquack-polyglot-check/src/runs/models.rs deleted file mode 100644 index d0c578274..000000000 --- a/tools/sqlquack-polyglot-check/src/runs/models.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Run summaries. - -use std::collections::BTreeMap; -use std::time::Duration; - -use crate::comparison::models::Spelling; - -/// A statement whose results differ, or whose tokens are not lossless. -#[derive(Clone, Debug)] -pub struct Finding { - /// Where the statement comes from. - pub from: String, - /// What differs. - pub detail: String, -} - -/// Counts and timings for one corpus. -#[derive(Clone, Debug, Default)] -pub struct RunSummary { - /// Entries compared. - pub entries: usize, - /// Entries both parsers parsed into identical trees. - pub same_trees: usize, - /// Entries both parsers refused with identical errors. - pub same_errors: usize, - /// Entries polyglot rejects and SQLQuack parses, as a reviewed fix explains. - pub fixed: BTreeMap, - /// Entries SQLQuack rejects where polyglot parses them or fails differently, as a reviewed rejection explains. - pub rejected: BTreeMap, - /// Entries whose trees differ only by quoting polyglot drops, as a reviewed entry explains. - pub kept_quoting: BTreeMap, - /// Entries whose trees differ only by wrapper nodes polyglot drops, as a reviewed entry explains. - pub kept_nodes: BTreeMap, - /// Entries whose trees differ only by a field kept as written, as a reviewed entry explains. - pub kept_fields: BTreeMap, - /// Entries polyglot keeps as a command and SQLQuack structures, as a reviewed entry explains. - pub structured: BTreeMap, - /// Entries whose results differ. - pub differences: Vec, - /// Entries whose lossless tokens do not reproduce the input. - pub tokens_lost: Vec, - /// Casts whose classified type name differs from polyglot's printed type. - pub type_names: Vec, - /// Kept call spellings and how many calls used each. - pub spellings: BTreeMap, - /// Kept spellings the expected-differences list does not allow. - pub unlisted: Vec, - /// Located parses that differ from the faithful parse beyond their spans. - pub located: Vec, - /// Faithful parses that differ from the lowered parse beyond their kept spellings. - pub unexplained: Vec, - /// polyglot's total parse time. - pub polyglot: Duration, - /// SQLQuack's total parse time. - pub sqlquack: Duration, -} - -impl RunSummary { - /// Whether every entry agrees, keeps its tokens and cast names, and differs only as listed. - pub fn exact(&self) -> bool { - self.differences.is_empty() - && self.tokens_lost.is_empty() - && self.type_names.is_empty() - && self.unlisted.is_empty() - && self.unexplained.is_empty() - && self.located.is_empty() - } -} - -/// Median whole-corpus parse times. -#[derive(Clone, Copy, Debug, Default)] -pub struct ParserTimes { - /// polyglot's. - pub polyglot: Duration, - /// SQLQuack's. - pub sqlquack: Duration, -} diff --git a/tools/sqlquack-polyglot-check/tests/runs.rs b/tools/sqlquack-polyglot-check/tests/runs.rs deleted file mode 100644 index 650114413..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "runs/helpers.rs"] -mod helpers; -#[path = "runs/test_exact.rs"] -mod test_exact; -#[path = "runs/test_kept_nodes.rs"] -mod test_kept_nodes; -#[path = "runs/test_kept_quoting.rs"] -mod test_kept_quoting; -#[path = "runs/test_rejections.rs"] -mod test_rejections; -#[path = "runs/test_structured.rs"] -mod test_structured; -#[path = "runs/test_types.rs"] -mod test_types; diff --git a/tools/sqlquack-polyglot-check/tests/runs/helpers.rs b/tools/sqlquack-polyglot-check/tests/runs/helpers.rs deleted file mode 100644 index 97136385d..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/helpers.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Corpora and outcomes shared by the exact-check tests. - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use sqlquack_polyglot_check::corpora::models::CorpusEntry; -use sqlquack_polyglot_check::expected::main::load_expected::load_expected; -use sqlquack_polyglot_check::runs::main::run_corpus::run_corpus; -use sqlquack_polyglot_check::runs::models::RunSummary; - -use crate::test_types::ExactOutcome; - -/// Findings kept in an outcome, so a failure shows the first few in full. -const SHOWN_FINDINGS: usize = 5; - -/// The repository root. -pub(crate) fn repository() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") -} - -/// The translation cases: under `SQLQUACK_EVIDENCE_ROOT` when the make targets set it, else in-repo. -pub(crate) fn repository_cases() -> PathBuf { - std::env::var_os("SQLQUACK_EVIDENCE_ROOT").map_or_else( - || repository().join("cases"), - |root| PathBuf::from(root).join("convert/snowflake/cases"), - ) -} - -/// The summary of comparing `entries` under the committed expected differences. -pub(crate) fn summary_of(entries: &[CorpusEntry]) -> RunSummary { - let expected = - load_expected(&repository().join("corpora/polyglot-check/expected_differences.toml")) - .expect("the expected differences load"); - run_corpus(entries, &expected) -} - -/// The findings of comparing every entry, or a failed load. -pub(crate) fn exact_outcome(entries: &Result, String>) -> ExactOutcome { - let Ok(entries) = entries else { - return ExactOutcome { - loaded: false, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }; - }; - let summary = summary_of(entries); - let shown = |findings: &[sqlquack_polyglot_check::runs::models::Finding]| -> Vec { - findings - .iter() - .take(SHOWN_FINDINGS) - .map(|finding| format!("{}\n{}", finding.from, finding.detail)) - .collect() - }; - ExactOutcome { - loaded: true, - rejected: summary.rejected.clone(), - differences: shown(&summary.differences), - tokens_lost: shown(&summary.tokens_lost), - type_names: shown(&summary.type_names), - unlisted: shown(&summary.unlisted), - unexplained: shown(&summary.unexplained), - located: shown(&summary.located), - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_exact.rs b/tools/sqlquack-polyglot-check/tests/runs/test_exact.rs deleted file mode 100644 index d16ebd12d..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_exact.rs +++ /dev/null @@ -1,304 +0,0 @@ -//! The exact check on every corpus committed with SQLQuack: zero differences, lossless tokens. - -use std::collections::BTreeMap; - -use sqlquack_polyglot_check::corpora::main::case_corpus::case_corpus; -use sqlquack_polyglot_check::corpora::main::load_corpus::load_corpus; - -use crate::helpers::{exact_outcome, repository, repository_cases}; -use crate::test_types::{CorpusTestCase, ExactOutcome}; - -#[test] -fn given_committed_corpora_when_parsed_by_both_parsers_then_results_are_identical() { - let root = repository(); - let corpora = root.join("corpora/polyglot-check"); - let test_cases = [ - CorpusTestCase { - description: "the SQL inputs of polyglot's tests", - entries: load_corpus(&corpora.join("polyglot_tests.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::from([ - ("malformed_typed_type_text".to_owned(), 1), - ("snowflake_bare_values_cte".to_owned(), 1), - ("snowflake_table_unset_projection_policy".to_owned(), 1), - ("snowflake_put_prose_with_quoted_file_uri".to_owned(), 1), - ("snowflake_put_prose_with_placeholder".to_owned(), 1), - ("snowflake_put_prose_with_quoted_stage".to_owned(), 1), - ("snowflake_put_prose_minimal_with".to_owned(), 1), - ("snowflake_put_prose_roundtrip".to_owned(), 1), - ( - "snowflake_put_prose_with_dollar_quoted_stage_target".to_owned(), - 1, - ), - ("snowflake_put_prose_with_uuid_name_path".to_owned(), 1), - ( - "snowflake_put_prose_with_uuid_compressupload_path".to_owned(), - 1, - ), - ( - "snowflake_put_prose_with_stage_uuid_auto_compress_false".to_owned(), - 1, - ), - ( - "snowflake_put_prose_with_stage_uuid_auto_compress_true".to_owned(), - 1, - ), - ( - "snowflake_put_prose_with_long_path_stage_uuid".to_owned(), - 1, - ), - ("snowflake_cte_without_as_prose".to_owned(), 1), - ("snowflake_copy_prose_into_with".to_owned(), 1), - ("snowflake_copy_prose_on_error".to_owned(), 1), - ("snowflake_copy_prose_reversed".to_owned(), 1), - ("snowflake_copy_prose_minimal".to_owned(), 1), - ("snowflake_copy_prose_roundtrip".to_owned(), 1), - ("snowflake_reserved_type_name".to_owned(), 1), - ("snowflake_string_part_extract".to_owned(), 1), - ("snowflake_function_body_number".to_owned(), 1), - ("snowflake_collate_needs_string".to_owned(), 1), - ("snowflake_dot_after_literal".to_owned(), 1), - ("snowflake_by_name_on_except".to_owned(), 1), - ("snowflake_by_name_on_intersect".to_owned(), 1), - ("snowflake_empty_select_list".to_owned(), 1), - ("snowflake_empty_select_list_before_from".to_owned(), 1), - ( - "snowflake_empty_select_list_keyword_qualifier".to_owned(), - 1, - ), - ("snowflake_empty_select_list_lower_case".to_owned(), 1), - ("snowflake_empty_select_list_stage_text".to_owned(), 1), - ( - "snowflake_empty_select_list_trailing_order_by".to_owned(), - 1, - ), - ("snowflake_empty_select_list_trailing_space".to_owned(), 1), - ("snowflake_reserved_distinct_alias".to_owned(), 1), - ("snowflake_reserved_is_alias".to_owned(), 1), - ("snowflake_set_operation_join_side".to_owned(), 1), - ("snowflake_describe_bare_name".to_owned(), 1), - ("snowflake_show_profile".to_owned(), 1), - ("snowflake_show_groups".to_owned(), 1), - ("snowflake_show_create_table".to_owned(), 1), - ("snowflake_stage_reserved_name".to_owned(), 1), - ("snowflake_stage_quoted_blank_name".to_owned(), 1), - ("snowflake_list_function_call".to_owned(), 1), - ("snowflake_list_function_order_by".to_owned(), 1), - ("snowflake_list_without_stage".to_owned(), 1), - ("snowflake_unpivot_reserved_value_column".to_owned(), 1), - ("snowflake_show_grants_for".to_owned(), 1), - ("snowflake_role_grant_without_role".to_owned(), 1), - ( - "snowflake_role_grant_without_role_quoted_user".to_owned(), - 1, - ), - ( - "snowflake_role_grant_without_role_admin_option".to_owned(), - 1, - ), - ("snowflake_role_grant_without_role_lists".to_owned(), 1), - ("snowflake_role_grant_without_role_user_list".to_owned(), 1), - ("snowflake_role_grant_quoted_roles".to_owned(), 1), - ( - "snowflake_role_grant_without_role_after_select".to_owned(), - 1, - ), - ( - "snowflake_role_grant_without_role_unterminated".to_owned(), - 1, - ), - ("snowflake_role_revoke_without_role".to_owned(), 1), - ( - "snowflake_role_revoke_admin_option_without_role".to_owned(), - 1, - ), - ("snowflake_role_revoke_without_role_lists".to_owned(), 1), - ]), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "sqlglot's fixtures", - entries: load_corpus(&corpora.join("sqlglot.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::from([ - ("snowflake_function_body_return".to_owned(), 1), - ("snowflake_function_body_return_contains_sql".to_owned(), 1), - ("snowflake_function_body_return_modifies_sql".to_owned(), 1), - ("snowflake_function_body_return_reads_sql".to_owned(), 1), - ("snowflake_cte_without_as".to_owned(), 1), - ("snowflake_empty_call_argument".to_owned(), 1), - ("snowflake_reserved_cte_name".to_owned(), 1), - ("snowflake_reserved_truncate_target".to_owned(), 1), - ( - "snowflake_copy_unload_reserved_table_credentials".to_owned(), - 1, - ), - ( - "snowflake_copy_unload_reserved_table_credentials_flat".to_owned(), - 1, - ), - ( - "snowflake_copy_unload_reserved_table_integration".to_owned(), - 1, - ), - ( - "snowflake_copy_unload_reserved_table_integration_flat".to_owned(), - 1, - ), - ( - "snowflake_empty_select_list_trailing_order_by".to_owned(), - 1, - ), - ("snowflake_string_part_date_part".to_owned(), 1), - ("snowflake_frame_bare_preceding".to_owned(), 1), - ("snowflake_frame_value_without_direction".to_owned(), 1), - ("snowflake_frame_value_without_direction_end".to_owned(), 1), - ("snowflake_describe_bare_name".to_owned(), 1), - ("snowflake_describe_bare_qualified_name".to_owned(), 1), - ("snowflake_describe_extended".to_owned(), 1), - ("snowflake_describe_formatted".to_owned(), 1), - ("snowflake_describe_query".to_owned(), 1), - ("snowflake_describe_bare_quoted_name".to_owned(), 1), - ("snowflake_stage_string_name".to_owned(), 1), - ("snowflake_bare_interval_type_double_colon".to_owned(), 1), - ("snowflake_bare_interval_type_cast".to_owned(), 1), - ( - "snowflake_bare_interval_type_double_colon_window".to_owned(), - 1, - ), - ("snowflake_bare_interval_type_cast_window".to_owned(), 1), - ("snowflake_bare_interval_type_cast_string".to_owned(), 1), - ("snowflake_int_precision_default_uuid".to_owned(), 1), - ("snowflake_view_without_query_columns".to_owned(), 1), - ("snowflake_view_without_query_column_comments".to_owned(), 1), - ("snowflake_merge_insert_star".to_owned(), 1), - ("snowflake_view_row_access_policy_without_on".to_owned(), 1), - ("snowflake_int_precision_default_null_comment".to_owned(), 1), - ("snowflake_int_precision_not_null_default".to_owned(), 1), - ( - "snowflake_int_precision_not_null_default_negative".to_owned(), - 1, - ), - ( - "snowflake_int_precision_collate_auto_increment".to_owned(), - 1, - ), - ]), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "sqlglot's PostgreSQL tests", - entries: load_corpus(&corpora.join("sqlglot_postgres.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "PostgreSQL's regression statements", - entries: load_corpus(&corpora.join("postgres_regress.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "sqlglot's SQLite tests", - entries: load_corpus(&corpora.join("sqlglot_sqlite.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "sqllogictest's statements", - entries: load_corpus(&corpora.join("sqllogictest.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "every Snowflake type spelling in each cast form", - entries: load_corpus(&corpora.join("snowflake_types.json")), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::from([ - ("snowflake_type_bare_interval_cast".to_owned(), 1), - ("snowflake_type_bare_interval_try_cast".to_owned(), 1), - ("snowflake_type_bare_interval_double_colon".to_owned(), 1), - ]), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - CorpusTestCase { - description: "SQLQuack's cases, source and converted", - entries: case_corpus( - &repository_cases(), - &root.join(sqlquack_policy::policy::constants::EVIDENCE_FILE), - ), - expected_outcome: ExactOutcome { - loaded: true, - rejected: BTreeMap::new(), - differences: Vec::new(), - tokens_lost: Vec::new(), - type_names: Vec::new(), - unlisted: Vec::new(), - unexplained: Vec::new(), - located: Vec::new(), - }, - }, - ]; - - for test_case in test_cases { - assert_eq!( - exact_outcome(&test_case.entries), - test_case.expected_outcome, - "{}", - test_case.description - ); - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_kept_nodes.rs b/tools/sqlquack-polyglot-check/tests/runs/test_kept_nodes.rs deleted file mode 100644 index d340bf61a..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_kept_nodes.rs +++ /dev/null @@ -1,332 +0,0 @@ -//! Reviewed kept nodes and fields cover only what polyglot drops or infers, in statements with the marker. - -use sqlquack_polyglot_check::corpora::models::CorpusEntry; -use sqlquack_polyglot_check::expected::models::{ - ExpectedDifferences, KeptAll, KeptField, KeptNode, -}; -use sqlquack_syntax::Dialect; - -use crate::helpers::summary_of; -use crate::test_types::{KeptAllTestCase, KeptNodeTestCase, QuotingTreesTestCase}; - -#[test] -fn given_unary_plus_when_compared_then_only_the_kept_operator_is_explained() { - let test_cases = [ - KeptNodeTestCase { - description: "a unary plus on a literal is a kept node", - sql: "SELECT + 1", - expected_kept_nodes: 1, - expected_differences: 0, - }, - KeptNodeTestCase { - description: "a unary plus on a column is a kept node", - sql: "SELECT +IF FROM t", - expected_kept_nodes: 1, - expected_differences: 0, - }, - KeptNodeTestCase { - description: "a unary minus gives identical trees", - sql: "SELECT - 1", - expected_kept_nodes: 0, - expected_differences: 0, - }, - KeptNodeTestCase { - description: "a binary plus gives identical trees", - sql: "SELECT 1 + 2", - expected_kept_nodes: 0, - expected_differences: 0, - }, - ]; - - for test_case in test_cases { - let summary = summary_of(&[CorpusEntry { - dialect: Dialect::Snowflake, - sql: test_case.sql.to_owned(), - from: test_case.description.to_owned(), - }]); - assert_eq!( - ( - summary.kept_nodes.values().sum::(), - summary.differences.len() - ), - ( - test_case.expected_kept_nodes, - test_case.expected_differences - ), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_trees_with_a_marker_when_matched_then_only_dropped_wrappers_are_allowed() { - let column = |name: &str| format!(r#"{{"column":{{"name":"{name}"}}}}"#); - let pos = |inner: String| format!(r#"{{"pos":{{"this":{inner}}}}}"#); - let tree = |first: String, second: String| format!(r#"ok:[{{"select":[{first},{second}]}}]"#); - let test_cases = [ - QuotingTreesTestCase { - description: "a wrapper polyglot drops is the only difference", - sql: "SELECT +a, b", - polyglot: tree(column("a"), column("b")), - sqlquack: tree(pos(column("a")), column("b")), - expected_match: true, - }, - QuotingTreesTestCase { - description: "nested wrappers polyglot drops are the only difference", - sql: "SELECT + +a, b", - polyglot: tree(column("a"), column("b")), - sqlquack: tree(pos(pos(column("a"))), column("b")), - expected_match: true, - }, - QuotingTreesTestCase { - description: "a different name beside the wrapper is not matched", - sql: "SELECT +a, b", - polyglot: tree(column("a"), column("c")), - sqlquack: tree(pos(column("a")), column("b")), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a wrapper polyglot keeps and SQLQuack drops is not matched", - sql: "SELECT +a, b", - polyglot: tree(pos(column("a")), column("b")), - sqlquack: tree(column("a"), column("b")), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a wrapper with a field besides its operand is not matched", - sql: "SELECT +a, b", - polyglot: tree(column("a"), column("b")), - sqlquack: tree( - format!(r#"{{"pos":{{"this":{},"span":null}}}}"#, column("a")), - column("b"), - ), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a statement without the marker is not matched", - sql: "SELECT a, b", - polyglot: tree(column("a"), column("b")), - sqlquack: tree(pos(column("a")), column("b")), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a polyglot error is not matched", - sql: "SELECT +a, b", - polyglot: "err:Parse".to_owned(), - sqlquack: tree(pos(column("a")), column("b")), - expected_match: false, - }, - ]; - let expected = ExpectedDifferences { - kept_nodes: vec![KeptNode { - id: "unary_plus".to_owned(), - marker: "+".to_owned(), - node: "pos".to_owned(), - note: "a test entry".to_owned(), - }], - ..ExpectedDifferences::default() - }; - - for test_case in test_cases { - assert_eq!( - expected - .kept_node_for(test_case.sql, &test_case.polyglot, &test_case.sqlquack) - .is_some(), - test_case.expected_match, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_trees_with_a_marker_when_matched_then_only_the_kept_field_may_differ() { - let tree = |scope: &str, name: &str| { - format!(r#"ok:[{{"show":{{"this":"TABLES","scope_kind":{scope},"scope":"{name}"}}}}]"#) - }; - let test_cases = [ - QuotingTreesTestCase { - description: "a field polyglot infers is the only difference", - sql: "SHOW TABLES IN a.b", - polyglot: tree(r#""SCHEMA""#, "a.b"), - sqlquack: tree("null", "a.b"), - expected_match: true, - }, - QuotingTreesTestCase { - description: "a field polyglot lacks is the only difference", - sql: "SHOW TABLES IN a.b", - polyglot: r#"ok:[{"show":{"this":"TABLES","scope":"a.b"}}]"#.to_owned(), - sqlquack: tree("null", "a.b"), - expected_match: true, - }, - QuotingTreesTestCase { - description: "another difference beside the field is not matched", - sql: "SHOW TABLES IN a.b", - polyglot: tree(r#""SCHEMA""#, "a.b"), - sqlquack: tree("null", "a.c"), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a statement without the marker is not matched", - sql: "DESC TABLE a.b", - polyglot: tree(r#""SCHEMA""#, "a.b"), - sqlquack: tree("null", "a.b"), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a polyglot error is not matched", - sql: "SHOW TABLES IN a.b", - polyglot: "err:Parse".to_owned(), - sqlquack: tree("null", "a.b"), - expected_match: false, - }, - ]; - let expected = ExpectedDifferences { - kept_fields: vec![KeptField { - id: "show_container".to_owned(), - marker: "SHOW".to_owned(), - field: "scope_kind".to_owned(), - note: "a test entry".to_owned(), - }], - ..ExpectedDifferences::default() - }; - - for test_case in test_cases { - assert_eq!( - expected - .kept_field_for(test_case.sql, &test_case.polyglot, &test_case.sqlquack) - .is_some(), - test_case.expected_match, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_order_by_all_when_compared_then_the_kept_all_is_explained_with_its_options() { - let test_cases = [ - KeptAllTestCase { - description: "ORDER BY ALL is a kept ALL", - sql: "SELECT a, b FROM t ORDER BY ALL", - expected_kept_fields: 1, - expected_differences: 0, - }, - KeptAllTestCase { - description: "ORDER BY ALL DESC NULLS FIRST carries its options", - sql: "SELECT a, b FROM t ORDER BY ALL DESC NULLS FIRST", - expected_kept_fields: 1, - expected_differences: 0, - }, - KeptAllTestCase { - description: "ORDER BY a column gives identical trees", - sql: "SELECT a, b FROM t ORDER BY a DESC", - expected_kept_fields: 0, - expected_differences: 0, - }, - ]; - - for test_case in test_cases { - let summary = summary_of(&[CorpusEntry { - dialect: Dialect::Snowflake, - sql: test_case.sql.to_owned(), - from: test_case.description.to_owned(), - }]); - assert_eq!( - ( - summary.kept_fields.values().sum::(), - summary.differences.len() - ), - ( - test_case.expected_kept_fields, - test_case.expected_differences - ), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_trees_with_a_marker_when_matched_then_only_the_folded_all_may_differ() { - let key = |this: &str, desc: bool| { - format!(r#"{{"this":{this},"desc":{desc},"nulls_first":null,"explicit_asc":false}}"#) - }; - let all_key = |desc: bool| key(r#"{"var":{"this":"ALL"}}"#, desc); - let polyglot = - |keys: String| format!(r#"ok:[{{"select":{{"order_by":{{"expressions":[{keys}]}}}}}}]"#); - let sqlquack = |desc: bool| { - format!( - r#"ok:[{{"select":{{"order_by":{{"expressions":[],"all":{{"desc":{desc},"nulls_first":null,"explicit_asc":false}}}}}}}}]"# - ) - }; - let test_cases = [ - QuotingTreesTestCase { - description: "ALL read as a key is the only difference", - sql: "SELECT a FROM t ORDER BY ALL DESC", - polyglot: polyglot(all_key(true)), - sqlquack: sqlquack(true), - expected_match: true, - }, - QuotingTreesTestCase { - description: "a different direction is not matched", - sql: "SELECT a FROM t ORDER BY ALL DESC", - polyglot: polyglot(all_key(false)), - sqlquack: sqlquack(true), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a key other than ALL is not matched", - sql: "SELECT a FROM t ORDER BY ALL", - polyglot: polyglot(key(r#"{"column":{"name":"a"}}"#, false)), - sqlquack: sqlquack(false), - expected_match: false, - }, - QuotingTreesTestCase { - description: "ALL beside another key is not matched", - sql: "SELECT a FROM t ORDER BY ALL", - polyglot: polyglot(format!( - "{},{}", - all_key(false), - key(r#"{"column":{"name":"a"}}"#, false) - )), - sqlquack: sqlquack(false), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a statement without the marker is not matched", - sql: "SELECT a FROM t ORDER BY a", - polyglot: polyglot(all_key(false)), - sqlquack: sqlquack(false), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a polyglot error is not matched", - sql: "SELECT a FROM t ORDER BY ALL", - polyglot: "err:Parse".to_owned(), - sqlquack: sqlquack(false), - expected_match: false, - }, - ]; - let expected = ExpectedDifferences { - kept_alls: vec![KeptAll { - id: "order_by_all".to_owned(), - marker: "ORDERBYALL".to_owned(), - clause: "order_by".to_owned(), - note: "a test entry".to_owned(), - }], - ..ExpectedDifferences::default() - }; - - for test_case in test_cases { - assert_eq!( - expected - .kept_all_for(test_case.sql, &test_case.polyglot, &test_case.sqlquack) - .is_some(), - test_case.expected_match, - "{}", - test_case.description - ); - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_kept_quoting.rs b/tools/sqlquack-polyglot-check/tests/runs/test_kept_quoting.rs deleted file mode 100644 index 0a8e9f7fe..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_kept_quoting.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Reviewed kept quoting covers only names polyglot leaves unquoted, in statements with the marker. - -use sqlquack_polyglot_check::corpora::models::CorpusEntry; -use sqlquack_polyglot_check::expected::models::{ExpectedDifferences, KeptQuoting}; -use sqlquack_syntax::Dialect; - -use crate::helpers::summary_of; -use crate::test_types::{KeptQuotingTestCase, QuotingTreesTestCase}; - -#[test] -fn given_quoted_exclude_names_when_compared_then_only_the_marked_statement_is_kept_quoting() { - let test_cases = [ - KeptQuotingTestCase { - description: "a quoted name in a star's EXCLUDE list is kept quoting", - sql: "SELECT * EXCLUDE (\"note\") FROM orders", - expected_kept_quoting: 1, - expected_differences: 0, - }, - KeptQuotingTestCase { - description: "a quoted EXCLUDE name inside a call is kept quoting", - sql: "SELECT OBJECT_CONSTRUCT(* EXCLUDE (\"note\")) AS o FROM orders", - expected_kept_quoting: 1, - expected_differences: 0, - }, - KeptQuotingTestCase { - description: "an unquoted EXCLUDE name gives identical trees", - sql: "SELECT * EXCLUDE (note) FROM orders", - expected_kept_quoting: 0, - expected_differences: 0, - }, - KeptQuotingTestCase { - description: "a quoted EXCLUDE name without parentheses lacks the marker and still differs", - sql: "SELECT * EXCLUDE \"note\" FROM orders", - expected_kept_quoting: 0, - expected_differences: 1, - }, - ]; - - for test_case in test_cases { - let summary = summary_of(&[CorpusEntry { - dialect: Dialect::Snowflake, - sql: test_case.sql.to_owned(), - from: test_case.description.to_owned(), - }]); - assert_eq!( - ( - summary.kept_quoting.values().sum::(), - summary.differences.len() - ), - ( - test_case.expected_kept_quoting, - test_case.expected_differences - ), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_trees_with_a_marker_when_matched_then_only_dropped_quoting_is_allowed() { - let identifier = |name: &str, quoted: bool| { - format!(r#"{{"name":"{name}","quoted":{quoted},"trailing_comments":[]}}"#) - }; - let tree = |first: String, second: String| format!(r#"ok:[{{"except":[{first},{second}]}}]"#); - let quoted_only = tree(identifier("note", true), identifier("b", false)); - let test_cases = [ - QuotingTreesTestCase { - description: "polyglot unquoted and SQLQuack quoted is the only difference", - sql: "SELECT * EXCLUDE (\"note\", b) FROM orders", - polyglot: tree(identifier("note", false), identifier("b", false)), - sqlquack: quoted_only.clone(), - expected_match: true, - }, - QuotingTreesTestCase { - description: "a different name beside the dropped quoting is not matched", - sql: "SELECT * EXCLUDE (\"note\", b) FROM orders", - polyglot: tree(identifier("note", false), identifier("c", false)), - sqlquack: quoted_only.clone(), - expected_match: false, - }, - QuotingTreesTestCase { - description: "quoting SQLQuack drops and polyglot keeps is not matched", - sql: "SELECT * EXCLUDE (\"note\", b) FROM orders", - polyglot: tree(identifier("note", true), identifier("b", true)), - sqlquack: quoted_only.clone(), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a quoted flag on a node that is not a name is not matched", - sql: "SELECT * EXCLUDE (\"note\") FROM orders", - polyglot: r#"ok:[{"star":{"quoted":false}}]"#.to_owned(), - sqlquack: r#"ok:[{"star":{"quoted":true}}]"#.to_owned(), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a field only one tree has is not matched", - sql: "SELECT * EXCLUDE (\"note\") FROM orders", - polyglot: r#"ok:[{"name":"note","quoted":false}]"#.to_owned(), - sqlquack: r#"ok:[{"name":"note","quoted":true,"span":null}]"#.to_owned(), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a statement without the marker is not matched", - sql: "SELECT * EXCLUDE \"note\" FROM orders", - polyglot: tree(identifier("note", false), identifier("b", false)), - sqlquack: quoted_only.clone(), - expected_match: false, - }, - QuotingTreesTestCase { - description: "a polyglot error is not matched", - sql: "SELECT * EXCLUDE (\"note\") FROM orders", - polyglot: "err:Parse".to_owned(), - sqlquack: quoted_only, - expected_match: false, - }, - ]; - let expected = ExpectedDifferences { - kept_quotings: vec![KeptQuoting { - id: "star_exclude_quoted_name".to_owned(), - marker: "EXCLUDE(\"".to_owned(), - note: "a test entry".to_owned(), - }], - ..ExpectedDifferences::default() - }; - - for test_case in test_cases { - assert_eq!( - expected - .kept_quoting_for(test_case.sql, &test_case.polyglot, &test_case.sqlquack) - .is_some(), - test_case.expected_match, - "{}", - test_case.description - ); - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_rejections.rs b/tools/sqlquack-polyglot-check/tests/runs/test_rejections.rs deleted file mode 100644 index 4d122b22f..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_rejections.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Reviewed rejections cover only their exact statement and dialects. - -use sqlquack_polyglot_check::corpora::models::CorpusEntry; -use sqlquack_syntax::Dialect; - -use crate::helpers::summary_of; -use crate::test_types::RejectionTestCase; - -#[test] -fn given_rejected_statements_when_compared_then_only_the_reviewed_one_is_explained() { - let test_cases = [ - RejectionTestCase { - description: "the reviewed statement in DuckDB is rejected as reviewed", - dialect: Dialect::DuckDB, - sql: "SELECT __polyglot_typed(amount, 'NOT A TYPE (') FROM orders", - expected_rejected: 1, - expected_differences: 0, - }, - RejectionTestCase { - description: "the reviewed statement in Snowflake is rejected as reviewed", - dialect: Dialect::Snowflake, - sql: "SELECT __polyglot_typed(amount, 'NOT A TYPE (') FROM orders", - expected_rejected: 1, - expected_differences: 0, - }, - RejectionTestCase { - description: "a reviewed statement polyglot rejects with another error is rejected as reviewed", - dialect: Dialect::Snowflake, - sql: "SELECT FROM x ORDER BY", - expected_rejected: 1, - expected_differences: 0, - }, - RejectionTestCase { - description: "other malformed type text is still a difference", - dialect: Dialect::DuckDB, - sql: "SELECT __polyglot_typed(amount, 'INT garbage') FROM orders", - expected_rejected: 0, - expected_differences: 1, - }, - ]; - - for test_case in test_cases { - let summary = summary_of(&[CorpusEntry { - dialect: test_case.dialect, - sql: test_case.sql.to_owned(), - from: test_case.description.to_owned(), - }]); - assert_eq!( - ( - summary.rejected.values().sum::(), - summary.differences.len() - ), - (test_case.expected_rejected, test_case.expected_differences), - "{}", - test_case.description - ); - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_structured.rs b/tools/sqlquack-polyglot-check/tests/runs/test_structured.rs deleted file mode 100644 index 535423313..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_structured.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Reviewed structured entries cover only a polyglot command SQLQuack parses into the listed node. - -use sqlquack_polyglot_check::expected::models::{ExpectedDifferences, StructuredStatement}; - -use crate::test_types::StructuredTestCase; - -#[test] -fn given_a_command_polyglot_keeps_when_compared_then_only_the_listed_structure_is_matched() { - let block = "BEGIN RETURN 1; END"; - let structured = r#"ok:[{"scripting_block":{"body":[]}}]"#; - let test_cases = [ - StructuredTestCase { - description: "a command SQLQuack parses into the listed node is matched", - sql: block, - polyglot: r#"ok:[{"command":{"this":"BEGIN RETURN 1; END"}}]"#, - sqlquack: structured, - expected_match: true, - }, - StructuredTestCase { - description: "raw SQL polyglot keeps, which SQLQuack parses into the listed node, is matched", - sql: block, - polyglot: r#"ok:[{"raw":{"sql":"BEGIN RETURN 1; END"}}]"#, - sqlquack: structured, - expected_match: true, - }, - StructuredTestCase { - description: "a structured polyglot tree is not matched", - sql: block, - polyglot: r#"ok:[{"select":{}}]"#, - sqlquack: structured, - expected_match: false, - }, - StructuredTestCase { - description: "a SQLQuack tree of another node is not matched", - sql: block, - polyglot: r#"ok:[{"command":{"this":"BEGIN RETURN 1; END"}}]"#, - sqlquack: r#"ok:[{"command":{"this":"BEGIN RETURN 1; END"}}]"#, - expected_match: false, - }, - StructuredTestCase { - description: "a SQLQuack error is not matched", - sql: block, - polyglot: r#"ok:[{"command":{"this":"BEGIN RETURN 1; END"}}]"#, - sqlquack: "err:Parse", - expected_match: false, - }, - StructuredTestCase { - description: "a statement without the marker is not matched", - sql: "DECLARE x INT;", - polyglot: r#"ok:[{"command":{"this":"DECLARE x INT;"}}]"#, - sqlquack: structured, - expected_match: false, - }, - ]; - let expected = ExpectedDifferences { - structured: vec![StructuredStatement { - id: "snowflake_scripting_block".to_owned(), - marker: "BEGIN".to_owned(), - sqlquack_node: "scripting_block".to_owned(), - note: "a test entry".to_owned(), - }], - ..ExpectedDifferences::default() - }; - - for test_case in test_cases { - assert_eq!( - expected - .structured_for(test_case.sql, test_case.polyglot, test_case.sqlquack) - .is_some(), - test_case.expected_match, - "{}", - test_case.description - ); - } -} diff --git a/tools/sqlquack-polyglot-check/tests/runs/test_types.rs b/tools/sqlquack-polyglot-check/tests/runs/test_types.rs deleted file mode 100644 index fcf827a3e..000000000 --- a/tools/sqlquack-polyglot-check/tests/runs/test_types.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::collections::BTreeMap; - -use sqlquack_polyglot_check::corpora::models::CorpusEntry; -use sqlquack_syntax::Dialect; - -pub(crate) struct CorpusTestCase { - pub(crate) description: &'static str, - pub(crate) entries: Result, String>, - pub(crate) expected_outcome: ExactOutcome, -} - -#[derive(Debug, PartialEq, Eq)] -pub(crate) struct ExactOutcome { - pub(crate) loaded: bool, - pub(crate) rejected: BTreeMap, - pub(crate) differences: Vec, - pub(crate) tokens_lost: Vec, - pub(crate) type_names: Vec, - pub(crate) unlisted: Vec, - pub(crate) unexplained: Vec, - pub(crate) located: Vec, -} - -pub(crate) struct RejectionTestCase { - pub(crate) description: &'static str, - pub(crate) dialect: Dialect, - pub(crate) sql: &'static str, - pub(crate) expected_rejected: usize, - pub(crate) expected_differences: usize, -} - -pub(crate) struct KeptNodeTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_kept_nodes: usize, - pub(crate) expected_differences: usize, -} - -pub(crate) struct KeptAllTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_kept_fields: usize, - pub(crate) expected_differences: usize, -} - -pub(crate) struct KeptQuotingTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_kept_quoting: usize, - pub(crate) expected_differences: usize, -} - -pub(crate) struct QuotingTreesTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) polyglot: String, - pub(crate) sqlquack: String, - pub(crate) expected_match: bool, -} - -pub(crate) struct StructuredTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) polyglot: &'static str, - pub(crate) sqlquack: &'static str, - pub(crate) expected_match: bool, -}