diff --git a/Cargo.lock b/Cargo.lock index 149ecdafe..96d5f578d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1499,6 +1499,7 @@ dependencies = [ "serde_json", "sha2", "sqlquack-syntax", + "sqlquack-syntax-postgres", "tempfile", "toml", ] @@ -1512,6 +1513,14 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "sqlquack-syntax-postgres" +version = "0.2.0" +dependencies = [ + "serde_json", + "sqlquack-syntax", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index b9691a176..fead36671 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/sqlquack", "crates/sqlquack-polyglot-check", "crates/sqlquack-syntax-coverage", + "crates/sqlquack-syntax-postgres", "crates/sqlquack-python", ] exclude = ["fuzz"] @@ -62,6 +63,7 @@ sqlquack-semantics = { path = "crates/sqlquack-semantics" } sqlquack-snowflake = { path = "crates/sqlquack-snowflake" } sqlquack-syntax = { path = "crates/sqlquack-syntax" } sqlquack-syntax-derive = { path = "crates/sqlquack-syntax-derive" } +sqlquack-syntax-postgres = { path = "crates/sqlquack-syntax-postgres" } stacker = "0.1" syn = { version = "2", features = ["full"] } tempfile = "3" diff --git a/cases/relational/pivot_unaliased.toml b/cases/relational/pivot_unaliased.toml new file mode 100644 index 000000000..8597d8939 --- /dev/null +++ b/cases/relational/pivot_unaliased.toml @@ -0,0 +1,60 @@ +# PIVOT without an alias column list: the names Snowflake gives the key columns, as dbt pivot +# models leave them. +category = "relational" +construct = "node:pivot" +session_profile = "utc-v1" + +[[case]] +id = "pivot_star_without_alias_list" +description = "SELECT * over SUM(total) FOR channel IN ('web', 'store') with no alias list. Catches: key columns named web or WEB instead of the quoted text, a missing or NULL key, keys outside the list dropped, and the grouping column kept." +columns = [{ name = "order_id", type = "NUMBER(38,0)" }, { name = "channel", type = "VARCHAR" }, { name = "total", type = "NUMBER(12,2)" }] +rows = [ + ["1", "'web'", "10.50"], + ["1", "'web'", "2.25"], + ["1", "'store'", "4.00"], + ["2", "'store'", "NULL"], + ["2", "'phone'", "9.99"], + ["3", "NULL", "1.00"], + ["3", "'WEB'", "3.00"], +] +query = "SELECT * FROM (SELECT order_id, channel, total FROM input) PIVOT(SUM(total) FOR channel IN ('web', 'store')) ORDER BY order_id" +order = "ordered" +tags = ["null", "duplicates", "missing-key", "case-sensitive-key"] + +[[case]] +id = "pivot_quoted_key_reference" +description = "The pivoted key columns read back by their quoted names, \"'web'\" and \"'store'\", through a table alias without a column list. Catches: the names not matching the quoted key text." +columns = [{ name = "order_id", type = "NUMBER(38,0)" }, { name = "channel", type = "VARCHAR" }, { name = "total", type = "NUMBER(12,2)" }] +rows = [ + ["1", "'web'", "10.50"], + ["1", "'store'", "4.00"], + ["2", "'web'", "1.00"], +] +query = "SELECT p.order_id, p.\"'web'\" AS web_total, p.\"'store'\" AS store_total FROM (SELECT order_id, channel, total FROM input) PIVOT(SUM(total) FOR channel IN ('web', 'store')) AS p ORDER BY p.order_id" +order = "ordered" +tags = ["null", "missing-key"] + +[[case]] +id = "pivot_mixed_case_keys_without_alias_list" +description = "Keys with upper case, a space and an underscore, under MAX over BOOLEAN and no alias list. Catches: key names upper-cased, spaces changed, or the quotes dropped." +columns = [{ name = "customer_id", type = "VARCHAR" }, { name = "consent", type = "VARCHAR" }, { name = "granted", type = "BOOLEAN" }] +rows = [ + ["'c-1'", "'Email Opt'", "TRUE"], + ["'c-1'", "'SMS_OPT'", "FALSE"], + ["'c-2'", "'Email Opt'", "NULL"], + ["'c-2'", "'other'", "TRUE"], +] +query = "SELECT * FROM (SELECT customer_id, consent, granted FROM input) PIVOT(MAX(granted) FOR consent IN ('Email Opt', 'SMS_OPT')) ORDER BY customer_id" +order = "ordered" +tags = ["null", "missing-key", "case-sensitive-key"] + +[[case]] +id = "pivot_without_alias_list_over_no_rows" +description = "A PIVOT without an alias list over no rows. Catches: a row of NULLs instead of no rows, and the column names and types still described." +columns = [{ name = "order_id", type = "NUMBER(38,0)" }, { name = "channel", type = "VARCHAR" }, { name = "total", type = "NUMBER(12,2)" }] +rows = [ + ["1", "'web'", "10.50"], +] +query = "SELECT * FROM (SELECT order_id, channel, total FROM input WHERE total < 0) PIVOT(SUM(total) FOR channel IN ('web', 'store'))" +order = "bag" +tags = ["empty-input"] diff --git a/cases/semi-structured/table_flatten.toml b/cases/semi-structured/table_flatten.toml new file mode 100644 index 000000000..c332b66d8 --- /dev/null +++ b/cases/semi-structured/table_flatten.toml @@ -0,0 +1,33 @@ +# FLATTEN written as TABLE(FLATTEN(...)) after a comma, with a positional input, as dbt models +# write it: the same rows as LATERAL FLATTEN(input => ...). +category = "semi-structured" +construct = "fn:FLATTEN" +session_profile = "utc-v1" + +[[case]] +id = "table_flatten_positional_array" +description = "TABLE(FLATTEN(t.tags)) over an ARRAY column with a positional input: one row per element. Catches: the input not correlated to its row, empty arrays or NULL giving rows, or elements out of order." +columns = [{ name = "ticket_id", type = "NUMBER(38,0)" }, { name = "tags", type = "ARRAY" }] +rows = [ + ["1", "ARRAY_CONSTRUCT('billing', 'urgent')"], + ["2", "ARRAY_CONSTRUCT()"], + ["3", "NULL"], + ["4", "ARRAY_CONSTRUCT('refund')"], +] +query = "SELECT t.ticket_id, f.index, f.value::STRING AS tag FROM input t, TABLE(FLATTEN(t.tags)) f ORDER BY t.ticket_id, f.index" +order = "ordered" +tags = ["null", "array-index"] + +[[case]] +id = "table_flatten_named_variant_path" +description = "TABLE(FLATTEN(input => v:items)) over a VARIANT path, with KEY, PATH and VALUE read back. Catches: the named input ignored, objects read as arrays, or JSON null and missing paths giving rows." +columns = [{ name = "k", type = "NUMBER(2,0)" }, { name = "v", type = "VARIANT" }] +rows = [ + ["1", "PARSE_JSON('{\"items\":[{\"sku\":\"A-10\"},{\"sku\":\"B-20\"}]}')"], + ["2", "PARSE_JSON('{\"items\":{\"b\":1,\"a\":2}}')"], + ["3", "PARSE_JSON('{\"items\":null}')"], + ["4", "PARSE_JSON('{}')"], +] +query = "SELECT i.k, f.index, f.key, f.path, f.value::VARCHAR AS value FROM input i, TABLE(FLATTEN(input => i.v:items)) f ORDER BY i.k, f.index, f.key" +order = "ordered" +tags = ["null", "json-null", "missing-key", "nested", "array-index", "case-sensitive-key"] diff --git a/coverage/syntax-postgres/README.md b/coverage/syntax-postgres/README.md new file mode 100644 index 000000000..763857eb2 --- /dev/null +++ b/coverage/syntax-postgres/README.md @@ -0,0 +1,172 @@ +# PostgreSQL syntax catalogue + +The denominator of SQLQuack's PostgreSQL syntax coverage: every documented PostgreSQL 18.6 syntax +form, one record per form, with a stable id `pg..[.]`. The layout and the +record formats are those of `coverage/syntax` (see `docs/syntax-coverage.md`); this file records +what differs. + +## Source + +The pinned source is the release tarball `postgresql-18.6.tar.bz2` (sha256 in `indexes.toml`), +not the rendered website. Its DocBook SGML is what postgresql.org/docs/18 is built from, so a +source id `doc.` names the page `https://www.postgresql.org/docs/18/.html`. +`indexes.toml` also binds the files the page list is read from (`reference.sgml`, +`ref/allfiles.sgml`, `filelist.sgml`, `postgres.sgml`) and the grammar the omission audit reads +(`src/backend/parser/gram.y`, `src/include/parser/kwlist.h`). + +A page is either + +- a SQL command reference page (a `refentry` of the SQL Commands part, 183 pages), or +- a `sect1` (or a chapter's introduction) of the SQL-language chapters: syntax, ddl, dml, + queries, datatype, json, array, rowtypes, rangetypes, func, typeconv, indices, textsearch, mvcc, + perform and parallel. + +`page_sha256` is the sha256 of that SGML text; `syntax_sha256` of its synopsis blocks and +signatures. + +## Regenerating + +```sh +python -m scripts.fetch_syntax_docs_pg CACHE # private cache, never committed +python -m scripts.build_syntax_catalogue_pg --cache CACHE --out coverage/syntax-postgres +``` + +The build writes nothing while any input is unresolved: a synopsis block with no disposition, a +disposition naming no block, a restatement of an uncatalogued form, or a hand form whose page +changed. PostgreSQL 19 is an in-place rebuild against its tarball; ids follow the same keeping +rules as the Snowflake catalogue (`scripts/syntax_catalogue/_helpers/forms.py`, reused by import). + +## Omission audit (`audit.toml`) + +```sh +python -m scripts.audit_syntax_catalogue_pg --cache CACHE +``` + +An independent check against the server's own grammar, `gram.y` and `kwlist.h` of the pinned +release. It reads every rule of gram.y and spells each alternative of every rule reachable from +`toplevel_stmt`, `stmt` and the expression rules (720 of 733 rules, with no depth bound) as keyword +sequences, +expanding nonterminals that are keyword sets (`DROP object_type_any_name` checks DROP TABLE, +DROP SEQUENCE and so on). A sequence is covered when some catalogued form, with its page's +definitions inlined, contains its keywords in order; for a statement the form's leading required +keywords must also appear in the sequence, so DROP FOREIGN TABLE does not cover DROP TABLE. +Every kwlist.h keyword that a production uses must appear in some grammar. + +`audit.toml` binds the sha256 of gram.y and kwlist.h and gives a reviewed reason for every +production that no form covers (`[[production]]`, keyed by rule and production text) and every +keyword that no grammar names (`[[keyword]]`). The audit fails on an unreviewed gap, a review +naming no current gap, or a changed grammar file. The reasons are of three kinds: syntax gram.y +accepts that the 18.6 documentation does not document (the spelling ANALYSE, GLOBAL and LOCAL +before TEMPORARY outside CREATE TABLE, `BETWEEN ASYMMETRIC`, `TREAT`, `NCHAR`); productions that +only raise an error (CREATE ASSERTION, `ALTER TYPE ... DROP VALUE`, UNENCRYPTED PASSWORD); and +documented values of a placeholder (`NFC`, `OUT`, `SETOF`). Single-keyword alternatives below a +statement are checked by the keyword rule only. + +`[[unwalked]]` lists each of the 13 rules the walk does not reach with its reason: the start +symbol and top-level wrappers, the PL/pgSQL-only parse modes and the five keyword category lists. +The audit fails on an unwalked rule that is not listed. `limits` states what the method does not +check, such as identifiers, punctuation and adjacency of keywords. + +## Form rules + +1. **Command pages.** The `` synopsis is split into statements at each unindented + line that starts with the command's first word (SELECT also `WITH` and `TABLE`). Each + statement is a form. `where X is:` (and `and X is`, `X in ... is`) starts a + sub-grammar definition, `[[definition]]` `doc.::`; `can be one of` / `is one of` + lists its alternatives one per line. +2. **Actions.** A statement whose grammar takes ` [, ... ]`, where the page lists + `action` as alternatives (ALTER TABLE and six other ALTER pages), becomes one form per action, with the + action substituted into the statement. +3. **Other synopsis blocks** (on command pages outside the Synopsis, and in the chapters) each + have a reviewed disposition in `hand.toml` `[[block]]`, keyed by page and block sha256: + `form` (each production a form; on a command page, older syntax still accepted, named + `.