Snowflake DESCRIBE/SHOW, audit withholds, coverage-diff regression line, DuckShop functions

Makefile

@@ -208,15 +208,16 @@ duckshop-measure: duckshop-compiled
 	library="$$(dirname "$$(ls "$$target"/duckdb-download/*/*/libduckdb.so | head -1)")"; \
 	compiled="$(abspath $(word 1,$(DUCKSHOP_COMPILED)))"; out="$(DUCKSHOP_MEASURE_DIR)"; \
 	uv run python -m scripts.measure_duckshop schema --environment referee/environments/duckshop.json \
-	  --recorded "$(DUCKSHOP_RESULTS)" --out "$$out/schema.json"; \
+	  --recorded "$(DUCKSHOP_RESULTS)" --functions "$$compiled/functions" --out "$$out/schema.json"; \
 	SQLQUACK_CONVERTED_DIR="$$out/converted" SQLQUACK_GUARDS_DIR="$$out/guards" \
 	  LD_LIBRARY_PATH="$$library" "$$target/release/sqlquack" census --policy sqlquack.toml --json \
 	  --names --schema "$$out/schema.json" "$$compiled/models" > "$$out/census.json"; \
 	uv run python -m scripts.measure_duckshop tests --compiled "$$compiled" \
 	  --sqlquack "$$target/release/sqlquack" --library "$$library" --out "$$out/tests.json"; \
 	uv run python -m scripts.measure_duckshop snowflake --results "$(DUCKSHOP_RESULTS)" \
 	  --compiled "$$compiled" --census "$$out/census.json" --converted "$$out/converted" \
-	  --guards "$$out/guards" --sqlquack "$$target/release/sqlquack" --library "$$library" \
+	  --guards "$$out/guards" --functions "$$compiled/functions" --schema "$$out/schema.json" \
+	  --sqlquack "$$target/release/sqlquack" --library "$$library" \
 	  --work "$$out/work" --out "$$out/snowflake.json"
 
 # Fuzzes each of FUZZ_TARGETS from the exact check's statements for FUZZ_SECONDS: tolerant parsing,

corpora/polyglot-check/expected_differences.toml

@@ -912,6 +912,69 @@ dialects = ["Snowflake"]
 sqlquack_error = "Snowflake set operations take no LEFT, RIGHT, FULL, INNER or OUTER"
 note = "INNER UNION ... BY NAME is BigQuery's; Snowflake rejects a join side before a set operation (recorded errno 1003)"
 
+[[rejection]]
+id = "snowflake_describe_bare_name"
+sql = "DESCRIBE x"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "Snowflake DESCRIBE names its object kind before the name: it refuses DESC v1 with syntax error 1003"
+
+[[rejection]]
+id = "snowflake_show_profile"
+sql = "SHOW PROFILE FOR QUERY 5"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after SHOW"
+note = "SHOW PROFILE is MySQL's; PROFILE is no Snowflake SHOW kind"
+
+[[rejection]]
+id = "snowflake_show_groups"
+sql = "SHOW GROUPS FOR ROLE"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after SHOW"
+note = "GROUPS is no Snowflake SHOW kind; Snowflake shows user groups as SHOW ORGANIZATION USER GROUPS"
+
+[[rejection]]
+id = "snowflake_show_create_table"
+sql = "SHOW CREATE TABLE t"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after SHOW"
+note = "SHOW CREATE TABLE is MySQL's; Snowflake returns DDL through GET_DDL, and CREATE is no SHOW kind"
+
+[[rejection]]
+id = "snowflake_describe_extended"
+sql = "DESCRIBE EXTENDED a.b"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "EXTENDED is Hive's and Spark's; Snowflake DESCRIBE names an object kind, and EXTENDED is none"
+
+[[rejection]]
+id = "snowflake_describe_formatted"
+sql = "DESCRIBE FORMATTED a.b"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "FORMATTED is Hive's; Snowflake DESCRIBE names an object kind, and FORMATTED is none"
+
+[[rejection]]
+id = "snowflake_describe_query"
+sql = "DESCRIBE SELECT 1"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "Snowflake describes a query's result with DESCRIBE RESULT and a query id, not the query itself"
+
+[[rejection]]
+id = "snowflake_describe_bare_quoted_name"
+sql = "DESCRIBE \"history\""
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "a quoted name is still a bare name: Snowflake refuses DESC v1 with syntax error 1003"
+
+[[rejection]]
+id = "snowflake_describe_bare_qualified_name"
+sql = "DESCRIBE db.table"
+dialects = ["Snowflake"]
+sqlquack_error = "Expected an object kind after DESCRIBE"
+note = "a qualified name is still a bare name: Snowflake refuses DESC v1 with syntax error 1003"
+
 [[rejection]]
 id = "snowflake_by_name_on_intersect"
 sql = "SELECT 1 AS a, 2 AS b INTERSECT DISTINCT BY NAME ON (b, a) SELECT 3 AS b, 4 AS a"
@@ -996,6 +1059,28 @@ marker = "+"
 node = "pos"
 note = "SELECT + 1 keeps its unary plus, a documented Snowflake operator; polyglot drops it and keeps only the operand; crates/sqlquack-syntax/tests/parser/test_fixes.rs pins the tree"
 
+# Fields SQLQuack keeps as written where polyglot drops or infers them. The check counts an entry
+# only when the statement, without whitespace and ignoring case, contains `marker`, the trees differ,
+# and both are equal once every `<field>` key is removed from each.
+
+[[kept_field]]
+id = "snowflake_secondary_role_list"
+marker = "SECONDARYROLES"
+field = "roles"
+note = "USE SECONDARY ROLES a, b, c lists its roles one by one with their quoting; polyglot joins them into one unquoted name; crates/sqlquack-syntax/tests/parser/test_fixes.rs pins the tree"
+
+[[kept_field]]
+id = "snowflake_show_unnamed_container"
+marker = "SHOW"
+field = "scope_kind"
+note = "SHOW ... IN a.b names no container kind, so SQLQuack records none; polyglot infers SCHEMA from the two-part name, which makes IN a.b and IN SCHEMA a.b one tree"
+
+[[kept_field]]
+id = "snowflake_timestamp_without_time_zone"
+marker = "WITHOUTTIMEZONE"
+field = "to"
+note = "Snowflake documents TIMESTAMP WITHOUT TIME ZONE as a synonym of TIMESTAMP_NTZ, so SQLQuack's cast target is TIMESTAMPNTZ, as for TIMESTAMP_NTZ; polyglot reads it as bare TIMESTAMP, whose meaning follows TIMESTAMP_TYPE_MAPPING; crates/sqlquack-syntax/tests/parser/test_fixes.rs pins the tree"
+
 # Statements polyglot keeps as one opaque command where SQLQuack parses them into structured nodes.
 # The check counts an entry only when polyglot's tree is a single command, SQLQuack's tree starts
 # with `sqlquack_node`, and the statement, without whitespace and ignoring case, contains `marker`.

crates/sqlquack-core/src/classification/_helpers/clauses.rs

@@ -56,6 +56,7 @@ fn select_clauses(select: &Select) -> Vec<String> {
         limit_by,
         fetch,
         distinct,
+        all: _,
         distinct_on,
         top,
         with,
@@ -188,6 +189,8 @@ fn join_constructs(join: &Join) -> Vec<Construct> {
         comments: _,
         nesting_group: _,
         directed,
+        first_last,
+        first_last_alias: _,
         span: _,
     } = join;
     let mut constructs = vec![Construct::new(
@@ -200,6 +203,7 @@ fn join_constructs(join: &Join) -> Vec<Construct> {
         (match_condition.is_some(), "match_condition"),
         (!pivots.is_empty(), "join_pivot"),
         (*directed, "join_directed"),
+        (first_last.is_some(), "join_first_last"),
     ]);
     constructs.extend(
         names

crates/sqlquack-polyglot-check/src/cli/_helpers/report.rs

@@ -25,7 +25,7 @@ pub(crate) fn report(label: &str, summary: &RunSummary, counts_only: bool) -> St
     }
     let _ = writeln!(
         text,
-        "{label}: {} entries, {} same trees, {} same errors, {} fixed, {} rejected, {} kept quoting, {} kept nodes, {} structured, {} differences, {} without lossless \
+        "{label}: {} entries, {} same trees, {} same errors, {} fixed, {} rejected, {} kept quoting, {} kept nodes, {} kept fields, {} structured, {} differences, {} without lossless \
          tokens, {} cast type names unlike polyglot's, {} faithful calls kept, {} unlisted, {} \
          unexplained, {} located parses unlike faithful; parse time polyglot {:.1} ms, sqlquack \
          {:.1} ms",
@@ -36,6 +36,7 @@ pub(crate) fn report(label: &str, summary: &RunSummary, counts_only: bool) -> St
         summary.rejected.values().sum::<usize>(),
         summary.kept_quoting.values().sum::<usize>(),
         summary.kept_nodes.values().sum::<usize>(),
+        summary.kept_fields.values().sum::<usize>(),
         summary.structured.values().sum::<usize>(),
         summary.differences.len(),
         summary.tokens_lost.len(),

crates/sqlquack-polyglot-check/src/expected/_helpers/nodes.rs

@@ -1,4 +1,4 @@
-//! Trees that differ only by a wrapper node SQLQuack keeps and polyglot drops.
+//! Trees that differ only by a wrapper node or a field SQLQuack keeps and polyglot drops or infers.
 
 use serde_json::Value;
 
@@ -52,3 +52,42 @@ fn without_node(value: Value, node: &str) -> Value {
         other => other,
     }
 }
+
+/// Whether both rendered parses are trees that differ, and are equal once every `field` key goes from each.
+pub(crate) fn differ_only_by_kept_field(polyglot: &str, sqlquack: &str, field: &str) -> bool {
+    let (Some(theirs), Some(ours)) = (
+        polyglot.strip_prefix(OK_PREFIX),
+        sqlquack.strip_prefix(OK_PREFIX),
+    ) else {
+        return false;
+    };
+    match (
+        serde_json::from_str::<Value>(theirs),
+        serde_json::from_str::<Value>(ours),
+    ) {
+        (Ok(theirs), Ok(ours)) => {
+            theirs != ours && without_field(theirs, field) == without_field(ours, field)
+        }
+        (Err(_), _) | (_, Err(_)) => false,
+    }
+}
+
+/// `value` with every object key `field` removed.
+fn without_field(value: Value, field: &str) -> Value {
+    match value {
+        Value::Object(object) => Value::Object(
+            object
+                .into_iter()
+                .filter(|(key, _)| key != field)
+                .map(|(key, value)| (key, without_field(value, field)))
+                .collect(),
+        ),
+        Value::Array(items) => Value::Array(
+            items
+                .into_iter()
+                .map(|value| without_field(value, field))
+                .collect(),
+        ),
+        other => other,
+    }
+}

crates/sqlquack-polyglot-check/src/expected/main/load_expected.rs

@@ -15,6 +15,7 @@ pub fn load_expected(path: &Path) -> Result<ExpectedDifferences, String> {
         rejections: file.rejections,
         kept_quotings: file.kept_quotings,
         kept_nodes: file.kept_nodes,
+        kept_fields: file.kept_fields,
         structured: file.structured,
         ..ExpectedDifferences::default()
     };

crates/sqlquack-polyglot-check/src/expected/models.rs

@@ -1,6 +1,6 @@
 //! Expected differences.
 
-use crate::expected::_helpers::nodes::differ_only_by_kept_node;
+use crate::expected::_helpers::nodes::{differ_only_by_kept_field, differ_only_by_kept_node};
 use crate::expected::_helpers::quoting::{compact, differ_only_by_kept_quoting};
 use crate::expected::constants::{COMMAND_TREE, ERROR_PREFIX, OK_PREFIX};
 use std::collections::{BTreeMap, BTreeSet};
@@ -26,6 +26,9 @@ pub struct ExpectedFile {
     /// One entry per reviewed wrapper node SQLQuack keeps and polyglot drops.
     #[serde(default, rename = "kept_node")]
     pub kept_nodes: Vec<KeptNode>,
+    /// One entry per reviewed field SQLQuack keeps as written where polyglot drops or infers it.
+    #[serde(default, rename = "kept_field")]
+    pub kept_fields: Vec<KeptField>,
     /// One entry per reviewed statement polyglot keeps as a command and SQLQuack structures.
     #[serde(default, rename = "structured")]
     pub structured: Vec<StructuredStatement>,
@@ -71,6 +74,20 @@ pub struct KeptNode {
     pub note: String,
 }
 
+/// A statement both parsers parse, where SQLQuack keeps a field as written, such as a list polyglot joins into one name.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KeptField {
+    /// A lower-case identifier.
+    pub id: String,
+    /// Text the statement contains, with whitespace removed and ignoring case, such as `SECONDARYROLES`.
+    pub marker: String,
+    /// The field, such as `roles`; without it in either tree the trees may differ in nothing.
+    pub field: String,
+    /// Why SQLQuack's field is right; a test in sqlquack-syntax pins its tree.
+    pub note: String,
+}
+
 /// A statement SQLQuack rejects where polyglot parses it or fails differently, recognised by its exact text and dialect.
 #[derive(Clone, Debug, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -126,6 +143,8 @@ pub struct ExpectedDifferences {
     pub kept_quotings: Vec<KeptQuoting>,
     /// The reviewed wrapper nodes SQLQuack keeps and polyglot drops.
     pub kept_nodes: Vec<KeptNode>,
+    /// The reviewed fields SQLQuack keeps as written where polyglot drops or infers them.
+    pub kept_fields: Vec<KeptField>,
     /// The reviewed statements polyglot keeps as a command and SQLQuack structures.
     pub structured: Vec<StructuredStatement>,
 }
@@ -188,6 +207,15 @@ impl ExpectedDifferences {
         })
     }
 
+    /// The entry explaining where SQLQuack keeps a field as written that polyglot drops or infers.
+    pub fn kept_field_for(&self, sql: &str, polyglot: &str, sqlquack: &str) -> Option<&KeptField> {
+        let compact = compact(sql);
+        self.kept_fields.iter().find(|entry| {
+            compact.contains(&entry.marker.to_ascii_uppercase())
+                && differ_only_by_kept_field(polyglot, sqlquack, &entry.field)
+        })
+    }
+
     /// The entry explaining where SQLQuack structures a statement polyglot keeps as a command.
     pub fn structured_for(
         &self,

crates/sqlquack-polyglot-check/src/runs/main/run_corpus.rs

@@ -60,8 +60,16 @@ pub fn run_corpus(entries: &[CorpusEntry], expected: &ExpectedDifferences) -> Ru
                 let rejection = expected.rejection_for(&entry.sql, &dialect, &sqlquack);
                 let kept_quoting = expected.kept_quoting_for(&entry.sql, &polyglot, &sqlquack);
                 let kept_node = expected.kept_node_for(&entry.sql, &polyglot, &sqlquack);
+                let kept_field = expected.kept_field_for(&entry.sql, &polyglot, &sqlquack);
                 let structured = expected.structured_for(&entry.sql, &polyglot, &sqlquack);
-                match (fix, rejection, kept_quoting, kept_node, structured) {
+                match (
+                    fix,
+                    rejection,
+                    kept_quoting,
+                    kept_node,
+                    kept_field,
+                    structured,
+                ) {
                     (Some(fix), ..) => *summary.fixed.entry(fix.id.clone()).or_default() += 1,
                     (None, Some(rejection), ..) => {
                         *summary.rejected.entry(rejection.id.clone()).or_default() += 1;
@@ -72,13 +80,19 @@ pub fn run_corpus(entries: &[CorpusEntry], expected: &ExpectedDifferences) -> Ru
                             .entry(kept_quoting.id.clone())
                             .or_default() += 1;
                     }
-                    (None, None, None, Some(kept_node), _) => {
+                    (None, None, None, Some(kept_node), ..) => {
                         *summary.kept_nodes.entry(kept_node.id.clone()).or_default() += 1;
                     }
-                    (None, None, None, None, Some(structured)) => {
+                    (None, None, None, None, Some(kept_field), _) => {
+                        *summary
+                            .kept_fields
+                            .entry(kept_field.id.clone())
+                            .or_default() += 1;
+                    }
+                    (None, None, None, None, None, Some(structured)) => {
                         *summary.structured.entry(structured.id.clone()).or_default() += 1;
                     }
-                    (None, None, None, None, None) => summary.differences.push(Finding {
+                    (None, None, None, None, None, None) => summary.differences.push(Finding {
                         from,
                         detail: format!("polyglot: {polyglot}\nsqlquack: {sqlquack}"),
                     }),

crates/sqlquack-polyglot-check/src/runs/models.rs

@@ -31,6 +31,8 @@ pub struct RunSummary {
     pub kept_quoting: BTreeMap<String, usize>,
     /// Entries whose trees differ only by wrapper nodes polyglot drops, as a reviewed entry explains.
     pub kept_nodes: BTreeMap<String, usize>,
+    /// Entries whose trees differ only by a field kept as written, as a reviewed entry explains.
+    pub kept_fields: BTreeMap<String, usize>,
     /// Entries polyglot keeps as a command and SQLQuack structures, as a reviewed entry explains.
     pub structured: BTreeMap<String, usize>,
     /// Entries whose results differ.

crates/sqlquack-polyglot-check/tests/runs/test_exact.rs

@@ -44,6 +44,10 @@ fn given_committed_corpora_when_parsed_by_both_parsers_then_results_are_identica
                     ("snowflake_reserved_distinct_alias".to_owned(), 1),
                     ("snowflake_reserved_is_alias".to_owned(), 1),
                     ("snowflake_set_operation_join_side".to_owned(), 1),
+                    ("snowflake_describe_bare_name".to_owned(), 1),
+                    ("snowflake_show_profile".to_owned(), 1),
+                    ("snowflake_show_groups".to_owned(), 1),
+                    ("snowflake_show_create_table".to_owned(), 1),
                 ]),
                 differences: Vec::new(),
                 tokens_lost: Vec::new(),
@@ -72,6 +76,12 @@ fn given_committed_corpora_when_parsed_by_both_parsers_then_results_are_identica
                     ("snowflake_frame_bare_preceding".to_owned(), 1),
                     ("snowflake_frame_value_without_direction".to_owned(), 1),
                     ("snowflake_frame_value_without_direction_end".to_owned(), 1),
+                    ("snowflake_describe_bare_name".to_owned(), 1),
+                    ("snowflake_describe_bare_qualified_name".to_owned(), 1),
+                    ("snowflake_describe_extended".to_owned(), 1),
+                    ("snowflake_describe_formatted".to_owned(), 1),
+                    ("snowflake_describe_query".to_owned(), 1),
+                    ("snowflake_describe_bare_quoted_name".to_owned(), 1),
                 ]),
                 differences: Vec::new(),
                 tokens_lost: Vec::new(),