fix: stop false errors on stars, comments, set operations, snapshots and table-view switches

Cargo.toml

@@ -64,14 +64,14 @@ junction = { version = "2.1.0", default-features = false }
 mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "ca9e7b6e405fb6f5484472bb08780268789c5e04", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [
     "ast-tools",
     "builder",
     "all-dialects",
     "semantic",
     "stacker",
 ] }
-polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "ca9e7b6e405fb6f5484472bb08780268789c5e04", default-features = false, features = [
+polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [
     "dialect-duckdb",
 ] }
 blake3 = { version = "1", features = ["rayon"] }

Makefile

@@ -86,6 +86,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \
@@ -126,7 +127,8 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_validity_columns.py
 
 
 E2E_DUCKDB_CLI_DATA_PATHS := \

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs

@@ -170,12 +170,22 @@ fn projected(
         return Ok(Projected::Declined);
     };
     let shape: Option<&str> = analysis.get("shape").and_then(Value::as_str);
-    if !matches!(shape, Some(SELECT_SHAPE | SET_OPERATION_SHAPE)) || !eligible(projections) {
+    if !matches!(shape, Some(SELECT_SHAPE | SET_OPERATION_SHAPE))
+        || !(has_star_projection(analysis) || eligible(projections))
+    {
         return Ok(Projected::Declined);
     }
     native_projection(input, analysis, cleaned, projections).map(Projected::Native)
 }
 
+/// Whether the analysis expanded a star; the legacy analysis cannot, so it never answers one.
+fn has_star_projection(analysis: &Value) -> bool {
+    analysis
+        .get("starProjections")
+        .and_then(Value::as_array)
+        .is_some_and(|stars| !stars.is_empty())
+}
+
 /// The compact facts as Python's columns and lineage.
 fn native_projection(
     input: &EnrichmentInput<'_>,
@@ -248,10 +258,7 @@ fn native_projection(
             sources,
         });
     }
-    let has_star: bool = analysis
-        .get("starProjections")
-        .and_then(Value::as_array)
-        .is_some_and(|stars| !stars.is_empty());
+    let has_star: bool = has_star_projection(analysis);
     if has_star && input.model.references.len() == 1 {
         let order: HashMap<&str, usize> = declared_order(input);
         let fallback: usize = order.len();

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs

@@ -265,6 +265,17 @@ pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec<Vec<String>>, Vec<Vec<
     )
 }
 
+/// [`session_lines`]' steps with rich type inference as given, every model's types declared empty.
+pub(crate) fn session_steps(models: &[ModelSpec], rich_type_inference: bool) -> Vec<Vec<String>> {
+    let mut request: SessionRequest = orders_request(model_requests(models));
+    request.rich_type_inference = rich_type_inference;
+    for (name, ..) in models {
+        request.column_types.push(((*name).to_owned(), Vec::new()));
+    }
+    let (steps, _, _) = completed(started(request));
+    steps.iter().map(step_lines).collect()
+}
+
 /// Run `models` and describe the facts the finished session kept for each, or `none`.
 pub(crate) fn fact_lines(models: &[ModelSpec]) -> Vec<String> {
     let (_, _, finished) = completed(started(orders_request(model_requests(models))));

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs

@@ -6,11 +6,12 @@ use crate::assembly::analysis_session::models::PivotOutcome;
 use crate::assembly::analysis_session::tests::helpers::{
     catalog, expected_facts, fact_lines, failed, finished_session, kept_model_analyses,
     legacy_lines, lines, model_requests, orders_request, pivot_request, proven, recovered_facts,
-    session_lines,
+    session_lines, session_steps,
 };
 use crate::assembly::analysis_session::tests::test_types::{
     CteRecoveryTestCase, KeptAnalysisTestCase, LegacyAnalysisTestCase, ModelSpec, PivotTestCase,
-    SessionFactsTestCase, SessionFailureTestCase, SessionTestCase, UnscheduledTestCase,
+    SessionFactsTestCase, SessionFailureTestCase, SessionTestCase, StarExpansionTestCase,
+    UnscheduledTestCase,
 };
 
 /// A model whose missing column stays untyped over a known input, so completion re-analyses it.
@@ -21,6 +22,9 @@ const UNTYPED_ORDERS: ModelSpec = (
     &[],
 );
 
+/// A producer whose output types stay unknown without rich inference.
+const STAR_PRODUCER: ModelSpec = ("stg_orders", "SELECT 1 AS order_id, 10 AS amount", &[], &[]);
+
 #[test]
 fn given_models_when_running_the_session_then_publishes_as_python_does() {
     let test_cases = [
@@ -174,6 +178,68 @@ fn given_models_when_running_the_session_then_publishes_as_python_does() {
     }
 }
 
+#[test]
+fn given_a_star_beside_a_cast_constant_when_running_the_session_then_expands_the_star() {
+    let test_cases = [
+        StarExpansionTestCase {
+            description: "fast inference keeps the starred columns before the cast constant",
+            rich_type_inference: false,
+            models: &[
+                STAR_PRODUCER,
+                (
+                    "orders_flagged",
+                    "SELECT *, CAST(1 AS INTEGER) AS flag FROM __ref(\"stg_orders\")",
+                    &[],
+                    &["stg_orders"],
+                ),
+            ],
+            expected_steps: &[&[
+                "publish stg_orders order_id:UNKNOWN amount:UNKNOWN",
+                "publish orders_flagged order_id:UNKNOWN amount:UNKNOWN flag:INT",
+            ]],
+        },
+        StarExpansionTestCase {
+            description: "fast inference expands a star over a derived table",
+            rich_type_inference: false,
+            models: &[
+                STAR_PRODUCER,
+                (
+                    "orders_flagged",
+                    "SELECT *, CAST(1 AS INTEGER) AS flag \
+                     FROM (SELECT order_id, amount FROM __ref(\"stg_orders\")) AS edited",
+                    &[],
+                    &["stg_orders"],
+                ),
+            ],
+            expected_steps: &[&[
+                "publish stg_orders order_id:UNKNOWN amount:UNKNOWN",
+                "publish orders_flagged order_id:UNKNOWN amount:UNKNOWN flag:INT",
+            ]],
+        },
+        StarExpansionTestCase {
+            description: "rich inference keeps the starred columns beside the cast constant",
+            rich_type_inference: true,
+            models: &[
+                STAR_PRODUCER,
+                (
+                    "orders_flagged",
+                    "SELECT *, CAST(1 AS INTEGER) AS flag FROM __ref(\"stg_orders\")",
+                    &[],
+                    &["stg_orders"],
+                ),
+            ],
+            expected_steps: &[&[
+                "publish stg_orders order_id:INT amount:INT",
+                "publish orders_flagged order_id:INT amount:INT flag:INT",
+            ]],
+        },
+    ];
+    for test_case in test_cases {
+        let steps = session_steps(test_case.models, test_case.rich_type_inference);
+        assert_eq!(steps, test_case.expected_steps, "{}", test_case.description);
+    }
+}
+
 #[test]
 fn given_finished_sessions_when_reading_model_facts_then_keeps_only_native_successes() {
     let test_cases = [

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs

@@ -17,6 +17,15 @@ pub(crate) struct SessionTestCase {
     pub(crate) expected_outcomes: &'static [&'static [&'static str]],
 }
 
+/// A session over the orders request, run with or without rich type inference.
+pub(crate) struct StarExpansionTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) rich_type_inference: bool,
+    pub(crate) models: &'static [ModelSpec],
+    /// `step_lines` of each step, in order.
+    pub(crate) expected_steps: &'static [&'static [&'static str]],
+}
+
 pub(crate) struct SessionFactsTestCase {
     pub(crate) description: &'static str,
     pub(crate) models: &'static [ModelSpec],

crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs

@@ -9,6 +9,8 @@ use std::sync::LazyLock;
 
 const DBT_REF: &str = "dbt_ref";
 const UDF: &str = "udf";
+/// Snowflake's second line comment prefix; only Snowflake SQL reaches the variant scan.
+const SNOWFLAKE_LINE_COMMENT: &str = "//";
 type Edit = (usize, usize, MappedSql);
 static REFERENCES: LazyLock<Result<Regex, String>> = LazyLock::new(|| {
     Regex::new(r#"__(ref|seed|source|dbt_ref)\((?:"[^"]+"\s*,\s*)?"([^"]+)"\)"#)
@@ -201,6 +203,13 @@ fn skip_non_code(sql: &str, index: usize) -> Result<Option<usize>, String> {
                 .map_or(sql.len(), |end| index + 4 + end),
         ));
     }
+    if sql[index..].starts_with(SNOWFLAKE_LINE_COMMENT) {
+        return Ok(Some(
+            sql[index..]
+                .find('\n')
+                .map_or(sql.len(), |offset| index + offset + 1),
+        ));
+    }
     non_code_end(
         sql.as_bytes(),
         index,

crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs

@@ -30,6 +30,19 @@ fn given_batch_when_normalizing_on_pool_or_in_turn_then_matches_single_calls_in_
             ],
             expected_failures: &[0, 2],
         },
+        NormalizationBatchTestCase {
+            description: "double-slash comments hide quotes and brackets",
+            dialect: "snowflake",
+            requests: &[
+                ("SELECT id // it's the key\nFROM orders", &[]),
+                (
+                    "// don't read \"items[0\nSELECT payload:item[0] AS item FROM inventory",
+                    &[],
+                ),
+                ("SELECT id FROM orders // trailing 'note", &[]),
+            ],
+            expected_failures: &[],
+        },
     ];
     let pool = rayon::ThreadPoolBuilder::new()
         .num_threads(3)

crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs

@@ -48,6 +48,7 @@ use crate::bindings::_helpers::records::fields::field as native_field;
 use crate::bindings::_helpers::records::fields::mapping_get;
 use crate::bindings::_helpers::records::fields::mapping_items;
 use crate::bindings::_helpers::records::ids::RecordId;
+use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns;
 use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal;
 
 const CHANGES_LOGGER: &str = "sqlbuild.planner.changes";
@@ -531,6 +532,11 @@ fn model_schema_findings<'py>(
     if warehouse.is_none() {
         return Ok(Vec::new());
     }
+    let validity: Vec<String> = snapshot_validity_columns(model)?;
+    let mut warehouse: Vec<(String, String)> = column_pairs(&warehouse)?;
+    for column in &validity {
+        warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name));
+    }
     let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?;
     let mut yml: Vec<(String, String)> = Vec::new();
     let mut type_enforcement: bool = false;
@@ -559,7 +565,7 @@ fn model_schema_findings<'py>(
         SchemaColumns {
             yml,
             inferred,
-            warehouse: column_pairs(&warehouse)?,
+            warehouse,
         },
         (type_enforcement, inferred_schema_complete),
         (dynamic, dialect),

crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs

@@ -107,27 +107,36 @@ pub(crate) fn published_shape<'py>(
     inputs: &Bound<'py, PyDict>,
     profile: &Bound<'py, PyAny>,
 ) -> PyResult<Bound<'py, PyDict>> {
-    let py: Python<'py> = model.py();
     let shape: Bound<'py, PyDict> = binding_shape(model, columns, inputs, profile)?;
-    let values: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")?;
-    let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?;
-    if mapping_get(&values, "materialized")?.eq(snapshot)? {
-        for (key, default) in [
-            ("valid_from_column", VALID_FROM_DEFAULT),
-            ("valid_to_column", VALID_TO_DEFAULT),
-        ] {
-            let name: Bound<'py, PyAny> = mapping_get(&values, key)?;
-            let name: Bound<'py, PyAny> = if name.is_truthy()? {
-                name
-            } else {
-                PyString::new(py, default).into_any()
-            };
-            shape.set_item(name.str()?, UNKNOWN)?;
-        }
+    for name in snapshot_validity_columns(model)? {
+        shape.set_item(name, UNKNOWN)?;
     }
     Ok(shape)
 }
 
+/// A snapshot model's validity column names, which the snapshot adds to its query's columns.
+pub(crate) fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
+    let py: Python<'_> = model.py();
+    let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?;
+    let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?;
+    if !mapping_get(&values, "materialized")?.eq(snapshot)? {
+        return Ok(Vec::new());
+    }
+    let mut names: Vec<String> = Vec::with_capacity(2);
+    for (key, default) in [
+        ("valid_from_column", VALID_FROM_DEFAULT),
+        ("valid_to_column", VALID_TO_DEFAULT),
+    ] {
+        let name: Bound<'_, PyAny> = mapping_get(&values, key)?;
+        names.push(if name.is_truthy()? {
+            name.str()?.extract()?
+        } else {
+            default.to_owned()
+        });
+    }
+    Ok(names)
+}
+
 /// Inferred columns, re-quoted from authored SQL where the dialect keeps quoted case.
 fn binding_shape<'py>(
     model: &Bound<'py, PyAny>,

src/sqlbuild/adapter/relations/constants.py

@@ -5,3 +5,4 @@
 METADATA_NAME_FILTER_LIMIT: int = 250
 RELATION_COLUMNS_LOOKUP: str = "columns"
 RELATION_EXISTS_LOOKUP: str = "exists"
+RELATION_TYPE_LOOKUP: str = "type"