feat: govern dynamic pivot column families

Makefile

@@ -61,6 +61,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \

crates/sqlbuild-rules-native/src/constants.rs

@@ -13,6 +13,7 @@ pub(crate) const REFERENCE_KIND: &str = "ref";
 pub(crate) const SOURCE_REFERENCE_KIND: &str = "source";
 pub(crate) const VIEW_MATERIALIZATION: &str = "view";
 pub(crate) const ENFORCED_CONTRACT: &str = "enforced";
+pub(crate) const SELECT_STAR_MODEL_RULE_CODE: &str = "SQBRMODEL102";
 pub(crate) const BOOLEAN_TYPE: &str = "BOOLEAN";
 pub(crate) const TIMESTAMP_TYPE: &str = "TIMESTAMP";
 pub(crate) const DATE_TYPE: &str = "DATE";

crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs

@@ -544,6 +544,52 @@ fn given_enforced_contract_outputs_when_evaluating_explicit_type_rule_then_retur
     Ok(())
 }
 
+#[test]
+fn given_compiler_proven_duckdb_dynamic_pivot_when_evaluating_contract_rules_then_skips_query_parse()
+-> Result<(), String> {
+    let test_cases = [test_types::DynamicPivotEvaluationTestCase {
+        description: "compiler-proven bare DuckDB pivot",
+        expected_faults: json!([]),
+    }];
+    for test_case in test_cases {
+        let project_dir = TempDir::new().map_err(|error| error.to_string())?;
+        let config = json!({
+            "select": ["SQBRMODEL102", "SQBRCONTRACT101", "SQBRCONTRACT105", "SQBRCONTRACT106"],
+            "cache": {"enabled": false}
+        });
+        let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config))
+            .map_err(|error| error.to_string())?;
+        request["dialect"] = json!("duckdb");
+        request["models"][0]["query_sql"] =
+            json!("PIVOT orders ON category USING MAX(amount) GROUP BY customer_id");
+        request["models"][0]["authored_sql"] = request["models"][0]["query_sql"].clone();
+        request["models"][0]["config"] = json!({"contract": "enforced"});
+        request["models"][0]["columns"] = json!([
+            {"name": "customer_id", "type": "INTEGER", "type_proven": true}
+        ]);
+        request["models"][0]["dynamic_columns"] = json!([{
+            "name": "category_amounts",
+            "pivot_column": "category",
+            "value_column": "amount",
+            "aggregate": "MAX",
+            "type": "DECIMAL(12,2)",
+            "type_proven": true
+        }]);
+        request["models"][0]["dynamic_columns_proven"] = json!(true);
+        request["models"][0]["bare_dynamic_pivot"] = json!(true);
+
+        let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?)
+            .map_err(|error| error.to_string())?;
+
+        assert_eq!(
+            result["faults"], test_case.expected_faults,
+            "{}",
+            test_case.description
+        );
+    }
+    Ok(())
+}
+
 #[test]
 fn given_contract_name_type_options_when_evaluating_then_accepts_configured_interfaces()
 -> Result<(), String> {

crates/sqlbuild-rules-native/src/engine/tests/test_types.rs

@@ -6,6 +6,11 @@ pub(crate) struct NativeEvaluationTestCase {
     pub(crate) expected_faults: Value,
 }
 
+pub(crate) struct DynamicPivotEvaluationTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) expected_faults: Value,
+}
+
 pub(crate) struct DeclarationScopeTestCase {
     pub(crate) description: &'static str,
     pub(crate) scope: &'static str,

crates/sqlbuild-rules-native/src/models.rs

@@ -185,6 +185,19 @@ pub(crate) struct Column {
     pub type_proven: bool,
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(default, deny_unknown_fields)]
+pub(crate) struct DynamicColumnFamily {
+    pub name: String,
+    pub pivot_column: String,
+    pub value_column: String,
+    pub aggregate: String,
+    #[serde(rename = "type", alias = "data_type")]
+    pub data_type: String,
+    pub name_pattern: Option<String>,
+    pub type_proven: bool,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(default, deny_unknown_fields)]
 pub(crate) struct DeclarationMember {
@@ -497,6 +510,9 @@ pub(crate) struct Model {
     pub config: BTreeMap<String, Value>,
     pub references: Vec<Reference>,
     pub columns: Vec<Column>,
+    pub dynamic_columns: Vec<DynamicColumnFamily>,
+    pub dynamic_columns_proven: bool,
+    pub bare_dynamic_pivot: bool,
     pub enum_columns: Vec<String>,
     pub enum_declarations: Vec<Declaration>,
     pub constant_declarations: Vec<Declaration>,

crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs

@@ -0,0 +1,120 @@
+use std::collections::BTreeMap;
+
+use crate::constants::{ENFORCED_CONTRACT, SELECT_STAR_MODEL_RULE_CODE};
+use crate::models::{Fault, Model, RuleMetadata, RulesConfig};
+use crate::rules::_helpers::{contract_name_types, explicit_output_types, typed_contract_columns};
+use crate::rules::models::FaultCollector;
+use sqlparser::ast::{PivotValueSource, Query, Select, SetExpr, TableFactor};
+
+pub(crate) fn evaluate_without_query(
+    model: &Model,
+    config: &RulesConfig,
+    selected: &BTreeMap<String, &RuleMetadata>,
+) -> Option<Vec<Fault>> {
+    if !model.dynamic_columns_proven || !rules_need_no_query(model, selected) {
+        return None;
+    }
+    let faults = FaultCollector::default();
+    if let Some(rule) = selected.get("SQBRCONTRACT101")
+        && model
+            .config
+            .get("contract")
+            .and_then(serde_json::Value::as_str)
+            != Some(ENFORCED_CONTRACT)
+    {
+        faults.push(fault(model, rule));
+    }
+    if let Some(rule) = selected.get("SQBRCONTRACT106") {
+        typed_contract_columns::evaluate(model, rule, &faults);
+    }
+    if let Some(rule) = selected.get("SQBRCONTRACT105") {
+        explicit_output_types::evaluate_proven_dynamic(model, rule, &faults);
+    }
+    for column in &model.columns {
+        if let Some((code, message)) = contract_name_types::finding(column, config)
+            && let Some(rule) = selected.get(code)
+        {
+            faults.push(Fault {
+                code: rule.code.clone(),
+                path: model.relative_path.clone(),
+                line: 1,
+                column: 1,
+                message,
+                remediation: rule.remediation.clone(),
+            });
+        }
+    }
+    Some(faults.into_inner())
+}
+
+pub(crate) fn allows_output_star(query: &Query, root: &Query, model: &Model) -> bool {
+    let Some(select) = root_select(query) else {
+        return false;
+    };
+    let dynamic_pivot_star = select.projection.len() == 1
+        && select.from.len() == 1
+        && select.from[0].joins.is_empty()
+        && matches!(
+            &select.from[0].relation,
+            TableFactor::Pivot {
+                value_source: PivotValueSource::Any(_),
+                ..
+            }
+        );
+    dynamic_pivot_star
+        || (model.dynamic_columns_proven
+            && std::ptr::eq(query, root)
+            && root
+                .with
+                .as_ref()
+                .and_then(|with| with.cte_tables.last())
+                .is_some_and(|cte| {
+                    sole_table_name(query)
+                        .is_some_and(|name| name.eq_ignore_ascii_case(&cte.alias.name.value))
+                }))
+}
+
+fn root_select(query: &Query) -> Option<&Select> {
+    let SetExpr::Select(select) = query.body.as_ref() else {
+        return None;
+    };
+    Some(select)
+}
+
+fn sole_table_name(query: &Query) -> Option<String> {
+    let select = root_select(query)?;
+    if select.from.len() != 1 || !select.from[0].joins.is_empty() {
+        return None;
+    }
+    let TableFactor::Table {
+        name, args: None, ..
+    } = &select.from[0].relation
+    else {
+        return None;
+    };
+    Some(name.to_string())
+}
+
+fn rules_need_no_query(model: &Model, selected: &BTreeMap<String, &RuleMetadata>) -> bool {
+    selected.values().all(|rule| {
+        rule.custom
+            || rule.code.starts_with("SQBRSQL")
+            || rule.code.starts_with("SQBRCONTRACT10")
+            || (model.bare_dynamic_pivot && rule.code == SELECT_STAR_MODEL_RULE_CODE)
+            || matches!(
+                rule.code.as_str(),
+                "SQBRDECLARATION201" | "SQBRDECLARATION301" | "SQBRTEST301"
+            )
+    })
+}
+
+fn fault(model: &Model, rule: &RuleMetadata) -> Fault {
+    Fault {
+        code: rule.code.clone(),
+        path: model.relative_path.clone(),
+        line: 1,
+        column: 1,
+        message: rule.message.clone(),
+        remediation: rule.remediation.clone(),
+    }
+}

crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs

@@ -5,15 +5,16 @@ use crate::constants::{
 use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig};
 use crate::rules::_helpers::domain_layout::folder_layer_details;
 use crate::rules::_helpers::{
-    contract_name_types, explicit_output_types, numeric_decisions, typed_contract_columns,
+    contract_name_types, dynamic_contracts, explicit_output_types, numeric_decisions,
+    typed_contract_columns,
 };
 use crate::rules::models::{
     FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride,
 };
 use globset::{Glob, GlobSetBuilder};
 use sqlparser::ast::{
-    BinaryOperator, Expr, GroupByExpr, JoinConstraint, JoinOperator, PivotValueSource, Query,
-    Select, SelectItem, SetExpr, Spanned, Statement, TableFactor, Value, Visit, Visitor,
+    BinaryOperator, Expr, GroupByExpr, JoinConstraint, JoinOperator, Query, Select, SelectItem,
+    SetExpr, Spanned, Statement, TableFactor, Value, Visit, Visitor,
 };
 use sqlparser::dialect::{
     BigQueryDialect, ClickHouseDialect, DatabricksDialect, Dialect, DuckDbDialect, GenericDialect,
@@ -120,6 +121,10 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result<Vec<Fault
     } = request;
     let metadata = |code: &str| selected.get(code).copied();
     let faults = FaultCollector::default();
+    if let Some(dynamic_faults) = dynamic_contracts::evaluate_without_query(model, config, selected)
+    {
+        return Ok(dynamic_faults);
+    }
     if !requires_parsed_model(selected) {
         return Ok(faults.into_inner());
     }
@@ -1006,22 +1011,14 @@ fn select_star(
         let Some(select) = root_select(query) else {
             continue;
         };
-        let dynamic_pivot_star = select.projection.len() == 1
-            && select.from.len() == 1
-            && select.from[0].joins.is_empty()
-            && matches!(
-                &select.from[0].relation,
-                TableFactor::Pivot {
-                    value_source: PivotValueSource::Any(_),
-                    ..
-                }
-            );
+        let dynamic_output_star =
+            dynamic_contracts::allows_output_star(query, &parsed.query, parsed.model);
         let stars = stars_in_select(select);
         for at in stars {
             if import_positions.contains(&(at.line, at.column)) {
                 continue;
             }
-            if dynamic_pivot_star {
+            if dynamic_output_star {
                 continue;
             }
             let lone = select.projection.len() == 1;

crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs

@@ -25,10 +25,14 @@ pub(crate) fn evaluate(query: &Query, model: &Model, rule: &RuleMetadata, faults
         .get("contract")
         .and_then(serde_json::Value::as_str)
         != Some(ENFORCED_CONTRACT)
-        || !model
+        || (!model
             .columns
             .iter()
             .any(|column| !column.data_type.is_empty())
+            && !model
+                .dynamic_columns
+                .iter()
+                .any(|family| !family.data_type.is_empty()))
     {
         return;
     }
@@ -59,6 +63,49 @@ pub(crate) fn evaluate(query: &Query, model: &Model, rule: &RuleMetadata, faults
     );
 }
 
+pub(crate) fn evaluate_proven_dynamic(model: &Model, rule: &RuleMetadata, faults: &FaultCollector) {
+    for column in model
+        .columns
+        .iter()
+        .filter(|column| !column.data_type.is_empty() && !column.type_proven)
+    {
+        faults.push(Fault {
+            code: rule.code.clone(),
+            path: model.relative_path.clone(),
+            line: 1,
+            column: 1,
+            message: format!(
+                "fixed dynamic-pivot output {:?} is not proven against declared type {}",
+                column.name, column.data_type
+            ),
+            remediation: format!(
+                "Establish type {} on the authoritative pivot input column.",
+                column.data_type
+            ),
+        });
+    }
+    for family in model
+        .dynamic_columns
+        .iter()
+        .filter(|family| !family.type_proven)
+    {
+        faults.push(Fault {
+            code: rule.code.clone(),
+            path: model.relative_path.clone(),
+            line: 1,
+            column: 1,
+            message: format!(
+                "dynamic output family {:?} is not proven against declared type {}",
+                family.name, family.data_type
+            ),
+            remediation: format!(
+                "Establish type {} on the authoritative pivot aggregate input.",
+                family.data_type
+            ),
+        });
+    }
+}
+
 impl OutputRule<'_> {
     fn evaluate_set(&self, set_expr: &SetExpr, scope: OutputScope<'_>) {
         match set_expr {
@@ -97,6 +144,15 @@ impl OutputRule<'_> {
                 item,
                 SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _)
             ) {
+                if self.model.dynamic_columns_proven
+                    && self
+                        .model
+                        .dynamic_columns
+                        .iter()
+                        .all(|family| family.type_proven)
+                {
+                    continue;
+                }
                 if proven_dependency_wildcard(select, scope.columns) {
                     continue;
                 }

crates/sqlbuild-rules-native/src/rules/_helpers/mod.rs

@@ -1,6 +1,7 @@
 pub(crate) mod catalogue;
 pub(crate) mod contract_name_types;
 pub(crate) mod domain_layout;
+pub(crate) mod dynamic_contracts;
 pub(crate) mod evaluation;
 pub(crate) mod explicit_output_types;
 pub(crate) mod numeric_decisions;

crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs

@@ -23,4 +23,19 @@ pub(crate) fn evaluate(model: &Model, rule: &RuleMetadata, faults: &FaultCollect
             });
         }
     }
+    for family in &model.dynamic_columns {
+        if family.data_type.trim().is_empty() {
+            faults.push(Fault {
+                code: rule.code.clone(),
+                path: model.relative_path.clone(),
+                line: 1,
+                column: 1,
+                message: format!(
+                    "dynamic contract family {:?} has no declared type",
+                    family.name
+                ),
+                remediation: rule.remediation.clone(),
+            });
+        }
+    }
 }