diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index 78f0b3bdba..88a2523cb7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -1,8 +1,11 @@ //! Python's `analyze_dynamic_column_contract` over the crate's parse of the model SQL. use std::collections::{HashMap, HashSet}; +use std::panic::{AssertUnwindSafe, catch_unwind}; use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; +use rayon::ThreadPool; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde_json::{Map, Value, json}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; @@ -17,7 +20,7 @@ use crate::assembly::analysis_session::constants::{ UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ - ColumnFact, ContractProof, DynamicFamily, PivotOutcome, + ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -33,6 +36,47 @@ pub(crate) struct PivotFacts<'a> { pub(crate) families_by_table: &'a [(String, Vec)], } +/// Each model's proof in order; a model whose proof panics is deferred to Python. +pub(crate) fn pivot_outcomes(models: &[PivotModel], tables: &PivotTables) -> Vec { + let facts = pivot_facts(tables); + models + .iter() + .map(|model| guarded_outcome(model, &facts)) + .collect() +} + +/// Each model's proof in order on `pool`; a model whose proof panics is deferred to Python. +pub(crate) fn pooled_pivot_outcomes( + pool: &ThreadPool, + models: &[PivotModel], + tables: &PivotTables, +) -> Vec { + let facts = pivot_facts(tables); + pool.install(|| { + models + .par_iter() + .map(|model| guarded_outcome(model, &facts)) + .collect() + }) +} + +fn pivot_facts(tables: &PivotTables) -> PivotFacts<'_> { + PivotFacts { + dialect: &tables.dialect, + column_types: &tables.column_types, + authoritative_types: &tables.authoritative_types, + column_nullability: &tables.column_nullability, + families_by_table: &tables.families_by_table, + } +} + +fn guarded_outcome(model: &PivotModel, facts: &PivotFacts<'_>) -> PivotOutcome { + catch_unwind(AssertUnwindSafe(|| { + pivot_outcome(&model.sql, &model.families, facts) + })) + .unwrap_or(PivotOutcome::Deferred) +} + /// One CTE body by folded name; None for a CTE with column aliases. type CteMap<'a> = HashMap>; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index cf945f7c71..de54b97e79 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -1,7 +1,9 @@ //! The session's phases: Python's uncached model analysis, wave by wave. use std::collections::HashMap; +use std::sync::Arc; +use rayon::ThreadPool; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; @@ -21,8 +23,9 @@ use crate::assembly::analysis_session::constants::{ DEFERRAL_ANALYSIS, DEFERRAL_ENRICHMENT, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ - AnalysisSession, Awaiting, ColumnFact, Deferral, DeferredAnalysis, LineageFacts, ModelAnalysis, - ModelOutcome, ModelRequest, Phase, PivotOutcome, SessionOutcome, SessionRequest, SessionStep, + AnalysisSession, Awaiting, ColumnFact, Deferral, DeferredAnalysis, FinishedSession, + LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, + PivotTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::ProjectCatalog; @@ -148,23 +151,43 @@ impl AnalysisSession { } /// Every model's outcome and the binding catalog changes Python records. - pub(crate) fn finish(self) -> Result { + pub(crate) fn finish(self) -> Result<(SessionOutcome, FinishedSession), String> { if !matches!(self.phase, Phase::Done) { return Err("the session has not finished".to_owned()); } let dynamic_contracts: Vec = self.dynamic_contracts()?; + let pool: Result, String> = self.catalog.native.analysis_pool(); let models: Vec = self .outcomes .into_iter() .collect::>() .ok_or("the session left a model unanalysed")?; let (schema_additions, analysis_names) = self.catalog.into_changes(); - Ok(SessionOutcome { + let request: SessionRequest = self.request; + let mut model_facts: HashMap = HashMap::new(); + for (model, outcome) in request.models.iter().zip(&models) { + if let Some(facts) = session_model_facts(&outcome.analysis) { + model_facts.insert(model.name.clone(), facts); + } + } + let finished = FinishedSession { + tables: PivotTables { + dialect: request.dialect, + column_types: request.column_types, + authoritative_types: request.complete_schemas, + column_nullability: request.column_nullability, + families_by_table: request.dynamic_families_by_table, + }, + pool, + models: model_facts, + }; + let outcome = SessionOutcome { models, schema_additions, analysis_names, dynamic_contracts, - }) + }; + Ok((outcome, finished)) } /// Python's dynamic pivot proof of each model over the relation facts before analysis. @@ -595,3 +618,28 @@ fn native_answer(model: usize, columns: Vec, has_star: bool) -> Defe binding_validated: false, } } + +/// A compiled model's output names and lineage from a successful analysis with native lineage. +fn session_model_facts(analysis: &ModelAnalysis) -> Option { + let rows: &Vec = match &analysis.lineage { + LineageFacts::Native(rows) | LineageFacts::NativeEnrichment(rows) => rows, + LineageFacts::PythonAnalysis | LineageFacts::PythonEnrichment => return None, + }; + analysis.analysis_succeeded.then(|| SessionModelFacts { + columns: analysis.columns.as_deref().map(column_names), + lineage: rows.iter().map(output_sources).collect(), + }) +} + +fn column_names(columns: &[ColumnFact]) -> Vec { + columns.iter().map(|column| column.name.clone()).collect() +} + +/// One lineage row as `(output column, [(resource name, column name)])`. +fn output_sources(row: &LineageRow) -> (String, Vec<(String, String)>) { + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (_, name, column) in &row.sources { + sources.push((name.clone(), column.clone())); + } + (row.output_column.clone(), sources) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs index f7a4fec8b3..11185a1ad6 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs @@ -1,8 +1,10 @@ //! Collect a finished native model analysis session's outcomes. -use crate::assembly::analysis_session::models::{AnalysisSession, SessionOutcome}; +use crate::assembly::analysis_session::models::{AnalysisSession, FinishedSession, SessionOutcome}; -/// Every model's outcome and the binding catalog changes Python records. -pub fn finish_analysis_session(session: AnalysisSession) -> Result { +/// Every model's outcome and catalog changes, and the session kept for later pivot proofs. +pub fn finish_analysis_session( + session: AnalysisSession, +) -> Result<(SessionOutcome, FinishedSession), String> { session.finish() } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs new file mode 100644 index 0000000000..13692c612d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs @@ -0,0 +1,8 @@ +//! The models whose output names and lineage a finished analysis session kept. + +use crate::assembly::analysis_session::models::FinishedSession; + +/// Names of the models whose facts the session kept, in no particular order. +pub fn finished_fact_models(session: &FinishedSession) -> Vec { + session.models.keys().cloned().collect() +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs new file mode 100644 index 0000000000..bac994679b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs @@ -0,0 +1,11 @@ +//! The output names and lineage a finished analysis session kept for compiled models. + +use crate::assembly::analysis_session::models::{FinishedSession, SessionModelFacts}; + +/// The named model's facts, or None where the session kept none for it. +pub fn finished_model_facts<'a>( + session: &'a FinishedSession, + name: &str, +) -> Option<&'a SessionModelFacts> { + session.models.get(name) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index 6c5394355c..d3a5e4c347 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -1,6 +1,9 @@ pub mod expression_shapes; pub mod finish_analysis_session; -pub mod prove_dynamic_contract; +pub mod finished_fact_models; +pub mod finished_model_facts; +pub mod prove_dynamic_contracts; +pub mod prove_finished_dynamic_contracts; pub mod provide_deferred_analyses; pub mod run_analysis_session; pub mod start_analysis_session; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs deleted file mode 100644 index 96759642d4..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Python's `analyze_dynamic_column_contract` for one model outside the analysis session. - -use crate::assembly::analysis_session::_helpers::dynamic_pivot::{PivotFacts, pivot_outcome}; -use crate::assembly::analysis_session::constants::PIVOT_WORKER_STACK_BYTES; -use crate::assembly::analysis_session::models::{PivotOutcome, PivotRequest}; - -/// The model's proof, or a deferral where Python's result is not reproduced exactly. -pub fn prove_dynamic_contract(request: &PivotRequest) -> Result { - let facts = PivotFacts { - dialect: &request.dialect, - column_types: &request.column_types, - authoritative_types: &request.authoritative_types, - column_nullability: &request.column_nullability, - families_by_table: &request.families_by_table, - }; - std::thread::scope(|scope| { - std::thread::Builder::new() - .stack_size(PIVOT_WORKER_STACK_BYTES) - .spawn_scoped(scope, || { - pivot_outcome(&request.sql, &request.families, &facts) - }) - .map_err(|error| error.to_string())? - .join() - .map_err(|_| "the native dynamic pivot proof panicked".to_owned()) - }) -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs new file mode 100644 index 0000000000..bbbb199898 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs @@ -0,0 +1,17 @@ +//! Python's `analyze_dynamic_column_contract` for models no analysis session analysed. + +use crate::assembly::analysis_session::_helpers::dynamic_pivot::pivot_outcomes; +use crate::assembly::analysis_session::constants::PIVOT_WORKER_STACK_BYTES; +use crate::assembly::analysis_session::models::{PivotBatchRequest, PivotOutcome}; + +/// Each model's proof in order; a model whose proof panics is deferred to Python. +pub fn prove_dynamic_contracts(request: &PivotBatchRequest) -> Result, String> { + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(PIVOT_WORKER_STACK_BYTES) + .spawn_scoped(scope, || pivot_outcomes(&request.models, &request.tables)) + .map_err(|error| error.to_string())? + .join() + .map_err(|_| "the native dynamic pivot proofs panicked".to_owned()) + }) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs new file mode 100644 index 0000000000..28a9f22db8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs @@ -0,0 +1,13 @@ +//! Python's `analyze_dynamic_column_contract` for models outside a finished analysis session. + +use crate::assembly::analysis_session::_helpers::dynamic_pivot::pooled_pivot_outcomes; +use crate::assembly::analysis_session::models::{FinishedSession, PivotModel, PivotOutcome}; + +/// Each model's proof in order from the session's own tables; a panicking proof is deferred. +pub fn prove_finished_dynamic_contracts( + session: &FinishedSession, + models: &[PivotModel], +) -> Result, String> { + let pool = session.pool.as_ref().map_err(Clone::clone)?; + Ok(pooled_pivot_outcomes(pool, models, &session.tables)) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index d10d56c0ad..f5988b2b08 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -1,10 +1,13 @@ //! Plain-data request, deferrals and outcomes of the native model analysis session. use std::collections::HashMap; +use std::sync::Arc; + +use rayon::ThreadPool; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; -use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::assembly::analysis_session::types::{OutputSources, Pairs, Shapes}; use crate::semantic_validation::types::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. @@ -61,18 +64,45 @@ pub struct ContractProof { pub bare_dynamic_pivot: bool, } -/// One model's dynamic pivot proof inputs, for a model the session does not analyse. +/// The relation facts every dynamic pivot proof reads: Python's tables before analysis. #[derive(Debug, Clone, Default)] -pub struct PivotRequest { +pub struct PivotTables { pub dialect: String, pub column_types: Shapes, pub authoritative_types: Shapes, pub column_nullability: Shapes, pub families_by_table: Vec<(String, Vec)>, +} + +/// One model's pivot SQL and declared families. +#[derive(Debug, Clone, Default)] +pub struct PivotModel { pub sql: String, pub families: Vec, } +/// Dynamic pivot proofs for models no session analysed. +#[derive(Debug, Clone, Default)] +pub struct PivotBatchRequest { + pub tables: PivotTables, + pub models: Vec, +} + +/// A finished session's pivot tables, analysis pool and model facts, kept for later stages. +#[derive(Debug)] +pub struct FinishedSession { + pub(crate) tables: PivotTables, + pub(crate) pool: Result, String>, + pub(crate) models: HashMap, +} + +/// A compiled model's output names and lineage from an analysis the session completed natively. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionModelFacts { + pub columns: Option>, + pub lineage: OutputSources, +} + /// A model's dynamic pivot proof: none declared, left to Python, or proven natively. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PivotOutcome { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 1f274fdfc1..5705bd4a1b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -1,12 +1,15 @@ +use std::collections::HashMap; + use crate::assembly::analysis_session::_helpers::mappings::catalog_relations; use crate::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; +use crate::assembly::analysis_session::main::finished_model_facts::finished_model_facts; use crate::assembly::analysis_session::main::provide_deferred_analyses::provide_deferred_analyses; use crate::assembly::analysis_session::main::run_analysis_session::run_analysis_session; use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, Deferral, DeferredAnalysis, DynamicFamily, - ModelOutcome, ModelReference, ModelRequest, PivotOutcome, PivotRequest, SessionRequest, - SessionStep, + FinishedSession, ModelOutcome, ModelReference, ModelRequest, PivotBatchRequest, PivotModel, + PivotOutcome, PivotTables, SessionModelFacts, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::tests::test_types::ModelSpec; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -113,11 +116,11 @@ pub(crate) fn started(request: SessionRequest) -> AnalysisSession { start_analysis_session(request, &catalog).expect("the models form no cycle") } -/// Run to completion, answering every deferral with `answer`; return the steps and outcomes. +/// Run to completion, answering every deferral with `answer`: the steps, outcomes and session. pub(crate) fn completed( mut session: AnalysisSession, answer: fn(&Deferral) -> DeferredAnalysis, -) -> (Vec, Vec) { +) -> (Vec, Vec, FinishedSession) { let mut done: bool = false; let steps: Vec = std::iter::from_fn(|| { (!done).then(|| { @@ -131,8 +134,8 @@ pub(crate) fn completed( }) }) .collect(); - let outcome = finish_analysis_session(session).expect("the session finished"); - (steps, outcome.models) + let (outcome, finished) = finish_analysis_session(session).expect("the session finished"); + (steps, outcome.models, finished) } /// `name type nullability` per column, or `failed` when analysis did not succeed. @@ -205,7 +208,7 @@ pub(crate) fn model_requests(models: &[ModelSpec]) -> Vec { /// Run `models` over the orders request: each step's lines and each model's description. pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec>) { - let (steps, outcomes) = completed( + let (steps, outcomes, _) = completed( started(orders_request(model_requests(models))), empty_answer, ); @@ -215,12 +218,35 @@ pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec Vec { + let (_, _, finished) = completed( + started(orders_request(model_requests(models))), + empty_answer, + ); + models + .iter() + .map(|(name, ..)| { + finished_model_facts(&finished, name).map_or("none".to_owned(), fact_line) + }) + .collect() +} + +fn fact_line(facts: &SessionModelFacts) -> String { + let lineage: Vec = facts + .lineage + .iter() + .map(|(output, sources)| format!("{output}<-{sources:?}")) + .collect(); + format!("{:?} {}", facts.columns, lineage.join(" ")) +} + +/// Proofs over typed `raw_orders` and pivot `status_amounts`: the model, then a family-less one. pub(crate) fn pivot_request( dialect: &str, sql: &str, family: Option<(&str, &str, &str)>, -) -> PivotRequest { +) -> PivotBatchRequest { let raw_orders: Shapes = shapes(&[( "raw_orders", &[ @@ -237,17 +263,36 @@ pub(crate) fn pivot_request( }) .into_iter() .collect(); - PivotRequest { - dialect: dialect.to_owned(), - column_types: [raw_orders.clone(), upstream].concat(), - authoritative_types: raw_orders, - column_nullability: shapes(&[("raw_orders", &[("customer_id", "non_null")])]), - families_by_table: vec![( - "status_amounts".to_owned(), - vec![amounts("status", "amount", "MAX")], - )], - sql: sql.to_owned(), - families, + PivotBatchRequest { + tables: PivotTables { + dialect: dialect.to_owned(), + column_types: [raw_orders.clone(), upstream].concat(), + authoritative_types: raw_orders, + column_nullability: shapes(&[("raw_orders", &[("customer_id", "non_null")])]), + families_by_table: vec![( + "status_amounts".to_owned(), + vec![amounts("status", "amount", "MAX")], + )], + }, + models: vec![ + PivotModel { + sql: sql.to_owned(), + families, + }, + PivotModel { + sql: "SELECT 1".to_owned(), + families: Vec::new(), + }, + ], + } +} + +/// A finished session holding `tables` and an analysis pool. +pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { + FinishedSession { + tables, + pool: catalog("duckdb", &Vec::new()).analysis_pool(), + models: HashMap::new(), } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs index c56625f012..a2faa129b4 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs @@ -1,11 +1,13 @@ -use crate::assembly::analysis_session::main::prove_dynamic_contract::prove_dynamic_contract; +use crate::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use crate::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::PivotOutcome; use crate::assembly::analysis_session::tests::helpers::{ - catalog, failed, model_requests, orders_request, pivot_request, proven, session_lines, + catalog, fact_lines, failed, finished_session, model_requests, orders_request, pivot_request, + proven, session_lines, }; use crate::assembly::analysis_session::tests::test_types::{ - PivotTestCase, SessionTestCase, UnscheduledTestCase, + PivotTestCase, SessionFactsTestCase, SessionTestCase, UnscheduledTestCase, }; #[test] @@ -125,6 +127,64 @@ fn given_models_when_running_the_session_then_defers_and_publishes_as_python_doe } } +#[test] +fn given_finished_sessions_when_reading_model_facts_then_keeps_only_native_successes() { + let test_cases = [ + SessionFactsTestCase { + description: "native lineage keeps output names and sources", + models: &[ + ( + "stg_orders", + "SELECT order_id, amount * 2 AS doubled FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ( + "orders_bad", + "SELECT missing_column FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ], + expected_facts: &[ + "Some([\"order_id\", \"doubled\"]) \ + order_id<-[(\"raw_orders\", \"order_id\")] \ + doubled<-[(\"raw_orders\", \"amount\")]", + "Some([\"missing_column\"]) missing_column<-[(\"raw_orders\", \"missing_column\")]", + ], + }, + SessionFactsTestCase { + description: "a model Python analysed keeps no session facts", + models: &[ + ( + "events", + "SELECT event_id FROM __source(\"raw_events\")", + &["raw_events"], + &[], + ), + ( + "events_mart", + "SELECT event_id, 1 AS one FROM __ref(\"events\")", + &[], + &["events"], + ), + ], + expected_facts: &[ + "Some([\"event_id\"]) event_id<-[(\"raw_events\", \"event_id\")]", + "none", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + fact_lines(test_case.models), + test_case.expected_facts, + "{}", + test_case.description + ); + } +} + #[test] fn given_unschedulable_models_when_starting_then_python_analyses() { let test_cases = [ @@ -281,11 +341,21 @@ fn given_dynamic_pivots_when_proving_then_matches_python_or_defers() { ]; for test_case in test_cases { let request = pivot_request(test_case.dialect, test_case.sql, test_case.family); + let session = finished_session(request.tables.clone()); - let outcome = prove_dynamic_contract(&request).expect("the proof runs"); + let standalone = prove_dynamic_contracts(&request).expect("the proofs run"); + let in_session = + prove_finished_dynamic_contracts(&session, &request.models).expect("the proofs run"); assert_eq!( - outcome, test_case.expected_outcome, + standalone, + vec![test_case.expected_outcome.clone(), PivotOutcome::Absent], + "{}", + test_case.description + ); + assert_eq!( + in_session, + vec![test_case.expected_outcome, PivotOutcome::Absent], "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index c09b783048..547582f4ff 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -17,6 +17,13 @@ pub(crate) struct SessionTestCase { pub(crate) expected_outcomes: &'static [&'static [&'static str]], } +pub(crate) struct SessionFactsTestCase { + pub(crate) description: &'static str, + pub(crate) models: &'static [ModelSpec], + /// `fact_lines` of each model, in request order. + pub(crate) expected_facts: &'static [&'static str], +} + pub(crate) struct UnscheduledTestCase { pub(crate) description: &'static str, pub(crate) models: &'static [ModelSpec], diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs index 5c36906409..0dbac8f6a5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs @@ -4,3 +4,5 @@ pub type Pairs = Vec<(String, String)>; /// Ordered relation shapes in Python dict order. pub type Shapes = Vec<(String, Pairs)>; +/// `(output column, [(resource name, column name)])` per output, in lineage order. +pub type OutputSources = Vec<(String, Vec<(String, String)>)>; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs new file mode 100644 index 0000000000..a2604be080 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs @@ -0,0 +1,28 @@ +//! Whether Python's syntax validation accepts every SQL string, stopping at the first it rejects. + +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::constants::PANIC_DEFERRAL; +use crate::assembly::project::models::SyntaxCheck; + +/// True when every check parses, false at the first Python rejects, Err to let Python decide. +pub fn check_sql_syntax(dialect: &str, checks: &[SyntaxCheck]) -> Result { + catch_compiler_panic(|| all_valid(dialect, checks)).map_err(|kind| { + if kind == PANIC_MESSAGE { + PANIC_DEFERRAL.to_owned() + } else { + kind + } + }) +} + +fn all_valid(dialect: &str, checks: &[SyntaxCheck]) -> Result { + for check in checks { + if !syntax_valid(check, dialect)? { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index d53093ec5a..d35ca0b269 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -1 +1,2 @@ pub mod assemble_project_resources; +pub mod check_sql_syntax; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs index 5f245315d3..e983316a09 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs @@ -6,6 +6,7 @@ use crate::assembly::project::_helpers::deps::audit_deps; use crate::assembly::project::_helpers::syntax::syntax_valid; use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::main::check_sql_syntax::check_sql_syntax; use crate::assembly::project::models::{ AuditFacts, InputRead, Namespace, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, @@ -114,6 +115,17 @@ pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)], dialect: &str) -> .ok() } +pub(crate) fn all_valid(sqls: &[&str]) -> Option { + let checks: Vec = sqls + .iter() + .map(|sql| SyntaxCheck { + sql: (*sql).to_owned(), + placeholders: vec![("x".to_owned(), "1".to_owned())], + }) + .collect(); + check_sql_syntax("generic", &checks).ok() +} + pub(crate) fn namespace(expected: ExpectedNamespace) -> Namespace { let (database, schema, logical_database, logical_schema) = expected; Namespace { diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs index e7a7b02ccc..184a8b3ff3 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs @@ -1,7 +1,10 @@ use crate::assembly::project::models::InputRead; -use crate::assembly::project::tests::helpers::{deps, namespace, seed, seed_reads, source, valid}; +use crate::assembly::project::tests::helpers::{ + all_valid, deps, namespace, seed, seed_reads, source, valid, +}; use crate::assembly::project::tests::test_types::{ - DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxTestCase, keys, + DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxBatchTestCase, + SyntaxTestCase, keys, }; #[test] @@ -287,3 +290,34 @@ fn given_environment_templates_when_resolving_seeds_then_matches_python_values_a assert_eq!(reads, test_case.expected_reads, "{}", test_case.description); } } + +#[test] +fn given_sql_batches_when_checking_syntax_then_stops_at_python_first_rejection_or_defers() { + let test_cases = [ + SyntaxBatchTestCase { + description: "every statement parses", + sqls: &["SELECT @@@x AS a", "SELECT 2"], + expected_valid: Some(true), + }, + SyntaxBatchTestCase { + description: "a later statement Python rejects", + sqls: &["SELECT 1", "SELECT FROM WHERE ("], + expected_valid: Some(false), + }, + SyntaxBatchTestCase { + description: "Python stops at the rejection before a statement native defers", + sqls: &["SELECT FROM WHERE (", "SELECT @@@x\u{e9}"], + expected_valid: Some(false), + }, + SyntaxBatchTestCase { + description: "a statement native defers before any rejection", + sqls: &["SELECT @@@x\u{e9}", "SELECT FROM WHERE ("], + expected_valid: None, + }, + ]; + for test_case in test_cases { + let valid = all_valid(test_case.sqls); + + assert_eq!(valid, test_case.expected_valid, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index f39fb71bfd..ca67d431c2 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -73,3 +73,10 @@ pub(crate) struct EnvironmentTestCase { pub(crate) expected_schema: Option<&'static str>, pub(crate) expected_reads: Vec, } + +pub(crate) struct SyntaxBatchTestCase { + pub(crate) description: &'static str, + pub(crate) sqls: &'static [&'static str], + /// None where native defers to Python. + pub(crate) expected_valid: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 87af8f9b33..59e9dd7a86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -4,14 +4,17 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contract::prove_dynamic_contract; +use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; +use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::provide_deferred_analyses::provide_deferred_analyses; use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use sqlbuild_analysis::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, Deferral, DeferredAnalysis, DynamicFamily, ExpressionShape, - ExpressionShapeRequest, LineageFacts, LineageRow, ModelOutcome, ModelReference, ModelRequest, - PivotOutcome, PivotRequest, SessionOutcome, SessionRequest, SessionStep, + ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, + ModelReference, ModelRequest, PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, + SessionOutcome, SessionRequest, SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; @@ -97,15 +100,16 @@ type ProofRow = ( Option, bool, ); -/// The dialect, relation facts, families by table, SQL and families of one pivot proof. +/// One model's pivot SQL and declared families. +type PivotModelRow = (String, Vec); +/// The dialect, relation facts and families by table every proof reads, and the models. type PivotRequestRow = ( String, Shapes, Shapes, Shapes, Vec<(String, Vec)>, - String, - Vec, + Vec, ); /// `("absent" | "deferred" | "proof", proof)`. type ContractRow = (&'static str, Option); @@ -116,10 +120,16 @@ type FinishRow = (Vec, Shapes, Vec, Vec); #[pyclass(module = "sqlbuild._native")] pub(crate) struct NativeModelAnalysisSession { inner: Option, + finished: Option, failure: Option, } impl NativeModelAnalysisSession { + /// The finished session later compile stages read, once `finish` has succeeded. + pub(crate) fn finished_session(&self) -> Option<&FinishedSession> { + self.finished.as_ref() + } + /// Keep `result`'s value, or end the session and keep its failure for Python's debug log. fn kept(&mut self, result: Result) -> Option { match result { @@ -135,6 +145,15 @@ impl NativeModelAnalysisSession { #[pymethods] impl NativeModelAnalysisSession { + /// Models whose output names and lineage the finished session keeps for later stages. + #[getter] + fn fact_models(&self) -> Vec { + self.finished + .as_ref() + .map(finished_fact_models) + .unwrap_or_default() + } + /// Advance; None means Python must analyse every model, no deferrals means done. fn run(&mut self, py: Python<'_>) -> Option { let mut session: AnalysisSession = self.inner.take()?; @@ -162,9 +181,24 @@ impl NativeModelAnalysisSession { /// Every model's outcome once the session is done, or None to analyse in Python. fn finish(&mut self) -> Option { let session: AnalysisSession = self.inner.take()?; - let outcome: Result = - catch_compiler_panic(|| finish_analysis_session(session).map(finish_row)); - self.kept(outcome) + let outcome = catch_compiler_panic(|| finish_analysis_session(session)); + let (outcome, finished) = self.kept(outcome)?; + self.finished = Some(finished); + Some(finish_row(outcome)) + } + + /// Each model's dynamic pivot proof from the finished session's tables, or None for Python. + fn prove_dynamic_contracts( + &self, + py: Python<'_>, + models: Vec, + ) -> Option> { + let session: &FinishedSession = self.finished.as_ref()?; + let models: Vec = models.into_iter().map(pivot_model).collect(); + match py.compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) { + Ok(outcomes) => Some(outcomes.into_iter().map(contract_row).collect()), + Err(_) => None, + } } /// Why the session handed the analysis back to Python, when it did. @@ -188,6 +222,7 @@ fn start_model_analysis_session( .map_err(compiler_error)?; Ok(session.map(|session| NativeModelAnalysisSession { inner: Some(session), + finished: None, failure: None, })) } @@ -425,28 +460,42 @@ fn outcome_row(outcome: ModelOutcome) -> OutcomeRow { ) } -/// One model's dynamic pivot proof, or `("deferred", None)` for Python's proof. +/// Each model's dynamic pivot proof outside any session, or None for Python's proofs. #[pyfunction] -fn prove_dynamic_column_contract(py: Python<'_>, request: PivotRequestRow) -> ContractRow { - let (dialect, column_types, authoritative_types, column_nullability, by_table, sql, families) = +fn prove_dynamic_column_contracts( + py: Python<'_>, + request: PivotRequestRow, +) -> Option> { + let (dialect, column_types, authoritative_types, column_nullability, by_table, models) = request; - let request = PivotRequest { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: families_by_table(by_table), + let request = PivotBatchRequest { + tables: PivotTables { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: families_by_table(by_table), + }, + models: models.into_iter().map(pivot_model).collect(), + }; + match py.compiler_detach(|| prove_dynamic_contracts(&request)) { + Ok(outcomes) => Some(outcomes.into_iter().map(contract_row).collect()), + Err(_) => None, + } +} + +fn pivot_model(row: PivotModelRow) -> PivotModel { + let (sql, families) = row; + PivotModel { sql, families: families.into_iter().map(dynamic_family).collect(), - }; - py.compiler_detach(|| prove_dynamic_contract(&request)) - .map_or(("deferred", None), contract_row) + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(start_model_analysis_session, module)?)?; module.add_function(wrap_pyfunction!(infer_expression_source_shapes, module)?)?; - module.add_function(wrap_pyfunction!(prove_dynamic_column_contract, module)?)?; + module.add_function(wrap_pyfunction!(prove_dynamic_column_contracts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index d2fbcc1350..48d24695f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; use sqlbuild_analysis::assembly::project::models::{ AuditFacts, InputRead, ModelFacts, Namespace, ProjectRequest, ProjectResources, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, @@ -84,6 +85,23 @@ fn assemble_project_resource_facts( } } +/// Whether every `(sql, placeholders)` parses, or None with the reason Python must check it. +#[pyfunction] +fn check_native_sql_syntax( + py: Python<'_>, + request: (String, Vec<(String, Pairs)>), +) -> (Option, Option) { + let (dialect, rows) = request; + let checks: Vec = rows + .into_iter() + .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) + .collect(); + match py.compiler_detach(|| check_sql_syntax(&dialect, &checks)) { + Ok(valid) => (Some(valid), None), + Err(reason) => (None, Some(reason)), + } +} + fn project_request(request: RequestRow) -> ProjectRequest { let ( dialect, @@ -203,5 +221,6 @@ fn namespace_row(namespace: Namespace) -> NamespaceRow { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; + module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index c9ac573eaa..c01e4e67d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -8,7 +8,9 @@ use sqlbuild_analysis::semantic_checks::models::{ SemanticLocation, }; +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::semantic_checks::session_facts::session_model_facts; use crate::bindings::_helpers::semantic_checks::type_recovery::{LineageInput, lineage_outputs}; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; @@ -33,8 +35,8 @@ type ModelInput = ( String, String, String, - Vec, - Vec, + Option>, + Option>, Vec, Option, ); @@ -66,14 +68,16 @@ type OutcomeRow = ( Vec, ); -/// Recover, explain and reject opt-outs on `catalog`'s analysis pool. +/// Recover, explain and reject opt-outs, reading facts the payload omits from `session`. #[pyfunction] +#[pyo3(signature = (catalog, request, session=None))] fn complete_semantic_checks( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: RequestInput, + session: Option>, ) -> PyResult { - let request = completion_request(request); + let request = completion_request(request, session.as_deref()).map_err(compiler_error)?; let catalog = &catalog.inner; let outcome = py .compiler_detach(|| complete_semantic_diagnostics(&request, catalog)) @@ -106,13 +110,19 @@ fn location_row(location: SemanticLocation) -> LocationRow { ) } -fn completion_request((dialect, diagnostics, models, shapes): RequestInput) -> CompletionRequest { - CompletionRequest { +fn completion_request( + (dialect, diagnostics, models, shapes): RequestInput, + session: Option<&NativeModelAnalysisSession>, +) -> Result { + Ok(CompletionRequest { dialect, diagnostics: diagnostics.into_iter().map(semantic_diagnostic).collect(), - models: models.into_iter().map(completion_model).collect(), + models: models + .into_iter() + .map(|model| completion_model(model, session)) + .collect::>()?, shapes, - } + }) } fn semantic_diagnostic( @@ -134,8 +144,20 @@ fn semantic_diagnostic( fn completion_model( (name, query_sql, authored_sql, inferred_columns, lineage, binding_codes, opt_out): ModelInput, -) -> CompletionModel { - CompletionModel { + session: Option<&NativeModelAnalysisSession>, +) -> Result { + let inferred_columns: Vec = match inferred_columns { + Some(names) => names, + None => session_model_facts(session, &name)? + .columns + .clone() + .unwrap_or_default(), + }; + let lineage: Vec = match lineage { + Some(lineage) => lineage, + None => session_model_facts(session, &name)?.lineage.clone(), + }; + Ok(CompletionModel { name, query_sql, authored_sql, @@ -143,7 +165,7 @@ fn completion_model( lineage: lineage_outputs(lineage), binding_codes, rejected_opt_out_file: opt_out, - } + }) } fn completed_row(diagnostic: CompletedDiagnostic) -> CompletedRow { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 581ba801c2..1565c82749 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod completion; pub(crate) mod metadata; pub(crate) mod registration; +pub(crate) mod session_facts; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs new file mode 100644 index 0000000000..eaf244fa10 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs @@ -0,0 +1,18 @@ +//! Model output names and lineage the semantic stages read from a finished analysis session. + +use sqlbuild_analysis::assembly::analysis_session::main::finished_model_facts::finished_model_facts; +use sqlbuild_analysis::assembly::analysis_session::models::SessionModelFacts; + +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; + +/// The facts the finished `session` kept for model `name`, which the payload left out. +pub(crate) fn session_model_facts<'a>( + session: Option<&'a NativeModelAnalysisSession>, + name: &str, +) -> Result<&'a SessionModelFacts, String> { + let finished = session + .and_then(NativeModelAnalysisSession::finished_session) + .ok_or("a model asked for session facts without a finished session")?; + finished_model_facts(finished, name) + .ok_or_else(|| format!("the finished session kept no facts for model {name}")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 29d48a1739..801971e2a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -10,7 +10,9 @@ use sqlbuild_analysis::semantic_checks::models::{ }; use sqlbuild_analysis::semantic_checks::types::RevisedBinding; +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::semantic_checks::session_facts::session_model_facts; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; @@ -22,7 +24,7 @@ type ModelInput = ( String, Option>, Vec, - Vec, + Option>, Vec<(usize, String, String, bool)>, Vec<(usize, String, String, Option, Option)>, ); @@ -84,14 +86,16 @@ impl SemanticTypeRecovery { } } -/// Plan type recovery on `catalog`'s analysis pool. +/// Plan type recovery, reading names and lineage from `session` for models with no lineage. #[pyfunction] +#[pyo3(signature = (catalog, request, session=None))] fn plan_semantic_type_recovery( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: RequestInput, + session: Option>, ) -> PyResult { - let request = recovery_request(request); + let request = recovery_request(request, session.as_deref()).map_err(compiler_error)?; let catalog = &catalog.inner; let step = py .compiler_detach(|| plan_type_recovery(&request, catalog)) @@ -109,10 +113,16 @@ pub(crate) fn lineage_outputs(lineage: Vec) -> Vec .collect() } -fn recovery_request((dialect, models, diagnostics): RequestInput) -> TypeRecoveryRequest { - TypeRecoveryRequest { +fn recovery_request( + (dialect, models, diagnostics): RequestInput, + session: Option<&NativeModelAnalysisSession>, +) -> Result { + Ok(TypeRecoveryRequest { dialect, - models: models.into_iter().map(recovery_model).collect(), + models: models + .into_iter() + .map(|model| recovery_model(model, session)) + .collect::>()?, diagnostics: diagnostics .into_iter() .map(|(id, code, is_model, resource_name)| DiagnosticOwner { @@ -122,13 +132,21 @@ fn recovery_request((dialect, models, diagnostics): RequestInput) -> TypeRecover resource_name, }) .collect(), - } + }) } fn recovery_model( (name, query_sql, inferred_columns, references, lineage, bindings, raw_bindings): ModelInput, -) -> RecoveryModel { - RecoveryModel { + session: Option<&NativeModelAnalysisSession>, +) -> Result { + let (inferred_columns, lineage) = match lineage { + Some(lineage) => (inferred_columns, lineage), + None => { + let facts = session_model_facts(session, &name)?; + (facts.columns.clone(), facts.lineage.clone()) + } + }; + Ok(RecoveryModel { name, query_sql, inferred_columns, @@ -153,7 +171,7 @@ fn recovery_model( end, }) .collect(), - } + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py index ddf0404a2c..753027e0f3 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -19,6 +19,13 @@ "amount, status", "amount, status, CAST('2026-01-01' AS TIMESTAMP) AS ordered_at" ) +_INTERMEDIATE_PATH: str = "models/intermediate/int_orders.sql" +_INTERMEDIATE: str = ( + 'MODEL (\n description "Doubled order amounts",\n);\n\n' + "SELECT customer_id, amount * 2 AS doubled_amount\n" + 'FROM __ref("stg_orders")\n' +) + _FUNCTION_PATH: str = "functions/sql/scaled_amount.sql" _SCALED_AMOUNT: str = ( "FUNCTION (\n" @@ -60,6 +67,24 @@ def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="semantic-recovered-uses-through-intermediate", + expected_code="B002", + expected_message="Unknown column 'amonut' in raw_orders", + expected_help="did you mean 'amount'?", + expected_notes=( + "raw_orders has: amount, status, customer_id, order_id", + "1 downstream uses of stg_orders.amount were not checked because of this error", + ), + files={ + **staging_files(FAILURE_BASE_STAGING.replace("amount, status", "amonut, status")), + _INTERMEDIATE_PATH: _INTERMEDIATE, + **mart_body_files( + "SELECT customer_id, SUM(doubled_amount) AS total_amount\n" + 'FROM __ref("int_orders")\nGROUP BY customer_id\n' + ), + }, + ), failure_case( name="semantic-qualified-alias-unknown-column", expected_code="B002", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5d194a0bf1..dbcdd70b6c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -349,7 +349,22 @@ def normalize_type( # Native analysis: model analysis session. type _AnalysisDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] +type _PivotContractRow = tuple[ + str, + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None, +] + class NativeModelAnalysisSession: + @property + def fact_models(self) -> list[str]: ... def run( self, ) -> ( @@ -420,35 +435,26 @@ class NativeModelAnalysisSession: ] | None ): ... + def prove_dynamic_contracts( + self, models: list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], / + ) -> list[_PivotContractRow] | None: ... @property def failure(self) -> str | None: ... def start_model_analysis_session( catalog: object, request: tuple[object, ...], / ) -> NativeModelAnalysisSession | None: ... -def prove_dynamic_column_contract( +def prove_dynamic_column_contracts( request: tuple[ str, list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - str, - list[tuple[str, str, str, str, str, str | None]], + list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], ], /, -) -> tuple[ - str, - tuple[ - bool, - list[tuple[str, str | None, str]], - list[tuple[str, str | None]], - list[str], - str | None, - bool, - ] - | None, -]: ... +) -> list[_PivotContractRow] | None: ... def infer_expression_source_shapes( catalog: object, request: tuple[str, bool, list[tuple[str, str]], list[str]], / ) -> tuple[list[tuple[bool, list[tuple[str, str]] | None]], str | None]: ... @@ -477,13 +483,14 @@ def plan_semantic_type_recovery( str, list[str] | None, list[str], - list[tuple[str, list[tuple[str, str]]]], + list[tuple[str, list[tuple[str, str]]]] | None, list[tuple[int, str, str, bool]], list[tuple[int, str, str, int | None, int | None]], ] ], list[tuple[int, str, bool, str | None]], ], + session: NativeModelAnalysisSession | None = None, /, ) -> SemanticTypeRecovery: ... def check_semantic_metadata_rows( @@ -540,14 +547,15 @@ def complete_semantic_checks( str, str, str, - list[str], - list[tuple[str, list[tuple[str, str]]]], + list[str] | None, + list[tuple[str, list[tuple[str, str]]]] | None, list[str], str | None, ] ], list[tuple[str, list[tuple[str, str]]]], ], + session: NativeModelAnalysisSession | None = None, /, ) -> tuple[ str | None, @@ -622,6 +630,9 @@ def plan_compiled_sql_tests( def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. +def check_native_sql_syntax( + request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / +) -> tuple[bool | None, str | None]: ... def assemble_project_resource_facts( request: tuple[ str, diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py deleted file mode 100644 index 803935326f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Prove one model's dynamic column contract natively, outside the analysis session.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session._helpers.deferral_records import record_analysis_deferral -from sqlbuild.compiler.analysis_session._helpers.session_rows import ( - contract_proof, - family_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.constants import CONTRACT_DEFERRED, DEFERRAL_DYNAMIC_PIVOT -from sqlbuild.compiler.analysis_session.types import ContractRow, ShapeRows -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import DynamicColumnContractProof -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def native_dynamic_contract( - *, - query_sql: str, - dialect: str | None, - families: tuple[SchemaDynamicColumnFamily, ...], - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> DynamicColumnContractProof | None: - """Python's proof for declared families, or None (recorded) where Python must prove it.""" - - shapes: tuple[ShapeRows | None, ...] = ( - shape_rows(column_types_by_table), - shape_rows(authoritative_column_types_by_table), - shape_rows(column_nullability_by_table), - ) - row: ContractRow = (CONTRACT_DEFERRED, None) - if all(rows is not None for rows in shapes): - row = _native.prove_dynamic_column_contract( - ( - dialect or "generic", - shapes[0] or [], - shapes[1] or [], - shapes[2] or [], - family_rows(dynamic_families_by_table), - query_sql, - [PythonModelAnalysis.family_row(family) for family in families], - ) - ) - if row[0] == CONTRACT_DEFERRED: - record_analysis_deferral(kind=DEFERRAL_DYNAMIC_PIVOT) - return contract_proof(row[1]) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py new file mode 100644 index 0000000000..560870c25c --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py @@ -0,0 +1,73 @@ +"""Prove dynamic column contracts natively for models outside the analysis session.""" + +from __future__ import annotations + +from typing import Any + +import sqlbuild._native as _native +from sqlbuild.compiler.analysis_session._helpers.deferral_records import record_analysis_deferral +from sqlbuild.compiler.analysis_session._helpers.session_rows import ( + contract_proof, + family_rows, + shape_rows, +) +from sqlbuild.compiler.analysis_session.constants import CONTRACT_DEFERRED, DEFERRAL_DYNAMIC_PIVOT +from sqlbuild.compiler.analysis_session.models import NativePivotTables +from sqlbuild.compiler.analysis_session.types import ContractRow, ShapeRows +from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis +from sqlbuild.compiler.compile.models import DynamicColumnContractProof +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + +type _ModelRow = tuple[str, list[tuple[str, str, str, str, str, str | None]]] + + +def native_dynamic_contracts( + *, + session: Any | None, + tables: NativePivotTables, + models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], +) -> tuple[DynamicColumnContractProof | None, ...]: + """Each model's proof in order, None (recorded) where Python must prove it.""" + + if not models: + return () + rows: list[_ModelRow] = [_model_row(sql=sql, families=families) for sql, families in models] + contracts: list[ContractRow] | None = ( + session.prove_dynamic_contracts(rows) + if session is not None + else _standalone_contracts(tables=tables, rows=rows) + ) + if contracts is None: + record_analysis_deferral(kind=DEFERRAL_DYNAMIC_PIVOT, count=len(models)) + return (None,) * len(models) + record_analysis_deferral( + kind=DEFERRAL_DYNAMIC_PIVOT, + count=sum(kind == CONTRACT_DEFERRED for kind, _ in contracts), + ) + return tuple(contract_proof(proof) for _, proof in contracts) + + +def _model_row(*, sql: str, families: tuple[SchemaDynamicColumnFamily, ...]) -> _ModelRow: + return (sql, list(map(PythonModelAnalysis.family_row, families))) + + +def _standalone_contracts( + *, tables: NativePivotTables, rows: list[_ModelRow] +) -> list[ContractRow] | None: + shapes: tuple[ShapeRows | None, ...] = ( + shape_rows(tables.column_types_by_table), + shape_rows(tables.authoritative_column_types_by_table), + shape_rows(tables.column_nullability_by_table), + ) + if any(shape is None for shape in shapes): + return None + return _native.prove_dynamic_column_contracts( + ( + tables.dialect or "generic", + shapes[0] or [], + shapes[1] or [], + shapes[2] or [], + family_rows(tables.dynamic_families_by_table), + rows, + ) + ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py index c0ec38e947..a03302a8a2 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py @@ -13,15 +13,16 @@ DEFERRAL_NO_COMPACT_ANALYSIS, DEFERRAL_SESSION, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, +) from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.models import ModelSqlAnalysis -def native_model_analyses( - *, request: NativeModelAnalysisRequest -) -> dict[str, ModelSqlAnalysis] | None: - """Each model's analysis by name, or None (recorded) where Python must analyse.""" +def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses | None: + """Each model's analysis by name and the finished session, or None (recorded) for Python.""" catalog: Any = request.inference_profile.binding_catalog if not request.allow_compact_analysis: @@ -31,7 +32,7 @@ def native_model_analyses( record_analysis_deferral(kind=DEFERRAL_NO_CATALOG) return None if not request.model_inputs: - return {} + return NativeModelAnalyses(analyses={}, session=None) python: PythonModelAnalysis = PythonModelAnalysis( model_inputs=request.model_inputs, inference_profile=request.inference_profile, @@ -46,4 +47,7 @@ def native_model_analyses( if session is None: record_analysis_deferral(kind=DEFERRAL_SESSION) return None - return NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses(session) + analyses: dict[str, ModelSqlAnalysis] | None = NativeModelAnalysis( + request=request, python=python, catalog=catalog + ).analyses(session) + return None if analyses is None else NativeModelAnalyses(analyses=analyses, session=session) diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py index f7807824f2..8f72279583 100644 --- a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py +++ b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py @@ -3,13 +3,13 @@ from __future__ import annotations from sqlbuild.compiler.analysis_session._helpers.model_analysis import native_model_analyses -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.compile.models import ModelSqlAnalysis +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, +) -def analyze_native_model_sql( - *, request: NativeModelAnalysisRequest -) -> dict[str, ModelSqlAnalysis] | None: - """Return each model's analysis (and any pivot proof) by name, or None to defer to Python.""" +def analyze_native_model_sql(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses | None: + """Return each model's analysis (and pivot proof) and the session, or None to defer.""" return native_model_analyses(request=request) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py deleted file mode 100644 index 9d47358f42..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Prove a dynamic column contract natively for the preview compiler engine.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.dynamic_contract import native_dynamic_contract -from sqlbuild.compiler.compile.models import DynamicColumnContractProof -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def prove_native_dynamic_contract( - *, - query_sql: str, - dialect: str | None, - families: tuple[SchemaDynamicColumnFamily, ...], - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> DynamicColumnContractProof | None: - """Return Python's proof for declared families, or None to prove it in Python.""" - - return native_dynamic_contract( - query_sql=query_sql, - dialect=dialect, - families=families, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=authoritative_column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py new file mode 100644 index 0000000000..4f01fb1e9f --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py @@ -0,0 +1,21 @@ +"""Prove dynamic column contracts natively for the preview compiler engine.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import native_dynamic_contracts +from sqlbuild.compiler.analysis_session.models import NativePivotTables +from sqlbuild.compiler.compile.models import DynamicColumnContractProof +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +def prove_native_dynamic_contracts( + *, + session: Any | None, + tables: NativePivotTables, + models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], +) -> tuple[DynamicColumnContractProof | None, ...]: + """Python's proof per `(sql, families)` model from the session or `tables`; None for Python.""" + + return native_dynamic_contracts(session=session, tables=tables, models=models) diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index 5323f6d344..2db27e023e 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -3,9 +3,14 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import AnalysisCacheContext, CompileModelInput +from sqlbuild.compiler.compile.models import ( + AnalysisCacheContext, + CompileModelInput, + ModelSqlAnalysis, +) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -25,3 +30,22 @@ class NativeModelAnalysisRequest: analysis_cache: AnalysisCacheContext | None complete_binding_schemas: dict[str, dict[str, str]] dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] + + +@dataclass(frozen=True, slots=True) +class NativeModelAnalyses: + """Each model's native analysis by name, and the finished session that produced them.""" + + analyses: dict[str, ModelSqlAnalysis] + session: Any | None + + +@dataclass(frozen=True, slots=True) +class NativePivotTables: + """The relation facts Python's dynamic pivot proofs read, for proofs outside a session.""" + + dialect: str | None + column_types_by_table: dict[str, dict[str, str]] + authoritative_column_types_by_table: dict[str, dict[str, str]] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py new file mode 100644 index 0000000000..9ed2ea2cad --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py @@ -0,0 +1,50 @@ +"""The SQL and declared families Python's dynamic pivot proof reads for each model.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_placeholders +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import CompileModelInput, ModelSqlAnalysis +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +def model_pivot_sql(*, query_sql: str, placeholders: dict[str, str] | None) -> str: + """Return the analysis SQL with placeholder defaults, as the pivot proof parses it.""" + + if not placeholders: + return query_sql + return substitute_placeholder_defaults(query_sql=query_sql, placeholders=placeholders) + + +def model_dynamic_families(model_input: CompileModelInput) -> tuple[SchemaDynamicColumnFamily, ...]: + """Return the dynamic column families the model's schema declares.""" + + if model_input.schema_entry is None: + return () + return model_input.schema_entry.dynamic_columns + + +def standalone_pivot_models( + *, model_inputs: tuple[CompileModelInput, ...], analyses: Mapping[str, ModelSqlAnalysis] +) -> dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]]: + """Return `(sql, families)` by model index for each pivot model no analysis covered.""" + + models: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = {} + for index, model_input in enumerate(model_inputs): + families: tuple[SchemaDynamicColumnFamily, ...] = model_dynamic_families(model_input) + if not families or model_input.model_file.file_path.stem in analyses: + continue + query_sql: str = cursor_intrinsics_analysis_sql( + sql=model_input.query_sql, + cursor_type=model_input.config.values.get("cursor_type"), + ) + models[index] = ( + model_pivot_sql(query_sql=query_sql, placeholders=model_placeholders(model_input)), + families, + ) + return models diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 87bf84e5ab..c30fde0930 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -23,7 +23,6 @@ record_analysis_cache_metrics, write_model_analyses, ) -from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( NativeCompactAnalysis, analyze_columns_and_lineage_with_polyglot, @@ -34,6 +33,10 @@ from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( analyze_dynamic_column_contract, ) +from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import ( + model_dynamic_families, + model_pivot_sql, +) from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( model_placeholders as _model_placeholders, ) @@ -86,7 +89,7 @@ analyze_model_sql_by_engine, dynamic_column_contract_by_engine, expression_source_shapes_by_engine, - project_resources_by_engine, + project_facts_by_engine, ) from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( assemble_sql_tests_by_engine, @@ -166,7 +169,11 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.models import ( + NativeModelFacts, + NativeProjectFacts, + NativeProjectResources, +) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.models import ScopeIndex @@ -325,6 +332,7 @@ def assemble_compiled_project( else None ) model_sql_analysis_by_name: dict[str, _ModelSqlAnalysis] = {} + native_session: Any | None = None if sql_analysis_enabled: with record_compile_timing("model_analysis_ms"): python_analysis: partial[dict[str, _ModelSqlAnalysis]] = partial( @@ -350,20 +358,22 @@ def assemble_compiled_project( analysis_cache=analysis_cache, complete_binding_schemas=complete_binding_schemas, ) - model_sql_analysis_by_name = analyze_model_sql_by_engine( + model_sql_analysis_by_name, native_session = analyze_model_sql_by_engine( python_analysis=python_analysis, dynamic_families_by_table=dynamic_families_by_table, ) - native: NativeProjectResources | None = project_resources_by_engine( + native: NativeProjectFacts | None = project_facts_by_engine( inputs=inputs, dialect=profile.sql_analysis_dialect, analysis_model_names=analysis_model_names, - analysis_succeeded=frozenset( - name - for name, analysis in model_sql_analysis_by_name.items() - if analysis.polyglot_analysis.analysis_succeeded - ), + analyses=model_sql_analysis_by_name, + session=native_session, + column_types_by_table=column_types_by_table, + authoritative_column_types_by_table=complete_binding_schemas, + column_nullability_by_table=column_nullability_by_table, + dynamic_families_by_table=dynamic_families_by_table, ) + resources: NativeProjectResources | None = native.resources if native else None scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -416,7 +426,7 @@ def assemble_compiled_project( sql_validation_enabled=( analysis_model_names is None or _model_name(model_input) in analysis_model_names ), - native_deps=native.model_deps[index] if native else None, + native_model=native.models[index] if native else None, column_nullability_by_table=column_nullability_by_table, column_types_by_table=column_types_by_table, dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, @@ -431,7 +441,7 @@ def assemble_compiled_project( source_input=source_input, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, - native_entry=native.source_entries[index] if native else None, + native_entry=resources.source_entries[index] if resources else None, ) for index, source_input in enumerate(inputs.source_inputs) ), @@ -441,7 +451,7 @@ def assemble_compiled_project( defaults=inputs.project_config.defaults, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, - native_destination=native.seed_destinations[index] if native else None, + native_destination=resources.seed_destinations[index] if resources else None, ) for index, seed_input in enumerate(inputs.seed_inputs) ), @@ -449,14 +459,14 @@ def assemble_compiled_project( _assemble_compiled_function( function_input=function_input, seed_names=seed_names, - native_deps=native.function_deps[index] if native else None, + native_deps=resources.function_deps[index] if resources else None, ) for index, function_input in enumerate(inputs.sql_function_inputs) ), audits=tuple( _assemble_compiled_audit( audit_input=audit_input, - native_scope_deps=native.audit_scope_deps[index] if native else None, + native_scope_deps=resources.audit_scope_deps[index] if resources else None, ) for index, audit_input in enumerate(inputs.audit_inputs) ), @@ -493,6 +503,7 @@ def assemble_compiled_project( project=project, profile=profile, resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, + native_session=native_session, binding_results={ name: analysis.polyglot_analysis.binding_diagnostics for name, analysis in model_sql_analysis_by_name.items() @@ -519,7 +530,7 @@ def _assemble_compiled_model( model_input: CompileModelInput, sql_analysis_enabled: bool, sql_validation_enabled: bool = True, - native_deps: tuple[CompiledObjectKey, ...] | None = None, + native_model: NativeModelFacts | None = None, column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, column_types_by_table: dict[str, dict[str, str]] | None = None, dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, @@ -528,6 +539,7 @@ def _assemble_compiled_model( allow_compact_analysis: bool = False, ) -> CompiledModel: model_name: str = model_input.model_file.file_path.stem + native_deps: tuple[CompiledObjectKey, ...] | None = native_model.deps if native_model else None syntax_validated: bool = native_deps is not None profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() analysis_query_sql: str = cursor_intrinsics_analysis_sql( @@ -595,22 +607,12 @@ def _assemble_compiled_model( ) dynamic_column_contract: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( sql_analysis=sql_analysis, + native_proof=native_model.dynamic_contract if native_model else None, python_proof=partial( analyze_dynamic_column_contract, - query_sql=( - substitute_placeholder_defaults( - query_sql=analysis_query_sql, - placeholders=placeholders, - ) - if placeholders - else analysis_query_sql - ), + query_sql=model_pivot_sql(query_sql=analysis_query_sql, placeholders=placeholders), dialect=profile.sql_analysis_dialect, - families=( - model_input.schema_entry.dynamic_columns - if model_input.schema_entry is not None - else () - ), + families=model_dynamic_families(model_input), column_types_by_table=column_types_by_table or {}, authoritative_column_types_by_table=( dynamic_contract_analysis_inputs.authoritative_column_types_by_table diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index 521430a59e..eb225ef54b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -5,6 +5,7 @@ import re from collections import Counter from dataclasses import replace +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( @@ -44,6 +45,7 @@ def complete_semantic_diagnostics( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool = True, + native_session: Any | None = None, ) -> CompiledProject: """Recover type facts before metadata checks, then explain retained root diagnostics.""" if native_stage_enabled(NativeStage.SEMANTIC_CHECKS): @@ -52,6 +54,7 @@ def complete_semantic_diagnostics( profile=profile, binding_results=binding_results, resource_sql_analysis=resource_sql_analysis, + session=native_session, ) if native_project is not None: return native_project diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py index 020aa3568e..6db1cd78b5 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py @@ -8,6 +8,7 @@ from pathlib import Path from sqlbuild.compiler.compile._helpers.analysis.validation import ( + hook_sql_statements, validate_hook_sql_syntax, validate_sql_syntax, ) @@ -31,6 +32,11 @@ SQL_ANALYSIS_CONFIG_KEY, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.project_assembly.main._check_native_sql_syntax import ( + check_native_sql_syntax, +) from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.errors.setting_help.main.setting_note import setting_note @@ -66,6 +72,24 @@ def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLo or not any(request.config.values.get(key) is False for key in _OPT_OUT_KEYS) ): return None + if not _parses(request): + return None + return _opt_out_location(request) + + +def _parses(request: SqlAnalysisOptOutRequest) -> bool: + """Whether the model query and every hook pass Python's syntax validation.""" + + if native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): + statements: list[str] = [request.query_sql] + for hook_name in _HOOK_KEYS: + statements.extend(hook_sql_statements(request.config.values.get(hook_name))) + native: bool | None = check_native_sql_syntax( + checks=tuple((statement, request.placeholders) for statement in statements), + dialect=None, + ) + if native is not None: + return native model_name: str = request.model_file.file_path.stem try: validate_sql_syntax( @@ -83,8 +107,8 @@ def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLo placeholders=request.placeholders, ) except CompileInputError: - return None - return _opt_out_location(request) + return False + return True def reject_unneeded_sql_analysis_opt_outs(project: CompiledProject) -> CompiledProject: diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 2c1f1bc21d..9130989292 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -3,6 +3,7 @@ from __future__ import annotations from functools import partial +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( @@ -11,10 +12,15 @@ from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, ) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contract import ( - prove_native_dynamic_contract, +from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( + prove_native_dynamic_contracts, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, + NativePivotTables, +) +from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import standalone_pivot_models from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_syntax_checks from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, @@ -26,32 +32,78 @@ ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( assemble_native_project_resources, ) -from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.models import ( + NativeModelFacts, + NativeProjectFacts, + NativeProjectResources, +) from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily -def project_resources_by_engine( +def project_facts_by_engine( *, inputs: CompileProjectInputs, dialect: str | None, analysis_model_names: frozenset[str] | None, - analysis_succeeded: frozenset[str], -) -> NativeProjectResources | None: - """Return the natively assembled resource facts, or None where Python must assemble them.""" + analyses: dict[str, ModelSqlAnalysis], + session: Any | None, + column_types_by_table: dict[str, dict[str, str]], + authoritative_column_types_by_table: dict[str, dict[str, str]], + column_nullability_by_table: dict[str, dict[str, InferredNullability]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], +) -> NativeProjectFacts | None: + """Return the native project facts, or None where Python must derive all of them.""" if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): return None - return assemble_native_project_resources( + resources: NativeProjectResources | None = assemble_native_project_resources( inputs=inputs, dialect=dialect, syntax_checks=model_syntax_checks( model_inputs=inputs.model_inputs, analysis_model_names=analysis_model_names, - analysis_succeeded=analysis_succeeded, + analysis_succeeded=frozenset( + name + for name, analysis in analyses.items() + if analysis.polyglot_analysis.analysis_succeeded + ), + ), + ) + pivots: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = ( + standalone_pivot_models(model_inputs=inputs.model_inputs, analyses=analyses) + if native_stage_enabled(NativeStage.MODEL_ANALYSIS) + else {} + ) + proofs: dict[int, DynamicColumnContractProof | None] = dict( + zip( + pivots, + prove_native_dynamic_contracts( + session=session, + tables=NativePivotTables( + dialect=dialect, + column_types_by_table=column_types_by_table, + authoritative_column_types_by_table=authoritative_column_types_by_table, + column_nullability_by_table=column_nullability_by_table, + dynamic_families_by_table=dynamic_families_by_table, + ), + models=tuple(pivots.values()), + ), + strict=True, + ) + ) + return NativeProjectFacts( + models=tuple( + NativeModelFacts( + deps=resources.model_deps[index] if resources is not None else None, + dynamic_contract=proofs.get(index), + ) + for index in range(len(inputs.model_inputs)) ), + resources=resources, ) @@ -73,37 +125,30 @@ def analyze_model_sql_by_engine( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> dict[str, ModelSqlAnalysis]: - """Analyze models natively with the Python analysis's arguments, or run the Python analysis.""" +) -> tuple[dict[str, ModelSqlAnalysis], Any | None]: + """Analyze models natively, or with Python; also return the finished native session.""" if native_stage_enabled(NativeStage.MODEL_ANALYSIS): - native_analyses: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + native: NativeModelAnalyses | None = analyze_native_model_sql( request=NativeModelAnalysisRequest( **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table ) ) - if native_analyses is not None: - return native_analyses - return python_analysis() + if native is not None: + return native.analyses, native.session + return python_analysis(), None def dynamic_column_contract_by_engine( *, sql_analysis: ModelSqlAnalysis | None, + native_proof: DynamicColumnContractProof | None, python_proof: partial[DynamicColumnContractProof | None], ) -> DynamicColumnContractProof | None: - """Return the proof native analysis carries or proves alone, or prove it in Python.""" + """Return the proof native analysis carries or proved alone, or prove it in Python.""" if sql_analysis is not None and sql_analysis.dynamic_column_contract is not None: return sql_analysis.dynamic_column_contract - if ( - sql_analysis is None - and python_proof.keywords["families"] - and native_stage_enabled(NativeStage.MODEL_ANALYSIS) - ): - native: DynamicColumnContractProof | None = prove_native_dynamic_contract( - **python_proof.keywords - ) - if native is not None: - return native + if native_proof is not None: + return native_proof return python_proof() diff --git a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py new file mode 100644 index 0000000000..a929ed239f --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py @@ -0,0 +1,24 @@ +"""Check SQL syntax natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.project_assembly._helpers.deferrals import record_project_assembly_deferral + + +def check_native_sql_syntax( + *, checks: tuple[tuple[str, dict[str, str] | None], ...], dialect: str | None +) -> bool | None: + """Whether Python's syntax validation accepts every `(sql, placeholders)`; None to check it.""" + + valid: bool | None + deferral: str | None + valid, deferral = _native.check_native_sql_syntax( + ( + dialect or "generic", + [(sql, list((placeholders or {}).items())) for sql, placeholders in checks], + ) + ) + if valid is None: + _ = record_project_assembly_deferral(kind=deferral or "unknown") + return valid diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py index 8c6631c2cf..7905511275 100644 --- a/src/sqlbuild/compiler/project_assembly/models.py +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -4,7 +4,11 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, + DynamicColumnContractProof, +) from sqlbuild.spec.contracts.models import SourceEntry @@ -17,3 +21,19 @@ class NativeProjectResources: seed_destinations: tuple[CompiledRelationLocation, ...] function_deps: tuple[tuple[CompiledObjectKey, ...], ...] audit_scope_deps: tuple[tuple[CompiledObjectKey, ...], ...] + + +@dataclass(frozen=True, slots=True) +class NativeModelFacts: + """One model's native facts; each None where Python derives it.""" + + deps: tuple[CompiledObjectKey, ...] | None + dynamic_contract: DynamicColumnContractProof | None + + +@dataclass(frozen=True, slots=True) +class NativeProjectFacts: + """Every model's native facts, and the resource facts unless Python must assemble them.""" + + models: tuple[NativeModelFacts, ...] + resources: NativeProjectResources | None diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py index a54b887900..8afb26b9d3 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py @@ -28,7 +28,11 @@ def complete_native_diagnostics( - *, project: CompiledProject, catalog: object + *, + project: CompiledProject, + catalog: object, + session: Any | None, + session_models: frozenset[str], ) -> CompiledProject | None: """Return the recovered, explained and opt-out-checked project, or None to use Python.""" @@ -40,10 +44,15 @@ def complete_native_diagnostics( request: Any = ( project.sql_analysis_dialect, [_diagnostic_payload(item=item, ids=ids) for item in project.diagnostics], - [_model_payload(model=model, lineage=recovers) for model in project.models], + [ + _model_payload(model=model, lineage=recovers, from_session=model.name in session_models) + for model in project.models + ], [(name, list(columns.items())) for name, columns in shapes.items()], ) - deferral, completed, model_bindings, order = _native.complete_semantic_checks(catalog, request) + deferral, completed, model_bindings, order = _native.complete_semantic_checks( + catalog, request, session + ) if deferral is not None: record_semantic_deferral(kind=deferral, site=COMPLETION_DEFERRAL_SITE) return None @@ -98,14 +107,14 @@ def _diagnostic_payload( ) -def _model_payload(*, model: CompiledModel, lineage: bool) -> tuple[Any, ...]: +def _model_payload(*, model: CompiledModel, lineage: bool, from_session: bool) -> tuple[Any, ...]: opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out return ( model.name, model.query_sql, model.authored_sql, - [column.name for column in model.inferred_columns or ()], - lineage_payload(model) if lineage else [], + None if from_session else [column.name for column in model.inferred_columns or ()], + _lineage(model=model, lineage=lineage, from_session=from_session), [item.code for item in model.binding_diagnostics], None if opt_out is None else opt_out.path.name, ) @@ -151,3 +160,15 @@ def _opt_out( notes=(note,), help=help_text, ) + + +def _lineage( + *, model: CompiledModel, lineage: bool, from_session: bool +) -> list[tuple[str, list[tuple[str, str]]]] | None: + """The model's lineage rows; None asks for the session's, and only recovery reads them.""" + + if not lineage: + return [] + if from_session: + return None + return lineage_payload(model) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py index b05811fc97..76d8e40a60 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py @@ -2,11 +2,14 @@ from __future__ import annotations +from typing import Any, cast + from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledModel, CompiledProject, CompilerDiagnostic, + DynamicColumnContractProof, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic @@ -17,6 +20,25 @@ def lineage_payload(model: CompiledModel) -> list[tuple[str, list[tuple[str, str return [_output_payload(output) for output in model.fast_lineage_columns or ()] +def session_fact_models(*, project: CompiledProject, session: Any | None) -> frozenset[str]: + """Models whose output names and lineage are the finished session's, so need no payload.""" + + if session is None: + return frozenset() + kept: frozenset[str] = frozenset(cast(list[str], session.fact_models)) + return frozenset( + model.name + for model in project.models + if model.name in kept and not _output_proven(model.dynamic_column_contract) + ) + + +def _output_proven(proof: DynamicColumnContractProof | None) -> bool: + """Whether a dynamic proof replaced the analysed output names.""" + + return proof is not None and proof.output_proven + + def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[str, str]]]: return ( output.output_column, diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py index 1468d8c319..922ca72ff2 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import replace +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs @@ -10,6 +11,7 @@ from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics +from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types from sqlbuild.compiler.semantic_checks.constants import ( COMPLETION_DEFERRAL_SITE, @@ -24,6 +26,7 @@ def completed_semantic_project( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool, + session: Any | None, ) -> CompiledProject | None: """Python's completed project, or None where any native step hands the stage back.""" @@ -31,8 +34,13 @@ def completed_semantic_project( record_semantic_deferral(kind=NATIVE_SEMANTIC_NO_CATALOG, site=COMPLETION_DEFERRAL_SITE) return None catalog: object = project.binding_catalog.native + session_models: frozenset[str] = session_fact_models(project=project, session=session) recovered: CompiledProject | None = recover_native_output_types( - project=project, binding_results=binding_results, catalog=catalog + project=project, + binding_results=binding_results, + catalog=catalog, + session=session, + session_models=session_models, ) if recovered is None: return None @@ -49,4 +57,6 @@ def completed_semantic_project( return complete_native_diagnostics( project=replace(recovered, diagnostics=(*recovered.diagnostics, *metadata)), catalog=catalog, + session=session, + session_models=session_models, ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py index c4ade28bdb..d433974d15 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py @@ -39,6 +39,8 @@ def recover_native_output_types( project: CompiledProject, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], catalog: object, + session: Any | None, + session_models: frozenset[str], ) -> CompiledProject | None: """Return `recover_output_types`'s project, or None where Python must recover the types.""" @@ -49,7 +51,13 @@ def recover_native_output_types( request: Any = ( project.sql_analysis_dialect, [ - _model_payload(model=model, binding_results=binding_results, ids=ids, raw_ids=raw_ids) + _model_payload( + model=model, + binding_results=binding_results, + ids=ids, + raw_ids=raw_ids, + from_session=model.name in session_models, + ) for model in project.models ], [ @@ -62,7 +70,9 @@ def recover_native_output_types( for item in project.diagnostics ], ) - recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery(catalog, request) + recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery( + catalog, request, session + ) if recovery.status == NATIVE_TYPE_RECOVERY_UNCHANGED: return project if recovery.status == NATIVE_TYPE_RECOVERY_DEFERRED: @@ -120,6 +130,7 @@ def _model_payload( binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], ids: dict[CompilerDiagnostic, int], raw_ids: dict[SqlBindingDiagnostic, int], + from_session: bool, ) -> tuple[Any, ...]: raws: tuple[SqlBindingDiagnostic, ...] = ( binding_results.get(model.name, ()) if model.binding_diagnostics else () @@ -128,10 +139,10 @@ def _model_payload( model.name, model.query_sql, None - if model.inferred_columns is None + if from_session or model.inferred_columns is None else [column.name for column in model.inferred_columns], [reference.ref_name for reference in model.references], - lineage_payload(model), + None if from_session else lineage_payload(model), [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], [(raw_ids[raw], raw.code, raw.message, raw.start, raw.end) for raw in raws], ) diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py index 762329e0a7..ca8f3e64ce 100644 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Any + from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project @@ -14,6 +16,7 @@ def complete_native_semantic_diagnostics( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool, + session: Any | None = None, ) -> CompiledProject | None: """Return the completed project, or None where Python must complete its diagnostics.""" @@ -22,4 +25,5 @@ def complete_native_semantic_diagnostics( profile=profile, binding_results=binding_results, resource_sql_analysis=resource_sql_analysis, + session=session, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index bd27e5c2d5..5df91de88a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,6 +8,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage @@ -81,7 +82,8 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ ) assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) assert isinstance(preview_seams["assemble_native_project_resources"][0], NativeProjectResources) - assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) + assert isinstance(preview_seams["analyze_native_model_sql"][0], NativeModelAnalyses) + assert preview_seams["analyze_native_model_sql"][0].session is not None assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] assert isinstance(planned_artifacts, tuple) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index f3ea3a0aac..25a92f2c22 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -48,4 +48,5 @@ class StandalonePivotProofTestCase: description: str analysed_models: frozenset[str] expected_native_proofs: int + expected_session_proofs: int expected_proven_by_model: dict[str, bool | None] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 47095774c0..9da32670a2 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -25,10 +25,14 @@ from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, ) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contract import ( - prove_native_dynamic_contract, +from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( + prove_native_dynamic_contracts, +) +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, + NativePivotTables, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, ) @@ -301,19 +305,28 @@ def pivot_project_files() -> dict[str, str]: } -def native_pivot_proofs( - *, monkeypatch: pytest.MonkeyPatch -) -> list[DynamicColumnContractProof | None]: - """Record every standalone native pivot proof assembly takes, failing any wheel pivot proof.""" +@dataclass +class NativePivotProofs: + """Every native pivot proof assembly took, and how many a finished session proved.""" + + proofs: list[DynamicColumnContractProof | None] = field(default_factory=list) + session_proofs: int = 0 + + +def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs: + """Record every batch of native pivot proofs assembly takes, failing any wheel pivot proof.""" - proofs: list[DynamicColumnContractProof | None] = [] - native_proof: Callable[..., DynamicColumnContractProof | None] = ( - native_stage_assembly.prove_native_dynamic_contract + recorded_proofs: NativePivotProofs = NativePivotProofs() + native_proofs: Callable[..., tuple[DynamicColumnContractProof | None, ...]] = ( + native_stage_assembly.prove_native_dynamic_contracts ) - def recorded(**keywords: Any) -> DynamicColumnContractProof | None: - proofs.append(native_proof(**keywords)) - return proofs[-1] + def recorded(**keywords: Any) -> tuple[DynamicColumnContractProof | None, ...]: + proofs: tuple[DynamicColumnContractProof | None, ...] = native_proofs(**keywords) + recorded_proofs.proofs.extend(proofs) + proven: int = sum(proof is not None for proof in proofs) + recorded_proofs.session_proofs += proven * (keywords["session"] is not None) + return proofs python_proof: Callable[..., DynamicColumnContractProof | None] = ( project_assembly.analyze_dynamic_column_contract @@ -323,9 +336,9 @@ def wheel_not_expected(**keywords: Any) -> DynamicColumnContractProof | None: assert not keywords["families"], "the Python wheel proved a pivot native should prove" return python_proof(**keywords) - monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contract", recorded) + monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contracts", recorded) monkeypatch.setattr(project_assembly, "analyze_dynamic_column_contract", wheel_not_expected) - return proofs + return recorded_proofs def generated_analysis_files(*, rng: random.Random, model_count: int) -> dict[str, str]: @@ -374,6 +387,7 @@ class AnalysisParity: expression_shapes: int = 0 pivot_proofs: int = 0 standalone_proofs: int = 0 + session_proofs: int = 0 proven_pivots: int = 0 native_enrichments: int = 0 @@ -387,15 +401,17 @@ def compare_analyses( ) -> None: """Assemble `inputs`, analysing every seam call with both engines into `parity`.""" + sessions: list[Any] = [] + def models_by_both( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], - ) -> dict[str, ModelSqlAnalysis]: + ) -> tuple[dict[str, ModelSqlAnalysis], Any]: keywords: dict[str, Any] = {**python_analysis.keywords, "analysis_cache": None} profile: ExpressionInferenceProfile = keywords["inference_profile"] native_catalog: Any = profile.binding_catalog.with_relations({}) - native: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + native_result: NativeModelAnalyses | None = analyze_native_model_sql( request=NativeModelAnalysisRequest( **{ **keywords, @@ -404,6 +420,8 @@ def models_by_both( dynamic_families_by_table=dynamic_families_by_table, ) ) + native: dict[str, ModelSqlAnalysis] | None = getattr(native_result, "analyses", None) + sessions.append(getattr(native_result, "session", None)) python: dict[str, ModelSqlAnalysis] = python_analysis.func(**keywords) parity.analysed_models += len(python) _append(parity, "model analyses", _analysis_views(python), _analysis_views(native)) @@ -413,25 +431,32 @@ def models_by_both( _catalog_view(profile.binding_catalog), _catalog_view(native_catalog), ) - return native or python + return native or python, sessions[-1] def proofs_by_both( *, sql_analysis: ModelSqlAnalysis | None, + native_proof: DynamicColumnContractProof | None, python_proof: partial[DynamicColumnContractProof | None], ) -> DynamicColumnContractProof | None: native: DynamicColumnContractProof | None = getattr( sql_analysis, "dynamic_column_contract", None ) python: DynamicColumnContractProof | None = python_proof() - standalone: DynamicColumnContractProof | None = prove_native_dynamic_contract( - **python_proof.keywords + session: Any = (sessions or [None])[-1] + standalone: DynamicColumnContractProof | None = _batched_proof( + session=None, python_proof=python_proof + ) + in_session: DynamicColumnContractProof | None = _batched_proof( + session=session, python_proof=python_proof ) + parity.session_proofs += in_session is not None and session is not None parity.pivot_proofs += native is not None parity.proven_pivots += native is not None and native.output_proven parity.standalone_proofs += standalone is not None _append(parity, "dynamic pivot proof", python, native or python) _append(parity, "standalone dynamic pivot proof", python, standalone or python) + _append(parity, "session dynamic pivot proof", python, in_session or python) return python def shapes_by_both( @@ -485,13 +510,13 @@ def captured( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], - ) -> dict[str, ModelSqlAnalysis]: + ) -> tuple[dict[str, ModelSqlAnalysis], None]: requests.append( NativeModelAnalysisRequest( **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table ) ) - return python_analysis() + return python_analysis(), None with monkeypatch.context() as patch, suppress(CompileInputError): patch.setattr(project_assembly, "analyze_model_sql_by_engine", captured) @@ -515,6 +540,23 @@ def deferral_kinds(record_dir: Path) -> Counter[str]: return Counter(f"{record['site']}:{record['kind']}" for record in records) +def _batched_proof( + *, session: Any, python_proof: partial[DynamicColumnContractProof | None] +) -> DynamicColumnContractProof | None: + keywords: dict[str, Any] = python_proof.keywords + return prove_native_dynamic_contracts( + session=session, + tables=NativePivotTables( + dialect=keywords["dialect"], + column_types_by_table=keywords["column_types_by_table"], + authoritative_column_types_by_table=keywords["authoritative_column_types_by_table"], + column_nullability_by_table=keywords["column_nullability_by_table"], + dynamic_families_by_table=keywords["dynamic_families_by_table"], + ), + models=((keywords["query_sql"], keywords["families"]),), + )[0] + + def _append(parity: AnalysisParity, name: str, python: object, native: object) -> None: parity.names.append(name) parity.python.append(python) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 4e9527ea64..d7aebb0d9b 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile.models import ( CompiledProject, CompileProjectInputs, - DynamicColumnContractProof, ) from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine @@ -32,6 +31,7 @@ from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( AnalysisParity, FailingProvideSession, + NativePivotProofs, analysis_request, compare_analyses, compile_inputs, @@ -97,6 +97,7 @@ def test_given_generated_projects_when_analysing_natively_then_matches_python( assert parity.expression_shapes >= test_case.expected_minimum_expression_shapes assert parity.pivot_proofs >= test_case.expected_minimum_pivot_proofs assert parity.standalone_proofs >= test_case.expected_minimum_pivot_proofs + assert parity.session_proofs >= test_case.expected_minimum_pivot_proofs assert parity.proven_pivots >= test_case.expected_minimum_proven_pivots assert ( kinds["analysis_session:input_enrichment"] @@ -199,12 +200,24 @@ def test_given_session_failure_after_deferrals_when_analysing_then_records_only_ description="a selection that leaves the pivots out of model analysis", analysed_models=frozenset({"orders_list"}), expected_native_proofs=2, + expected_session_proofs=2, expected_proven_by_model={ "orders_list": None, "status_amounts": True, "status_passthrough": True, }, - ) + ), + StandalonePivotProofTestCase( + description="no model analysis, so the proofs run without a session", + analysed_models=frozenset(), + expected_native_proofs=2, + expected_session_proofs=0, + expected_proven_by_model={ + "orders_list": None, + "status_amounts": True, + "status_passthrough": True, + }, + ), ], ids=lambda case: case.description, ) @@ -215,7 +228,7 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with project_dir=tmp_path / "project", files=pivot_project_files() ) monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) - proofs: list[DynamicColumnContractProof | None] = native_pivot_proofs(monkeypatch=monkeypatch) + recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) project: CompiledProject = assemble_compiled_project( inputs=inputs, @@ -224,13 +237,19 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ) assert ( - all(proof is not None for proof in proofs), - len(proofs), + all(proof is not None for proof in recorded.proofs), + len(recorded.proofs), + recorded.session_proofs, { model.name: getattr(model.dynamic_column_contract, "output_proven", None) for model in project.models }, - ) == (True, test_case.expected_native_proofs, test_case.expected_proven_by_model) + ) == ( + True, + test_case.expected_native_proofs, + test_case.expected_session_proofs, + test_case.expected_proven_by_model, + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index 680d93254c..1e010ed519 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -45,3 +45,13 @@ class WindowsEnvironmentTestCase: environment: dict[str, str] seed_schema: str expected_schema: str + + +@dataclass(frozen=True) +class OptOutTestCase: + """Models opting out of required SQL analysis, and which ones parse so Python rejects them.""" + + description: str + models: dict[str, str] + expected_rejected: dict[str, bool] + expected_python_validations: int diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 05d2a1188c..c8b4ded2df 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -15,6 +15,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly import project as project_module from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild.compiler.compile._helpers.diagnostics import sql_analysis_opt_outs from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -38,6 +39,7 @@ "audit_scope_deps", ) _ASSEMBLY_RECORD_PREFIX: str = "project_assembly:" +_OPT_OUT_VALIDATORS: tuple[str, ...] = ("validate_sql_syntax", "validate_hook_sql_syntax") _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, @@ -199,6 +201,38 @@ def assembly_view(project: CompiledProject) -> object: ) +def rejected_opt_outs( + *, + project_dir: Path, + files: dict[str, str], + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[dict[str, bool], int]: + """Attach `files` with `engine`; return each opt-out's rejection and Python validations.""" + + calls: Counter[str] = Counter() + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + with monkeypatch.context() as patch: + _ = [ + patch.setattr( + sql_analysis_opt_outs, + name, + _counted(name=name, function=getattr(sql_analysis_opt_outs, name), calls=calls), + ) + for name in _OPT_OUT_VALIDATORS + ] + inputs: CompileProjectInputs = project_inputs(project_dir=project_dir, files=files) + return ( + { + model_input.model_file.file_path.stem: ( + model_input.rejected_sql_analysis_opt_out is not None + ) + for model_input in inputs.model_inputs + }, + calls.total(), + ) + + def assembly_deferrals(record_dir: Path) -> Counter[str]: """Count the project assembly `site:kind` deferral records below `record_dir`.""" diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index 5b9ef5783e..e88e498591 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -19,6 +19,7 @@ AssemblyDeferralTestCase, DeferredAssemblyTestCase, GeneratedAssemblyParityTestCase, + OptOutTestCase, WindowsEnvironmentTestCase, ) from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( @@ -29,6 +30,7 @@ project_inputs, python_resource_calls, recorded_assembly, + rejected_opt_outs, seed_yml, windows_environ, ) @@ -37,6 +39,11 @@ 'name = "assembly"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n\n' ) _ORDERS_MODEL: str = 'MODEL (description "Orders");\n\nSELECT 1 AS n\n' +_REQUIRED_ANALYSIS_TOML: str = ( + 'name = "assembly"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[settings]\nrequire_sql_analysis = true\n' +) +_OPT_OUT_HEADER: str = 'MODEL (\n description "Orders",\n sql_analysis false,\n' _SEED_FILES: dict[str, str] = { "seeds/countries.csv": "n\n1\n", "models/orders.sql": _ORDERS_MODEL, @@ -261,3 +268,47 @@ def test_given_windows_environment_when_assembling_natively_then_env_lookups_mat assert python.seeds[0].destination.schema == test_case.expected_schema assert assembly_view(native) == assembly_view(python) assert assembly_deferrals(record_dir) == Counter() + + +@pytest.mark.parametrize( + "test_case", + [ + OptOutTestCase( + description="parseable, unparseable and unparseable-hook opt-outs", + models={ + "models/parses.sql": f"{_OPT_OUT_HEADER});\n\nSELECT 1 AS n\n", + "models/broken_query.sql": f"{_OPT_OUT_HEADER});\n\nSELECT FROM WHERE (\n", + "models/broken_hook.sql": ( + f'{_OPT_OUT_HEADER} post_hooks [inline_sql("SELECT FROM (")],\n);\n\n' + "SELECT 1 AS n\n" + ), + }, + expected_rejected={"broken_hook": False, "broken_query": False, "parses": True}, + expected_python_validations=7, + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejections_match( + test_case: OptOutTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + files: dict[str, str] = {"sqlbuild_project.toml": _REQUIRED_ANALYSIS_TOML, **test_case.models} + + python: tuple[dict[str, bool], int] = rejected_opt_outs( + project_dir=tmp_path / "python", + files=files, + engine=CompilerEngine.PYTHON, + monkeypatch=monkeypatch, + ) + native: tuple[dict[str, bool], int] = rejected_opt_outs( + project_dir=tmp_path / "native", + files=files, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + + assert python == (test_case.expected_rejected, test_case.expected_python_validations) + assert native == (test_case.expected_rejected, 0) + assert assembly_deferrals(record_dir) == Counter() diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py index 31dd072d29..10a8a6c3d3 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py @@ -40,3 +40,18 @@ class GeneratedMetadataParityTestCase: dialects: tuple[str, ...] expected_minimum_native: int expected_minimum_families: dict[str, int] + + +@dataclass(frozen=True) +class SessionCompletionTestCase: + """Seeded preview compiles whose completion reads analysed models from the native session.""" + + description: str + corpus: str + seed: int + count: int + model_count: int + expected_session_models: int + expected_payload_models: int + expected_proven_outputs: int + expected_minimum_codes: dict[str, int] diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 97ed5ccb0d..8cbb77988d 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,6 +13,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project +import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( @@ -26,10 +27,14 @@ from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics +from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( complete_native_semantic_diagnostics, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( + generated_analysis_files, +) _RELATIONS: dict[str, tuple[str, ...]] = { '__source("raw_orders")': ( @@ -81,6 +86,7 @@ class SemanticInputs: profile: ExpressionInferenceProfile binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]] resource_sql_analysis: bool + native_session: Any | None = None def _typo(rng: random.Random, column: str) -> str: @@ -173,9 +179,13 @@ def generated_semantic_files( def captured_semantic_inputs( - *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch + *, + project_dir: Path, + files: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + engine: str = "python", ) -> SemanticInputs: - """Compile with the Python engine and keep the semantic completion stage's inputs.""" + """Compile with `engine` and keep the semantic completion stage's inputs.""" for relative_path, contents in files.items(): path: Path = project_dir / relative_path @@ -188,7 +198,7 @@ def capture(**arguments: Any) -> CompiledProject: return complete_semantic_diagnostics(**arguments) with monkeypatch.context() as patch: - patch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + patch.setenv(COMPILER_ENGINE_ENV_VAR, engine) patch.setattr(assembly_project, "complete_semantic_diagnostics", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), @@ -248,9 +258,51 @@ def native_completion(inputs: SemanticInputs) -> CompiledProject | None: profile=inputs.profile, binding_results=inputs.binding_results, resource_sql_analysis=inputs.resource_sql_analysis, + session=inputs.native_session, ) +def session_corpus_files(*, corpus: str, rng: random.Random, model_count: int) -> dict[str, str]: + """One generated project from the failing-semantics or the analysis-session corpus.""" + + generators: dict[str, Callable[[], dict[str, str]]] = { + "semantic": lambda: generated_semantic_files( + rng=rng, model_count=model_count, require_analysis=False + ), + "analysis": lambda: generated_analysis_files(rng=rng, model_count=model_count), + } + return generators[corpus]() + + +def proven_output_count(project: CompiledProject) -> int: + """How many models a dynamic pivot proof gave their output names.""" + + return sum( + model.dynamic_column_contract is not None and model.dynamic_column_contract.output_proven + for model in project.models + ) + + +def without_session(inputs: SemanticInputs) -> SemanticInputs: + """The same stage inputs with every model's output names and lineage in the payload.""" + + return replace(inputs, native_session=None) + + +def record_session_models(*, monkeypatch: pytest.MonkeyPatch) -> list[frozenset[str]]: + """Keep each completion's models whose output names and lineage the session supplies.""" + + selections: list[frozenset[str]] = [] + + def recorded(**arguments: Any) -> frozenset[str]: + selected: frozenset[str] = session_fact_models(**arguments) + selections.append(selected) + return selected + + monkeypatch.setattr(semantic_stage, "session_fact_models", recorded) + return selections + + def completion_view(project: CompiledProject) -> tuple[object, ...]: """Everything the stage decides: diagnostics and each model's binding and type facts.""" diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 96d98e0bbb..b597aa098c 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -14,6 +14,7 @@ from tests.integration.src.sqlbuild.compiler.semantic_checks._test_types import ( DeferredSemanticTestCase, GeneratedSemanticParityTestCase, + SessionCompletionTestCase, ) from tests.integration.src.sqlbuild.compiler.semantic_checks.helpers import ( SemanticInputs, @@ -24,9 +25,13 @@ generated_semantic_files, native_completion, note_kinds, + proven_output_count, python_completion, record_native_statuses, + record_session_models, + session_corpus_files, with_dialect, + without_session, ) _NOTE_KINDS: tuple[str, ...] = ("downstream output uses", "downstream uses of", " has: ") @@ -154,5 +159,83 @@ def test_given_deferred_projects_when_completing_then_python_completes_and_recor assert deferral_records(record_dir) == test_case.expected_kinds +@pytest.mark.parametrize( + "test_case", + [ + SessionCompletionTestCase( + description="failing semantics corpus", + corpus="semantic", + seed=20261010, + count=6, + model_count=12, + expected_session_models=44, + expected_payload_models=28, + expected_proven_outputs=0, + expected_minimum_codes={"B002": 1, "downstream output uses": 1}, + ), + SessionCompletionTestCase( + description="analysis session corpus with pivots, stars and set operations", + corpus="analysis", + seed=20261011, + count=4, + model_count=16, + expected_session_models=67, + expected_payload_models=29, + expected_proven_outputs=11, + expected_minimum_codes={"B002": 1}, + ), + ], + ids=lambda case: case.description, +) +def test_given_preview_compiles_when_completing_from_the_session_then_matches_payload_and_python( + test_case: SessionCompletionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + captured: list[SemanticInputs] = [ + captured_semantic_inputs( + project_dir=tmp_path / f"project_{index}", + files=session_corpus_files( + corpus=test_case.corpus, rng=rng, model_count=test_case.model_count + ), + monkeypatch=monkeypatch, + engine="native-preview", + ) + for index in range(test_case.count) + ] + selections: list[frozenset[str]] = record_session_models(monkeypatch=monkeypatch) + codes: Counter[str] = Counter() + python_views: list[tuple[object, ...]] = [] + payload_views: list[tuple[object, ...]] = [] + session_views: list[tuple[object, ...]] = [] + for inputs in captured: + python: CompiledProject = python_completion(inputs=inputs, monkeypatch=monkeypatch) + codes.update(item.code for item in python.diagnostics) + codes.update(note_kinds(python, _NOTE_KINDS)) + python_views.append(completion_view(python)) + payload_views.append(completion_view(completed_natively(without_session(inputs)))) + session_views.append(completion_view(completed_natively(inputs))) + session_models: int = sum(len(selected) for selected in selections) + model_count: int = sum(len(inputs.project.models) for inputs in captured) + + assert all(inputs.native_session is not None for inputs in captured) + assert session_views == python_views + assert payload_views == python_views + assert ( + session_models, + model_count - session_models, + sum(proven_output_count(inputs.project) for inputs in captured), + ) == ( + test_case.expected_session_models, + test_case.expected_payload_models, + test_case.expected_proven_outputs, + ) + assert { + code: min(codes[code], minimum) + for code, minimum in test_case.expected_minimum_codes.items() + } == test_case.expected_minimum_codes + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py index a49bf87317..6fd230b52e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py @@ -10,6 +10,7 @@ class DynamicColumnContractDispatchTestCase: description: str sql_analysis: ModelSqlAnalysis | None + native_proof: DynamicColumnContractProof | None dialect: str families: tuple[SchemaDynamicColumnFamily, ...] expected_proof: DynamicColumnContractProof | None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py index 92ff874493..8addc9e694 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py @@ -44,6 +44,7 @@ ( DynamicColumnContractDispatchTestCase( description="native analysis proof replaces the python proof", + native_proof=None, sql_analysis=ModelSqlAnalysis( polyglot_analysis=_ANALYSIS, placeholders=None, @@ -55,6 +56,7 @@ ), DynamicColumnContractDispatchTestCase( description="analysis without a proof falls back to the python proof", + native_proof=None, sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), dialect="bigquery", families=(_FAMILY,), @@ -62,13 +64,23 @@ ), DynamicColumnContractDispatchTestCase( description="unanalyzed model falls back to the python proof", + native_proof=None, sql_analysis=None, dialect="bigquery", families=(_FAMILY,), expected_proof=_PYTHON_UNSUPPORTED_PROOF, ), + DynamicColumnContractDispatchTestCase( + description="a batched native proof answers an unanalyzed model", + native_proof=_NATIVE_PROOF, + sql_analysis=None, + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_NATIVE_PROOF, + ), DynamicColumnContractDispatchTestCase( description="model without dynamic families has no proof", + native_proof=None, sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), dialect="snowflake", families=(), @@ -82,6 +94,7 @@ def test_given_model_analysis_when_proving_dynamic_contract_then_native_proof_ta ) -> None: proof: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( sql_analysis=test_case.sql_analysis, + native_proof=test_case.native_proof, python_proof=partial( analyze_dynamic_column_contract, query_sql="SELECT * FROM order_amounts",