From 702b6be6aabfccfefec450ebb7c9447469f0bcf3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:25:59 +0100 Subject: [PATCH 1/3] feat(native): batch dynamic pivot proofs and opt-out syntax checks through the native session The finished analysis session is kept on CompiledProject.native_session and proves pivot models outside it from its own tables; without a session one batched standalone call sends the tables once. The per-model native proof path is deleted. SQL-analysis opt-out syntax checks use the native checker under the preview engine. Rule hosts drop the session before pickling. --- .../_helpers/dynamic_pivot.rs | 46 ++++++++- .../analysis_session/_helpers/session.rs | 26 ++++- .../main/finish_analysis_session.rs | 8 +- .../src/assembly/analysis_session/main/mod.rs | 3 +- .../main/prove_dynamic_contract.rs | 26 ----- .../main/prove_dynamic_contracts.rs | 17 ++++ .../main/prove_finished_dynamic_contracts.rs | 13 +++ .../src/assembly/analysis_session/models.rs | 26 ++++- .../analysis_session/tests/helpers.rs | 50 +++++++--- .../tests/test_analysis_session.rs | 20 +++- .../assembly/project/main/check_sql_syntax.rs | 28 ++++++ .../src/assembly/project/main/mod.rs | 1 + .../src/assembly/project/tests/helpers.rs | 12 +++ .../project/tests/test_project_assembly.rs | 38 ++++++- .../src/assembly/project/tests/test_types.rs | 7 ++ .../_helpers/analysis_session/session.rs | 78 ++++++++++----- .../_helpers/project_assembly/resources.rs | 19 ++++ src/sqlbuild/_native.pyi | 37 ++++--- .../_helpers/dynamic_contract.py | 52 ---------- .../_helpers/dynamic_contracts.py | 73 ++++++++++++++ .../_helpers/model_analysis.py | 18 ++-- .../main/_analyze_native_model_sql.py | 12 +-- .../main/_prove_native_dynamic_contract.py | 31 ------ .../main/_prove_native_dynamic_contracts.py | 21 ++++ .../compiler/analysis_session/models.py | 26 ++++- .../_helpers/analysis/pivot_requests.py | 50 ++++++++++ .../compile/_helpers/assembly/project.py | 60 +++++------ .../diagnostics/sql_analysis_opt_outs.py | 28 +++++- .../_helpers/native_stages/assembly.py | 99 ++++++++++++++----- .../main/_check_native_sql_syntax.py | 24 +++++ .../compiler/project_assembly/models.py | 22 ++++- .../_helpers/engine/custom_rules.py | 1 + .../compile/test_native_analysis_seams.py | 5 +- .../compiler/analysis_session/_test_types.py | 9 ++ .../compiler/analysis_session/helpers.py | 85 +++++++++++----- .../test_native_model_analysis.py | 67 +++++++++++-- .../compiler/project_assembly/_test_types.py | 10 ++ .../compiler/project_assembly/helpers.py | 34 +++++++ .../test_native_project_assembly.py | 51 ++++++++++ .../host_payload/test_host_payload_facts.py | 2 +- .../_helpers/native_stages/_test_types.py | 1 + .../test_dynamic_column_contract_by_engine.py | 13 +++ 42 files changed, 966 insertions(+), 283 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py create mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py create mode 100644 src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py create mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py create mode 100644 src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index 78f0b3bdba..88a2523cb7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -1,8 +1,11 @@ //! Python's `analyze_dynamic_column_contract` over the crate's parse of the model SQL. use std::collections::{HashMap, HashSet}; +use std::panic::{AssertUnwindSafe, catch_unwind}; use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; +use rayon::ThreadPool; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde_json::{Map, Value, json}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; @@ -17,7 +20,7 @@ use crate::assembly::analysis_session::constants::{ UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ - ColumnFact, ContractProof, DynamicFamily, PivotOutcome, + ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -33,6 +36,47 @@ pub(crate) struct PivotFacts<'a> { pub(crate) families_by_table: &'a [(String, Vec)], } +/// Each model's proof in order; a model whose proof panics is deferred to Python. +pub(crate) fn pivot_outcomes(models: &[PivotModel], tables: &PivotTables) -> Vec { + let facts = pivot_facts(tables); + models + .iter() + .map(|model| guarded_outcome(model, &facts)) + .collect() +} + +/// Each model's proof in order on `pool`; a model whose proof panics is deferred to Python. +pub(crate) fn pooled_pivot_outcomes( + pool: &ThreadPool, + models: &[PivotModel], + tables: &PivotTables, +) -> Vec { + let facts = pivot_facts(tables); + pool.install(|| { + models + .par_iter() + .map(|model| guarded_outcome(model, &facts)) + .collect() + }) +} + +fn pivot_facts(tables: &PivotTables) -> PivotFacts<'_> { + PivotFacts { + dialect: &tables.dialect, + column_types: &tables.column_types, + authoritative_types: &tables.authoritative_types, + column_nullability: &tables.column_nullability, + families_by_table: &tables.families_by_table, + } +} + +fn guarded_outcome(model: &PivotModel, facts: &PivotFacts<'_>) -> PivotOutcome { + catch_unwind(AssertUnwindSafe(|| { + pivot_outcome(&model.sql, &model.families, facts) + })) + .unwrap_or(PivotOutcome::Deferred) +} + /// One CTE body by folded name; None for a CTE with column aliases. type CteMap<'a> = HashMap>; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index cf945f7c71..fa9ebdc21f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -1,7 +1,9 @@ //! The session's phases: Python's uncached model analysis, wave by wave. use std::collections::HashMap; +use std::sync::Arc; +use rayon::ThreadPool; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; @@ -21,8 +23,9 @@ use crate::assembly::analysis_session::constants::{ DEFERRAL_ANALYSIS, DEFERRAL_ENRICHMENT, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ - AnalysisSession, Awaiting, ColumnFact, Deferral, DeferredAnalysis, LineageFacts, ModelAnalysis, - ModelOutcome, ModelRequest, Phase, PivotOutcome, SessionOutcome, SessionRequest, SessionStep, + AnalysisSession, Awaiting, ColumnFact, Deferral, DeferredAnalysis, FinishedSession, + LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, + SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::ProjectCatalog; @@ -148,23 +151,36 @@ impl AnalysisSession { } /// Every model's outcome and the binding catalog changes Python records. - pub(crate) fn finish(self) -> Result { + pub(crate) fn finish(self) -> Result<(SessionOutcome, FinishedSession), String> { if !matches!(self.phase, Phase::Done) { return Err("the session has not finished".to_owned()); } let dynamic_contracts: Vec = self.dynamic_contracts()?; + let pool: Result, String> = self.catalog.native.analysis_pool(); let models: Vec = self .outcomes .into_iter() .collect::>() .ok_or("the session left a model unanalysed")?; let (schema_additions, analysis_names) = self.catalog.into_changes(); - Ok(SessionOutcome { + let request: SessionRequest = self.request; + let finished = FinishedSession { + tables: PivotTables { + dialect: request.dialect, + column_types: request.column_types, + authoritative_types: request.complete_schemas, + column_nullability: request.column_nullability, + families_by_table: request.dynamic_families_by_table, + }, + pool, + }; + let outcome = SessionOutcome { models, schema_additions, analysis_names, dynamic_contracts, - }) + }; + Ok((outcome, finished)) } /// Python's dynamic pivot proof of each model over the relation facts before analysis. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs index f7a4fec8b3..11185a1ad6 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finish_analysis_session.rs @@ -1,8 +1,10 @@ //! Collect a finished native model analysis session's outcomes. -use crate::assembly::analysis_session::models::{AnalysisSession, SessionOutcome}; +use crate::assembly::analysis_session::models::{AnalysisSession, FinishedSession, SessionOutcome}; -/// Every model's outcome and the binding catalog changes Python records. -pub fn finish_analysis_session(session: AnalysisSession) -> Result { +/// Every model's outcome and catalog changes, and the session kept for later pivot proofs. +pub fn finish_analysis_session( + session: AnalysisSession, +) -> Result<(SessionOutcome, FinishedSession), String> { session.finish() } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index 6c5394355c..f50a22f2ca 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -1,6 +1,7 @@ pub mod expression_shapes; pub mod finish_analysis_session; -pub mod prove_dynamic_contract; +pub mod prove_dynamic_contracts; +pub mod prove_finished_dynamic_contracts; pub mod provide_deferred_analyses; pub mod run_analysis_session; pub mod start_analysis_session; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs deleted file mode 100644 index 96759642d4..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contract.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Python's `analyze_dynamic_column_contract` for one model outside the analysis session. - -use crate::assembly::analysis_session::_helpers::dynamic_pivot::{PivotFacts, pivot_outcome}; -use crate::assembly::analysis_session::constants::PIVOT_WORKER_STACK_BYTES; -use crate::assembly::analysis_session::models::{PivotOutcome, PivotRequest}; - -/// The model's proof, or a deferral where Python's result is not reproduced exactly. -pub fn prove_dynamic_contract(request: &PivotRequest) -> Result { - let facts = PivotFacts { - dialect: &request.dialect, - column_types: &request.column_types, - authoritative_types: &request.authoritative_types, - column_nullability: &request.column_nullability, - families_by_table: &request.families_by_table, - }; - std::thread::scope(|scope| { - std::thread::Builder::new() - .stack_size(PIVOT_WORKER_STACK_BYTES) - .spawn_scoped(scope, || { - pivot_outcome(&request.sql, &request.families, &facts) - }) - .map_err(|error| error.to_string())? - .join() - .map_err(|_| "the native dynamic pivot proof panicked".to_owned()) - }) -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs new file mode 100644 index 0000000000..bbbb199898 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_dynamic_contracts.rs @@ -0,0 +1,17 @@ +//! Python's `analyze_dynamic_column_contract` for models no analysis session analysed. + +use crate::assembly::analysis_session::_helpers::dynamic_pivot::pivot_outcomes; +use crate::assembly::analysis_session::constants::PIVOT_WORKER_STACK_BYTES; +use crate::assembly::analysis_session::models::{PivotBatchRequest, PivotOutcome}; + +/// Each model's proof in order; a model whose proof panics is deferred to Python. +pub fn prove_dynamic_contracts(request: &PivotBatchRequest) -> Result, String> { + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(PIVOT_WORKER_STACK_BYTES) + .spawn_scoped(scope, || pivot_outcomes(&request.models, &request.tables)) + .map_err(|error| error.to_string())? + .join() + .map_err(|_| "the native dynamic pivot proofs panicked".to_owned()) + }) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs new file mode 100644 index 0000000000..28a9f22db8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/prove_finished_dynamic_contracts.rs @@ -0,0 +1,13 @@ +//! Python's `analyze_dynamic_column_contract` for models outside a finished analysis session. + +use crate::assembly::analysis_session::_helpers::dynamic_pivot::pooled_pivot_outcomes; +use crate::assembly::analysis_session::models::{FinishedSession, PivotModel, PivotOutcome}; + +/// Each model's proof in order from the session's own tables; a panicking proof is deferred. +pub fn prove_finished_dynamic_contracts( + session: &FinishedSession, + models: &[PivotModel], +) -> Result, String> { + let pool = session.pool.as_ref().map_err(Clone::clone)?; + Ok(pooled_pivot_outcomes(pool, models, &session.tables)) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index d10d56c0ad..565ccf3544 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -1,6 +1,9 @@ //! Plain-data request, deferrals and outcomes of the native model analysis session. use std::collections::HashMap; +use std::sync::Arc; + +use rayon::ThreadPool; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; @@ -61,18 +64,37 @@ pub struct ContractProof { pub bare_dynamic_pivot: bool, } -/// One model's dynamic pivot proof inputs, for a model the session does not analyse. +/// The relation facts every dynamic pivot proof reads: Python's tables before analysis. #[derive(Debug, Clone, Default)] -pub struct PivotRequest { +pub struct PivotTables { pub dialect: String, pub column_types: Shapes, pub authoritative_types: Shapes, pub column_nullability: Shapes, pub families_by_table: Vec<(String, Vec)>, +} + +/// One model's pivot SQL and declared families. +#[derive(Debug, Clone, Default)] +pub struct PivotModel { pub sql: String, pub families: Vec, } +/// Dynamic pivot proofs for models no session analysed. +#[derive(Debug, Clone, Default)] +pub struct PivotBatchRequest { + pub tables: PivotTables, + pub models: Vec, +} + +/// A finished session's pivot tables and analysis pool, kept to prove models outside it. +#[derive(Debug)] +pub struct FinishedSession { + pub(crate) tables: PivotTables, + pub(crate) pool: Result, String>, +} + /// A model's dynamic pivot proof: none declared, left to Python, or proven natively. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PivotOutcome { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 1f274fdfc1..0970aafe27 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -5,8 +5,8 @@ use crate::assembly::analysis_session::main::run_analysis_session::run_analysis_ use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, Deferral, DeferredAnalysis, DynamicFamily, - ModelOutcome, ModelReference, ModelRequest, PivotOutcome, PivotRequest, SessionRequest, - SessionStep, + FinishedSession, ModelOutcome, ModelReference, ModelRequest, PivotBatchRequest, PivotModel, + PivotOutcome, PivotTables, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::tests::test_types::ModelSpec; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -131,7 +131,7 @@ pub(crate) fn completed( }) }) .collect(); - let outcome = finish_analysis_session(session).expect("the session finished"); + let (outcome, _) = finish_analysis_session(session).expect("the session finished"); (steps, outcome.models) } @@ -215,12 +215,12 @@ pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec, -) -> PivotRequest { +) -> PivotBatchRequest { let raw_orders: Shapes = shapes(&[( "raw_orders", &[ @@ -237,17 +237,35 @@ pub(crate) fn pivot_request( }) .into_iter() .collect(); - PivotRequest { - dialect: dialect.to_owned(), - column_types: [raw_orders.clone(), upstream].concat(), - authoritative_types: raw_orders, - column_nullability: shapes(&[("raw_orders", &[("customer_id", "non_null")])]), - families_by_table: vec![( - "status_amounts".to_owned(), - vec![amounts("status", "amount", "MAX")], - )], - sql: sql.to_owned(), - families, + PivotBatchRequest { + tables: PivotTables { + dialect: dialect.to_owned(), + column_types: [raw_orders.clone(), upstream].concat(), + authoritative_types: raw_orders, + column_nullability: shapes(&[("raw_orders", &[("customer_id", "non_null")])]), + families_by_table: vec![( + "status_amounts".to_owned(), + vec![amounts("status", "amount", "MAX")], + )], + }, + models: vec![ + PivotModel { + sql: sql.to_owned(), + families, + }, + PivotModel { + sql: "SELECT 1".to_owned(), + families: Vec::new(), + }, + ], + } +} + +/// A finished session holding `tables` and an analysis pool. +pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { + FinishedSession { + tables, + pool: catalog("duckdb", &Vec::new()).analysis_pool(), } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs index c56625f012..ef2f2315bf 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs @@ -1,8 +1,10 @@ -use crate::assembly::analysis_session::main::prove_dynamic_contract::prove_dynamic_contract; +use crate::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use crate::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::PivotOutcome; use crate::assembly::analysis_session::tests::helpers::{ - catalog, failed, model_requests, orders_request, pivot_request, proven, session_lines, + catalog, failed, finished_session, model_requests, orders_request, pivot_request, proven, + session_lines, }; use crate::assembly::analysis_session::tests::test_types::{ PivotTestCase, SessionTestCase, UnscheduledTestCase, @@ -281,11 +283,21 @@ fn given_dynamic_pivots_when_proving_then_matches_python_or_defers() { ]; for test_case in test_cases { let request = pivot_request(test_case.dialect, test_case.sql, test_case.family); + let session = finished_session(request.tables.clone()); - let outcome = prove_dynamic_contract(&request).expect("the proof runs"); + let standalone = prove_dynamic_contracts(&request).expect("the proofs run"); + let in_session = + prove_finished_dynamic_contracts(&session, &request.models).expect("the proofs run"); assert_eq!( - outcome, test_case.expected_outcome, + standalone, + vec![test_case.expected_outcome.clone(), PivotOutcome::Absent], + "{}", + test_case.description + ); + assert_eq!( + in_session, + vec![test_case.expected_outcome, PivotOutcome::Absent], "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs new file mode 100644 index 0000000000..a2604be080 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/check_sql_syntax.rs @@ -0,0 +1,28 @@ +//! Whether Python's syntax validation accepts every SQL string, stopping at the first it rejects. + +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::assembly::project::_helpers::syntax::syntax_valid; +use crate::assembly::project::constants::PANIC_DEFERRAL; +use crate::assembly::project::models::SyntaxCheck; + +/// True when every check parses, false at the first Python rejects, Err to let Python decide. +pub fn check_sql_syntax(dialect: &str, checks: &[SyntaxCheck]) -> Result { + catch_compiler_panic(|| all_valid(dialect, checks)).map_err(|kind| { + if kind == PANIC_MESSAGE { + PANIC_DEFERRAL.to_owned() + } else { + kind + } + }) +} + +fn all_valid(dialect: &str, checks: &[SyntaxCheck]) -> Result { + for check in checks { + if !syntax_valid(check, dialect)? { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index d53093ec5a..d35ca0b269 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -1 +1,2 @@ pub mod assemble_project_resources; +pub mod check_sql_syntax; diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs index 5f245315d3..e983316a09 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/helpers.rs @@ -6,6 +6,7 @@ use crate::assembly::project::_helpers::deps::audit_deps; use crate::assembly::project::_helpers::syntax::syntax_valid; use crate::assembly::project::_helpers::targets::{managed_source, seed_target}; use crate::assembly::project::_helpers::templates::TemplateInputs; +use crate::assembly::project::main::check_sql_syntax::check_sql_syntax; use crate::assembly::project::models::{ AuditFacts, InputRead, Namespace, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, @@ -114,6 +115,17 @@ pub(crate) fn valid(sql: &str, placeholders: &[(&str, &str)], dialect: &str) -> .ok() } +pub(crate) fn all_valid(sqls: &[&str]) -> Option { + let checks: Vec = sqls + .iter() + .map(|sql| SyntaxCheck { + sql: (*sql).to_owned(), + placeholders: vec![("x".to_owned(), "1".to_owned())], + }) + .collect(); + check_sql_syntax("generic", &checks).ok() +} + pub(crate) fn namespace(expected: ExpectedNamespace) -> Namespace { let (database, schema, logical_database, logical_schema) = expected; Namespace { diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs index e7a7b02ccc..184a8b3ff3 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_project_assembly.rs @@ -1,7 +1,10 @@ use crate::assembly::project::models::InputRead; -use crate::assembly::project::tests::helpers::{deps, namespace, seed, seed_reads, source, valid}; +use crate::assembly::project::tests::helpers::{ + all_valid, deps, namespace, seed, seed_reads, source, valid, +}; use crate::assembly::project::tests::test_types::{ - DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxTestCase, keys, + DepsTestCase, EnvironmentTestCase, SeedTestCase, SourceTestCase, SyntaxBatchTestCase, + SyntaxTestCase, keys, }; #[test] @@ -287,3 +290,34 @@ fn given_environment_templates_when_resolving_seeds_then_matches_python_values_a assert_eq!(reads, test_case.expected_reads, "{}", test_case.description); } } + +#[test] +fn given_sql_batches_when_checking_syntax_then_stops_at_python_first_rejection_or_defers() { + let test_cases = [ + SyntaxBatchTestCase { + description: "every statement parses", + sqls: &["SELECT @@@x AS a", "SELECT 2"], + expected_valid: Some(true), + }, + SyntaxBatchTestCase { + description: "a later statement Python rejects", + sqls: &["SELECT 1", "SELECT FROM WHERE ("], + expected_valid: Some(false), + }, + SyntaxBatchTestCase { + description: "Python stops at the rejection before a statement native defers", + sqls: &["SELECT FROM WHERE (", "SELECT @@@x\u{e9}"], + expected_valid: Some(false), + }, + SyntaxBatchTestCase { + description: "a statement native defers before any rejection", + sqls: &["SELECT @@@x\u{e9}", "SELECT FROM WHERE ("], + expected_valid: None, + }, + ]; + for test_case in test_cases { + let valid = all_valid(test_case.sqls); + + assert_eq!(valid, test_case.expected_valid, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index f39fb71bfd..ca67d431c2 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -73,3 +73,10 @@ pub(crate) struct EnvironmentTestCase { pub(crate) expected_schema: Option<&'static str>, pub(crate) expected_reads: Vec, } + +pub(crate) struct SyntaxBatchTestCase { + pub(crate) description: &'static str, + pub(crate) sqls: &'static [&'static str], + /// None where native defers to Python. + pub(crate) expected_valid: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 87af8f9b33..284a8545d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -4,14 +4,16 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contract::prove_dynamic_contract; +use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::provide_deferred_analyses::provide_deferred_analyses; use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use sqlbuild_analysis::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, Deferral, DeferredAnalysis, DynamicFamily, ExpressionShape, - ExpressionShapeRequest, LineageFacts, LineageRow, ModelOutcome, ModelReference, ModelRequest, - PivotOutcome, PivotRequest, SessionOutcome, SessionRequest, SessionStep, + ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, + ModelReference, ModelRequest, PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, + SessionOutcome, SessionRequest, SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; @@ -97,15 +99,16 @@ type ProofRow = ( Option, bool, ); -/// The dialect, relation facts, families by table, SQL and families of one pivot proof. +/// One model's pivot SQL and declared families. +type PivotModelRow = (String, Vec); +/// The dialect, relation facts and families by table every proof reads, and the models. type PivotRequestRow = ( String, Shapes, Shapes, Shapes, Vec<(String, Vec)>, - String, - Vec, + Vec, ); /// `("absent" | "deferred" | "proof", proof)`. type ContractRow = (&'static str, Option); @@ -116,6 +119,7 @@ type FinishRow = (Vec, Shapes, Vec, Vec); #[pyclass(module = "sqlbuild._native")] pub(crate) struct NativeModelAnalysisSession { inner: Option, + finished: Option, failure: Option, } @@ -162,9 +166,24 @@ impl NativeModelAnalysisSession { /// Every model's outcome once the session is done, or None to analyse in Python. fn finish(&mut self) -> Option { let session: AnalysisSession = self.inner.take()?; - let outcome: Result = - catch_compiler_panic(|| finish_analysis_session(session).map(finish_row)); - self.kept(outcome) + let outcome = catch_compiler_panic(|| finish_analysis_session(session)); + let (outcome, finished) = self.kept(outcome)?; + self.finished = Some(finished); + Some(finish_row(outcome)) + } + + /// Each model's dynamic pivot proof from the finished session's tables, or None for Python. + fn prove_dynamic_contracts( + &self, + py: Python<'_>, + models: Vec, + ) -> Option> { + let session: &FinishedSession = self.finished.as_ref()?; + let models: Vec = models.into_iter().map(pivot_model).collect(); + match py.compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) { + Ok(outcomes) => Some(outcomes.into_iter().map(contract_row).collect()), + Err(_) => None, + } } /// Why the session handed the analysis back to Python, when it did. @@ -188,6 +207,7 @@ fn start_model_analysis_session( .map_err(compiler_error)?; Ok(session.map(|session| NativeModelAnalysisSession { inner: Some(session), + finished: None, failure: None, })) } @@ -425,28 +445,42 @@ fn outcome_row(outcome: ModelOutcome) -> OutcomeRow { ) } -/// One model's dynamic pivot proof, or `("deferred", None)` for Python's proof. +/// Each model's dynamic pivot proof outside any session, or None for Python's proofs. #[pyfunction] -fn prove_dynamic_column_contract(py: Python<'_>, request: PivotRequestRow) -> ContractRow { - let (dialect, column_types, authoritative_types, column_nullability, by_table, sql, families) = +fn prove_dynamic_column_contracts( + py: Python<'_>, + request: PivotRequestRow, +) -> Option> { + let (dialect, column_types, authoritative_types, column_nullability, by_table, models) = request; - let request = PivotRequest { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: families_by_table(by_table), + let request = PivotBatchRequest { + tables: PivotTables { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: families_by_table(by_table), + }, + models: models.into_iter().map(pivot_model).collect(), + }; + match py.compiler_detach(|| prove_dynamic_contracts(&request)) { + Ok(outcomes) => Some(outcomes.into_iter().map(contract_row).collect()), + Err(_) => None, + } +} + +fn pivot_model(row: PivotModelRow) -> PivotModel { + let (sql, families) = row; + PivotModel { sql, families: families.into_iter().map(dynamic_family).collect(), - }; - py.compiler_detach(|| prove_dynamic_contract(&request)) - .map_or(("deferred", None), contract_row) + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(start_model_analysis_session, module)?)?; module.add_function(wrap_pyfunction!(infer_expression_source_shapes, module)?)?; - module.add_function(wrap_pyfunction!(prove_dynamic_column_contract, module)?)?; + module.add_function(wrap_pyfunction!(prove_dynamic_column_contracts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index d2fbcc1350..48d24695f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; +use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; use sqlbuild_analysis::assembly::project::models::{ AuditFacts, InputRead, ModelFacts, Namespace, ProjectRequest, ProjectResources, Reference, SeedDefaults, SeedFacts, SourceFacts, SyntaxCheck, TargetNamespace, Variable, @@ -84,6 +85,23 @@ fn assemble_project_resource_facts( } } +/// Whether every `(sql, placeholders)` parses, or None with the reason Python must check it. +#[pyfunction] +fn check_native_sql_syntax( + py: Python<'_>, + request: (String, Vec<(String, Pairs)>), +) -> (Option, Option) { + let (dialect, rows) = request; + let checks: Vec = rows + .into_iter() + .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) + .collect(); + match py.compiler_detach(|| check_sql_syntax(&dialect, &checks)) { + Ok(valid) => (Some(valid), None), + Err(reason) => (None, Some(reason)), + } +} + fn project_request(request: RequestRow) -> ProjectRequest { let ( dialect, @@ -203,5 +221,6 @@ fn namespace_row(namespace: Namespace) -> NamespaceRow { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; + module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5d194a0bf1..8bfeb38c3b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -349,6 +349,19 @@ def normalize_type( # Native analysis: model analysis session. type _AnalysisDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] +type _PivotContractRow = tuple[ + str, + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None, +] + class NativeModelAnalysisSession: def run( self, @@ -420,35 +433,26 @@ class NativeModelAnalysisSession: ] | None ): ... + def prove_dynamic_contracts( + self, models: list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], / + ) -> list[_PivotContractRow] | None: ... @property def failure(self) -> str | None: ... def start_model_analysis_session( catalog: object, request: tuple[object, ...], / ) -> NativeModelAnalysisSession | None: ... -def prove_dynamic_column_contract( +def prove_dynamic_column_contracts( request: tuple[ str, list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - str, - list[tuple[str, str, str, str, str, str | None]], + list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], ], /, -) -> tuple[ - str, - tuple[ - bool, - list[tuple[str, str | None, str]], - list[tuple[str, str | None]], - list[str], - str | None, - bool, - ] - | None, -]: ... +) -> list[_PivotContractRow] | None: ... def infer_expression_source_shapes( catalog: object, request: tuple[str, bool, list[tuple[str, str]], list[str]], / ) -> tuple[list[tuple[bool, list[tuple[str, str]] | None]], str | None]: ... @@ -622,6 +626,9 @@ def plan_compiled_sql_tests( def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. +def check_native_sql_syntax( + request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / +) -> tuple[bool | None, str | None]: ... def assemble_project_resource_facts( request: tuple[ str, diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py deleted file mode 100644 index 803935326f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contract.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Prove one model's dynamic column contract natively, outside the analysis session.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session._helpers.deferral_records import record_analysis_deferral -from sqlbuild.compiler.analysis_session._helpers.session_rows import ( - contract_proof, - family_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.constants import CONTRACT_DEFERRED, DEFERRAL_DYNAMIC_PIVOT -from sqlbuild.compiler.analysis_session.types import ContractRow, ShapeRows -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import DynamicColumnContractProof -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def native_dynamic_contract( - *, - query_sql: str, - dialect: str | None, - families: tuple[SchemaDynamicColumnFamily, ...], - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> DynamicColumnContractProof | None: - """Python's proof for declared families, or None (recorded) where Python must prove it.""" - - shapes: tuple[ShapeRows | None, ...] = ( - shape_rows(column_types_by_table), - shape_rows(authoritative_column_types_by_table), - shape_rows(column_nullability_by_table), - ) - row: ContractRow = (CONTRACT_DEFERRED, None) - if all(rows is not None for rows in shapes): - row = _native.prove_dynamic_column_contract( - ( - dialect or "generic", - shapes[0] or [], - shapes[1] or [], - shapes[2] or [], - family_rows(dynamic_families_by_table), - query_sql, - [PythonModelAnalysis.family_row(family) for family in families], - ) - ) - if row[0] == CONTRACT_DEFERRED: - record_analysis_deferral(kind=DEFERRAL_DYNAMIC_PIVOT) - return contract_proof(row[1]) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py new file mode 100644 index 0000000000..560870c25c --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py @@ -0,0 +1,73 @@ +"""Prove dynamic column contracts natively for models outside the analysis session.""" + +from __future__ import annotations + +from typing import Any + +import sqlbuild._native as _native +from sqlbuild.compiler.analysis_session._helpers.deferral_records import record_analysis_deferral +from sqlbuild.compiler.analysis_session._helpers.session_rows import ( + contract_proof, + family_rows, + shape_rows, +) +from sqlbuild.compiler.analysis_session.constants import CONTRACT_DEFERRED, DEFERRAL_DYNAMIC_PIVOT +from sqlbuild.compiler.analysis_session.models import NativePivotTables +from sqlbuild.compiler.analysis_session.types import ContractRow, ShapeRows +from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis +from sqlbuild.compiler.compile.models import DynamicColumnContractProof +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + +type _ModelRow = tuple[str, list[tuple[str, str, str, str, str, str | None]]] + + +def native_dynamic_contracts( + *, + session: Any | None, + tables: NativePivotTables, + models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], +) -> tuple[DynamicColumnContractProof | None, ...]: + """Each model's proof in order, None (recorded) where Python must prove it.""" + + if not models: + return () + rows: list[_ModelRow] = [_model_row(sql=sql, families=families) for sql, families in models] + contracts: list[ContractRow] | None = ( + session.prove_dynamic_contracts(rows) + if session is not None + else _standalone_contracts(tables=tables, rows=rows) + ) + if contracts is None: + record_analysis_deferral(kind=DEFERRAL_DYNAMIC_PIVOT, count=len(models)) + return (None,) * len(models) + record_analysis_deferral( + kind=DEFERRAL_DYNAMIC_PIVOT, + count=sum(kind == CONTRACT_DEFERRED for kind, _ in contracts), + ) + return tuple(contract_proof(proof) for _, proof in contracts) + + +def _model_row(*, sql: str, families: tuple[SchemaDynamicColumnFamily, ...]) -> _ModelRow: + return (sql, list(map(PythonModelAnalysis.family_row, families))) + + +def _standalone_contracts( + *, tables: NativePivotTables, rows: list[_ModelRow] +) -> list[ContractRow] | None: + shapes: tuple[ShapeRows | None, ...] = ( + shape_rows(tables.column_types_by_table), + shape_rows(tables.authoritative_column_types_by_table), + shape_rows(tables.column_nullability_by_table), + ) + if any(shape is None for shape in shapes): + return None + return _native.prove_dynamic_column_contracts( + ( + tables.dialect or "generic", + shapes[0] or [], + shapes[1] or [], + shapes[2] or [], + family_rows(tables.dynamic_families_by_table), + rows, + ) + ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py index c0ec38e947..a03302a8a2 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py @@ -13,15 +13,16 @@ DEFERRAL_NO_COMPACT_ANALYSIS, DEFERRAL_SESSION, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, +) from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.models import ModelSqlAnalysis -def native_model_analyses( - *, request: NativeModelAnalysisRequest -) -> dict[str, ModelSqlAnalysis] | None: - """Each model's analysis by name, or None (recorded) where Python must analyse.""" +def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses | None: + """Each model's analysis by name and the finished session, or None (recorded) for Python.""" catalog: Any = request.inference_profile.binding_catalog if not request.allow_compact_analysis: @@ -31,7 +32,7 @@ def native_model_analyses( record_analysis_deferral(kind=DEFERRAL_NO_CATALOG) return None if not request.model_inputs: - return {} + return NativeModelAnalyses(analyses={}, session=None) python: PythonModelAnalysis = PythonModelAnalysis( model_inputs=request.model_inputs, inference_profile=request.inference_profile, @@ -46,4 +47,7 @@ def native_model_analyses( if session is None: record_analysis_deferral(kind=DEFERRAL_SESSION) return None - return NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses(session) + analyses: dict[str, ModelSqlAnalysis] | None = NativeModelAnalysis( + request=request, python=python, catalog=catalog + ).analyses(session) + return None if analyses is None else NativeModelAnalyses(analyses=analyses, session=session) diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py index f7807824f2..8f72279583 100644 --- a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py +++ b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py @@ -3,13 +3,13 @@ from __future__ import annotations from sqlbuild.compiler.analysis_session._helpers.model_analysis import native_model_analyses -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.compile.models import ModelSqlAnalysis +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, +) -def analyze_native_model_sql( - *, request: NativeModelAnalysisRequest -) -> dict[str, ModelSqlAnalysis] | None: - """Return each model's analysis (and any pivot proof) by name, or None to defer to Python.""" +def analyze_native_model_sql(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses | None: + """Return each model's analysis (and pivot proof) and the session, or None to defer.""" return native_model_analyses(request=request) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py deleted file mode 100644 index 9d47358f42..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contract.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Prove a dynamic column contract natively for the preview compiler engine.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.dynamic_contract import native_dynamic_contract -from sqlbuild.compiler.compile.models import DynamicColumnContractProof -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def prove_native_dynamic_contract( - *, - query_sql: str, - dialect: str | None, - families: tuple[SchemaDynamicColumnFamily, ...], - column_types_by_table: dict[str, dict[str, str]], - authoritative_column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> DynamicColumnContractProof | None: - """Return Python's proof for declared families, or None to prove it in Python.""" - - return native_dynamic_contract( - query_sql=query_sql, - dialect=dialect, - families=families, - column_types_by_table=column_types_by_table, - authoritative_column_types_by_table=authoritative_column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - dynamic_families_by_table=dynamic_families_by_table, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py new file mode 100644 index 0000000000..4f01fb1e9f --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py @@ -0,0 +1,21 @@ +"""Prove dynamic column contracts natively for the preview compiler engine.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import native_dynamic_contracts +from sqlbuild.compiler.analysis_session.models import NativePivotTables +from sqlbuild.compiler.compile.models import DynamicColumnContractProof +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +def prove_native_dynamic_contracts( + *, + session: Any | None, + tables: NativePivotTables, + models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], +) -> tuple[DynamicColumnContractProof | None, ...]: + """Python's proof per `(sql, families)` model from the session or `tables`; None for Python.""" + + return native_dynamic_contracts(session=session, tables=tables, models=models) diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index 5323f6d344..2db27e023e 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -3,9 +3,14 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import AnalysisCacheContext, CompileModelInput +from sqlbuild.compiler.compile.models import ( + AnalysisCacheContext, + CompileModelInput, + ModelSqlAnalysis, +) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -25,3 +30,22 @@ class NativeModelAnalysisRequest: analysis_cache: AnalysisCacheContext | None complete_binding_schemas: dict[str, dict[str, str]] dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] + + +@dataclass(frozen=True, slots=True) +class NativeModelAnalyses: + """Each model's native analysis by name, and the finished session that produced them.""" + + analyses: dict[str, ModelSqlAnalysis] + session: Any | None + + +@dataclass(frozen=True, slots=True) +class NativePivotTables: + """The relation facts Python's dynamic pivot proofs read, for proofs outside a session.""" + + dialect: str | None + column_types_by_table: dict[str, dict[str, str]] + authoritative_column_types_by_table: dict[str, dict[str, str]] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py new file mode 100644 index 0000000000..9ed2ea2cad --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py @@ -0,0 +1,50 @@ +"""The SQL and declared families Python's dynamic pivot proof reads for each model.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults +from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_placeholders +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import CompileModelInput, ModelSqlAnalysis +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +def model_pivot_sql(*, query_sql: str, placeholders: dict[str, str] | None) -> str: + """Return the analysis SQL with placeholder defaults, as the pivot proof parses it.""" + + if not placeholders: + return query_sql + return substitute_placeholder_defaults(query_sql=query_sql, placeholders=placeholders) + + +def model_dynamic_families(model_input: CompileModelInput) -> tuple[SchemaDynamicColumnFamily, ...]: + """Return the dynamic column families the model's schema declares.""" + + if model_input.schema_entry is None: + return () + return model_input.schema_entry.dynamic_columns + + +def standalone_pivot_models( + *, model_inputs: tuple[CompileModelInput, ...], analyses: Mapping[str, ModelSqlAnalysis] +) -> dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]]: + """Return `(sql, families)` by model index for each pivot model no analysis covered.""" + + models: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = {} + for index, model_input in enumerate(model_inputs): + families: tuple[SchemaDynamicColumnFamily, ...] = model_dynamic_families(model_input) + if not families or model_input.model_file.file_path.stem in analyses: + continue + query_sql: str = cursor_intrinsics_analysis_sql( + sql=model_input.query_sql, + cursor_type=model_input.config.values.get("cursor_type"), + ) + models[index] = ( + model_pivot_sql(query_sql=query_sql, placeholders=model_placeholders(model_input)), + families, + ) + return models diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 87bf84e5ab..b05203f0f0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -23,7 +23,6 @@ record_analysis_cache_metrics, write_model_analyses, ) -from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( NativeCompactAnalysis, analyze_columns_and_lineage_with_polyglot, @@ -34,6 +33,10 @@ from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( analyze_dynamic_column_contract, ) +from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import ( + model_dynamic_families, + model_pivot_sql, +) from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import ( model_placeholders as _model_placeholders, ) @@ -86,7 +89,7 @@ analyze_model_sql_by_engine, dynamic_column_contract_by_engine, expression_source_shapes_by_engine, - project_resources_by_engine, + project_facts_by_engine, ) from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( assemble_sql_tests_by_engine, @@ -166,7 +169,11 @@ from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.models import ( + NativeModelFacts, + NativeProjectFacts, + NativeProjectResources, +) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.models import ScopeIndex @@ -325,6 +332,7 @@ def assemble_compiled_project( else None ) model_sql_analysis_by_name: dict[str, _ModelSqlAnalysis] = {} + native_session: Any | None = None if sql_analysis_enabled: with record_compile_timing("model_analysis_ms"): python_analysis: partial[dict[str, _ModelSqlAnalysis]] = partial( @@ -350,20 +358,22 @@ def assemble_compiled_project( analysis_cache=analysis_cache, complete_binding_schemas=complete_binding_schemas, ) - model_sql_analysis_by_name = analyze_model_sql_by_engine( + model_sql_analysis_by_name, native_session = analyze_model_sql_by_engine( python_analysis=python_analysis, dynamic_families_by_table=dynamic_families_by_table, ) - native: NativeProjectResources | None = project_resources_by_engine( + native: NativeProjectFacts | None = project_facts_by_engine( inputs=inputs, dialect=profile.sql_analysis_dialect, analysis_model_names=analysis_model_names, - analysis_succeeded=frozenset( - name - for name, analysis in model_sql_analysis_by_name.items() - if analysis.polyglot_analysis.analysis_succeeded - ), + analyses=model_sql_analysis_by_name, + session=native_session, + column_types_by_table=column_types_by_table, + authoritative_column_types_by_table=complete_binding_schemas, + column_nullability_by_table=column_nullability_by_table, + dynamic_families_by_table=dynamic_families_by_table, ) + resources: NativeProjectResources | None = native.resources if native else None scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -416,7 +426,7 @@ def assemble_compiled_project( sql_validation_enabled=( analysis_model_names is None or _model_name(model_input) in analysis_model_names ), - native_deps=native.model_deps[index] if native else None, + native_model=native.models[index] if native else None, column_nullability_by_table=column_nullability_by_table, column_types_by_table=column_types_by_table, dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, @@ -431,7 +441,7 @@ def assemble_compiled_project( source_input=source_input, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, - native_entry=native.source_entries[index] if native else None, + native_entry=resources.source_entries[index] if resources else None, ) for index, source_input in enumerate(inputs.source_inputs) ), @@ -441,7 +451,7 @@ def assemble_compiled_project( defaults=inputs.project_config.defaults, target_config=inputs.effective_target, effective_vars=inputs.effective_vars, - native_destination=native.seed_destinations[index] if native else None, + native_destination=resources.seed_destinations[index] if resources else None, ) for index, seed_input in enumerate(inputs.seed_inputs) ), @@ -449,14 +459,14 @@ def assemble_compiled_project( _assemble_compiled_function( function_input=function_input, seed_names=seed_names, - native_deps=native.function_deps[index] if native else None, + native_deps=resources.function_deps[index] if resources else None, ) for index, function_input in enumerate(inputs.sql_function_inputs) ), audits=tuple( _assemble_compiled_audit( audit_input=audit_input, - native_scope_deps=native.audit_scope_deps[index] if native else None, + native_scope_deps=resources.audit_scope_deps[index] if resources else None, ) for index, audit_input in enumerate(inputs.audit_inputs) ), @@ -488,6 +498,7 @@ def assemble_compiled_project( ), external_sql_reference_resolver=inputs.external_sql_reference_resolver, scope_index=scope_index, + native_session=native_session, ) return complete_semantic_diagnostics( project=project, @@ -519,7 +530,7 @@ def _assemble_compiled_model( model_input: CompileModelInput, sql_analysis_enabled: bool, sql_validation_enabled: bool = True, - native_deps: tuple[CompiledObjectKey, ...] | None = None, + native_model: NativeModelFacts | None = None, column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, column_types_by_table: dict[str, dict[str, str]] | None = None, dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, @@ -528,6 +539,7 @@ def _assemble_compiled_model( allow_compact_analysis: bool = False, ) -> CompiledModel: model_name: str = model_input.model_file.file_path.stem + native_deps: tuple[CompiledObjectKey, ...] | None = native_model.deps if native_model else None syntax_validated: bool = native_deps is not None profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() analysis_query_sql: str = cursor_intrinsics_analysis_sql( @@ -595,22 +607,12 @@ def _assemble_compiled_model( ) dynamic_column_contract: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( sql_analysis=sql_analysis, + native_proof=native_model.dynamic_contract if native_model else None, python_proof=partial( analyze_dynamic_column_contract, - query_sql=( - substitute_placeholder_defaults( - query_sql=analysis_query_sql, - placeholders=placeholders, - ) - if placeholders - else analysis_query_sql - ), + query_sql=model_pivot_sql(query_sql=analysis_query_sql, placeholders=placeholders), dialect=profile.sql_analysis_dialect, - families=( - model_input.schema_entry.dynamic_columns - if model_input.schema_entry is not None - else () - ), + families=model_dynamic_families(model_input), column_types_by_table=column_types_by_table or {}, authoritative_column_types_by_table=( dynamic_contract_analysis_inputs.authoritative_column_types_by_table diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py index 020aa3568e..6db1cd78b5 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py @@ -8,6 +8,7 @@ from pathlib import Path from sqlbuild.compiler.compile._helpers.analysis.validation import ( + hook_sql_statements, validate_hook_sql_syntax, validate_sql_syntax, ) @@ -31,6 +32,11 @@ SQL_ANALYSIS_CONFIG_KEY, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.project_assembly.main._check_native_sql_syntax import ( + check_native_sql_syntax, +) from sqlbuild.errors.setting_help.main.join_helps import join_helps from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.errors.setting_help.main.setting_note import setting_note @@ -66,6 +72,24 @@ def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLo or not any(request.config.values.get(key) is False for key in _OPT_OUT_KEYS) ): return None + if not _parses(request): + return None + return _opt_out_location(request) + + +def _parses(request: SqlAnalysisOptOutRequest) -> bool: + """Whether the model query and every hook pass Python's syntax validation.""" + + if native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): + statements: list[str] = [request.query_sql] + for hook_name in _HOOK_KEYS: + statements.extend(hook_sql_statements(request.config.values.get(hook_name))) + native: bool | None = check_native_sql_syntax( + checks=tuple((statement, request.placeholders) for statement in statements), + dialect=None, + ) + if native is not None: + return native model_name: str = request.model_file.file_path.stem try: validate_sql_syntax( @@ -83,8 +107,8 @@ def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLo placeholders=request.placeholders, ) except CompileInputError: - return None - return _opt_out_location(request) + return False + return True def reject_unneeded_sql_analysis_opt_outs(project: CompiledProject) -> CompiledProject: diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 2c1f1bc21d..9130989292 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -3,6 +3,7 @@ from __future__ import annotations from functools import partial +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( @@ -11,10 +12,15 @@ from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, ) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contract import ( - prove_native_dynamic_contract, +from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( + prove_native_dynamic_contracts, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, + NativePivotTables, +) +from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import standalone_pivot_models from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_syntax_checks from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, @@ -26,32 +32,78 @@ ) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( assemble_native_project_resources, ) -from sqlbuild.compiler.project_assembly.models import NativeProjectResources +from sqlbuild.compiler.project_assembly.models import ( + NativeModelFacts, + NativeProjectFacts, + NativeProjectResources, +) from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily -def project_resources_by_engine( +def project_facts_by_engine( *, inputs: CompileProjectInputs, dialect: str | None, analysis_model_names: frozenset[str] | None, - analysis_succeeded: frozenset[str], -) -> NativeProjectResources | None: - """Return the natively assembled resource facts, or None where Python must assemble them.""" + analyses: dict[str, ModelSqlAnalysis], + session: Any | None, + column_types_by_table: dict[str, dict[str, str]], + authoritative_column_types_by_table: dict[str, dict[str, str]], + column_nullability_by_table: dict[str, dict[str, InferredNullability]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], +) -> NativeProjectFacts | None: + """Return the native project facts, or None where Python must derive all of them.""" if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): return None - return assemble_native_project_resources( + resources: NativeProjectResources | None = assemble_native_project_resources( inputs=inputs, dialect=dialect, syntax_checks=model_syntax_checks( model_inputs=inputs.model_inputs, analysis_model_names=analysis_model_names, - analysis_succeeded=analysis_succeeded, + analysis_succeeded=frozenset( + name + for name, analysis in analyses.items() + if analysis.polyglot_analysis.analysis_succeeded + ), + ), + ) + pivots: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = ( + standalone_pivot_models(model_inputs=inputs.model_inputs, analyses=analyses) + if native_stage_enabled(NativeStage.MODEL_ANALYSIS) + else {} + ) + proofs: dict[int, DynamicColumnContractProof | None] = dict( + zip( + pivots, + prove_native_dynamic_contracts( + session=session, + tables=NativePivotTables( + dialect=dialect, + column_types_by_table=column_types_by_table, + authoritative_column_types_by_table=authoritative_column_types_by_table, + column_nullability_by_table=column_nullability_by_table, + dynamic_families_by_table=dynamic_families_by_table, + ), + models=tuple(pivots.values()), + ), + strict=True, + ) + ) + return NativeProjectFacts( + models=tuple( + NativeModelFacts( + deps=resources.model_deps[index] if resources is not None else None, + dynamic_contract=proofs.get(index), + ) + for index in range(len(inputs.model_inputs)) ), + resources=resources, ) @@ -73,37 +125,30 @@ def analyze_model_sql_by_engine( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> dict[str, ModelSqlAnalysis]: - """Analyze models natively with the Python analysis's arguments, or run the Python analysis.""" +) -> tuple[dict[str, ModelSqlAnalysis], Any | None]: + """Analyze models natively, or with Python; also return the finished native session.""" if native_stage_enabled(NativeStage.MODEL_ANALYSIS): - native_analyses: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + native: NativeModelAnalyses | None = analyze_native_model_sql( request=NativeModelAnalysisRequest( **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table ) ) - if native_analyses is not None: - return native_analyses - return python_analysis() + if native is not None: + return native.analyses, native.session + return python_analysis(), None def dynamic_column_contract_by_engine( *, sql_analysis: ModelSqlAnalysis | None, + native_proof: DynamicColumnContractProof | None, python_proof: partial[DynamicColumnContractProof | None], ) -> DynamicColumnContractProof | None: - """Return the proof native analysis carries or proves alone, or prove it in Python.""" + """Return the proof native analysis carries or proved alone, or prove it in Python.""" if sql_analysis is not None and sql_analysis.dynamic_column_contract is not None: return sql_analysis.dynamic_column_contract - if ( - sql_analysis is None - and python_proof.keywords["families"] - and native_stage_enabled(NativeStage.MODEL_ANALYSIS) - ): - native: DynamicColumnContractProof | None = prove_native_dynamic_contract( - **python_proof.keywords - ) - if native is not None: - return native + if native_proof is not None: + return native_proof return python_proof() diff --git a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py new file mode 100644 index 0000000000..a929ed239f --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py @@ -0,0 +1,24 @@ +"""Check SQL syntax natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.project_assembly._helpers.deferrals import record_project_assembly_deferral + + +def check_native_sql_syntax( + *, checks: tuple[tuple[str, dict[str, str] | None], ...], dialect: str | None +) -> bool | None: + """Whether Python's syntax validation accepts every `(sql, placeholders)`; None to check it.""" + + valid: bool | None + deferral: str | None + valid, deferral = _native.check_native_sql_syntax( + ( + dialect or "generic", + [(sql, list((placeholders or {}).items())) for sql, placeholders in checks], + ) + ) + if valid is None: + _ = record_project_assembly_deferral(kind=deferral or "unknown") + return valid diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py index 8c6631c2cf..7905511275 100644 --- a/src/sqlbuild/compiler/project_assembly/models.py +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -4,7 +4,11 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, + DynamicColumnContractProof, +) from sqlbuild.spec.contracts.models import SourceEntry @@ -17,3 +21,19 @@ class NativeProjectResources: seed_destinations: tuple[CompiledRelationLocation, ...] function_deps: tuple[tuple[CompiledObjectKey, ...], ...] audit_scope_deps: tuple[tuple[CompiledObjectKey, ...], ...] + + +@dataclass(frozen=True, slots=True) +class NativeModelFacts: + """One model's native facts; each None where Python derives it.""" + + deps: tuple[CompiledObjectKey, ...] | None + dynamic_contract: DynamicColumnContractProof | None + + +@dataclass(frozen=True, slots=True) +class NativeProjectFacts: + """Every model's native facts, and the resource facts unless Python must assemble them.""" + + models: tuple[NativeModelFacts, ...] + resources: NativeProjectResources | None diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index b806fb398c..3ce8285d28 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -277,6 +277,7 @@ def host_project(project: CompiledProject) -> CompiledProject: return replace( project, binding_catalog=None, + native_session=None, sql_expansions={}, loaded_macros={}, loader_functions=(), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index bd27e5c2d5..1f9ef3f3c5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,6 +8,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage @@ -81,7 +82,8 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ ) assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) assert isinstance(preview_seams["assemble_native_project_resources"][0], NativeProjectResources) - assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) + assert isinstance(preview_seams["analyze_native_model_sql"][0], NativeModelAnalyses) + assert preview_seams["analyze_native_model_sql"][0].session is not None assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] assert isinstance(planned_artifacts, tuple) @@ -91,6 +93,7 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ len(planned_artifacts) ] assert isinstance(preview_seams["complete_native_semantic_diagnostics"][0], CompiledProject) + assert preview_seams["complete_native_semantic_diagnostics"][0].native_session is not None assert isinstance(preview_seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert (native_run.returncode, preview_run.returncode) == (0, 0) assert report_without_engine(native_run) == report_without_engine(python_run) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index f3ea3a0aac..4aca1c7f8d 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -48,4 +48,13 @@ class StandalonePivotProofTestCase: description: str analysed_models: frozenset[str] expected_native_proofs: int + expected_session_answers: list[bool] expected_proven_by_model: dict[str, bool | None] + + +@dataclass(frozen=True) +class HostPayloadTestCase: + """A natively analysed project that custom-rule hosts receive without its native session.""" + + description: str + expected_models: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 47095774c0..669514450a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -25,10 +25,14 @@ from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, ) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contract import ( - prove_native_dynamic_contract, +from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( + prove_native_dynamic_contracts, +) +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, + NativePivotTables, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( get_expression_source_shapes, ) @@ -301,19 +305,27 @@ def pivot_project_files() -> dict[str, str]: } -def native_pivot_proofs( - *, monkeypatch: pytest.MonkeyPatch -) -> list[DynamicColumnContractProof | None]: - """Record every standalone native pivot proof assembly takes, failing any wheel pivot proof.""" +@dataclass +class NativePivotProofs: + """Every standalone native pivot proof assembly took, and whether a session answered it.""" + + proofs: list[DynamicColumnContractProof | None] = field(default_factory=list) + session_answers: list[bool] = field(default_factory=list) + + +def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs: + """Record every batch of native pivot proofs assembly takes, failing any wheel pivot proof.""" - proofs: list[DynamicColumnContractProof | None] = [] - native_proof: Callable[..., DynamicColumnContractProof | None] = ( - native_stage_assembly.prove_native_dynamic_contract + recorded_proofs: NativePivotProofs = NativePivotProofs() + native_proofs: Callable[..., tuple[DynamicColumnContractProof | None, ...]] = ( + native_stage_assembly.prove_native_dynamic_contracts ) - def recorded(**keywords: Any) -> DynamicColumnContractProof | None: - proofs.append(native_proof(**keywords)) - return proofs[-1] + def recorded(**keywords: Any) -> tuple[DynamicColumnContractProof | None, ...]: + proofs: tuple[DynamicColumnContractProof | None, ...] = native_proofs(**keywords) + recorded_proofs.proofs.extend(proofs) + recorded_proofs.session_answers.extend([keywords["session"] is not None] * len(proofs)) + return proofs python_proof: Callable[..., DynamicColumnContractProof | None] = ( project_assembly.analyze_dynamic_column_contract @@ -323,9 +335,9 @@ def wheel_not_expected(**keywords: Any) -> DynamicColumnContractProof | None: assert not keywords["families"], "the Python wheel proved a pivot native should prove" return python_proof(**keywords) - monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contract", recorded) + monkeypatch.setattr(native_stage_assembly, "prove_native_dynamic_contracts", recorded) monkeypatch.setattr(project_assembly, "analyze_dynamic_column_contract", wheel_not_expected) - return proofs + return recorded_proofs def generated_analysis_files(*, rng: random.Random, model_count: int) -> dict[str, str]: @@ -374,6 +386,7 @@ class AnalysisParity: expression_shapes: int = 0 pivot_proofs: int = 0 standalone_proofs: int = 0 + session_proofs: int = 0 proven_pivots: int = 0 native_enrichments: int = 0 @@ -387,15 +400,17 @@ def compare_analyses( ) -> None: """Assemble `inputs`, analysing every seam call with both engines into `parity`.""" + sessions: list[Any] = [] + def models_by_both( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], - ) -> dict[str, ModelSqlAnalysis]: + ) -> tuple[dict[str, ModelSqlAnalysis], Any]: keywords: dict[str, Any] = {**python_analysis.keywords, "analysis_cache": None} profile: ExpressionInferenceProfile = keywords["inference_profile"] native_catalog: Any = profile.binding_catalog.with_relations({}) - native: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + native_result: NativeModelAnalyses | None = analyze_native_model_sql( request=NativeModelAnalysisRequest( **{ **keywords, @@ -404,6 +419,8 @@ def models_by_both( dynamic_families_by_table=dynamic_families_by_table, ) ) + native: dict[str, ModelSqlAnalysis] | None = getattr(native_result, "analyses", None) + sessions.append(getattr(native_result, "session", None)) python: dict[str, ModelSqlAnalysis] = python_analysis.func(**keywords) parity.analysed_models += len(python) _append(parity, "model analyses", _analysis_views(python), _analysis_views(native)) @@ -413,25 +430,32 @@ def models_by_both( _catalog_view(profile.binding_catalog), _catalog_view(native_catalog), ) - return native or python + return native or python, sessions[-1] def proofs_by_both( *, sql_analysis: ModelSqlAnalysis | None, + native_proof: DynamicColumnContractProof | None, python_proof: partial[DynamicColumnContractProof | None], ) -> DynamicColumnContractProof | None: native: DynamicColumnContractProof | None = getattr( sql_analysis, "dynamic_column_contract", None ) python: DynamicColumnContractProof | None = python_proof() - standalone: DynamicColumnContractProof | None = prove_native_dynamic_contract( - **python_proof.keywords + session: Any = (sessions or [None])[-1] + standalone: DynamicColumnContractProof | None = _batched_proof( + session=None, python_proof=python_proof + ) + in_session: DynamicColumnContractProof | None = _batched_proof( + session=session, python_proof=python_proof ) + parity.session_proofs += in_session is not None and session is not None parity.pivot_proofs += native is not None parity.proven_pivots += native is not None and native.output_proven parity.standalone_proofs += standalone is not None _append(parity, "dynamic pivot proof", python, native or python) _append(parity, "standalone dynamic pivot proof", python, standalone or python) + _append(parity, "session dynamic pivot proof", python, in_session or python) return python def shapes_by_both( @@ -485,13 +509,13 @@ def captured( *, python_analysis: partial[dict[str, ModelSqlAnalysis]], dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], - ) -> dict[str, ModelSqlAnalysis]: + ) -> tuple[dict[str, ModelSqlAnalysis], None]: requests.append( NativeModelAnalysisRequest( **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table ) ) - return python_analysis() + return python_analysis(), None with monkeypatch.context() as patch, suppress(CompileInputError): patch.setattr(project_assembly, "analyze_model_sql_by_engine", captured) @@ -515,6 +539,23 @@ def deferral_kinds(record_dir: Path) -> Counter[str]: return Counter(f"{record['site']}:{record['kind']}" for record in records) +def _batched_proof( + *, session: Any, python_proof: partial[DynamicColumnContractProof | None] +) -> DynamicColumnContractProof | None: + keywords: dict[str, Any] = python_proof.keywords + return prove_native_dynamic_contracts( + session=session, + tables=NativePivotTables( + dialect=keywords["dialect"], + column_types_by_table=keywords["column_types_by_table"], + authoritative_column_types_by_table=keywords["authoritative_column_types_by_table"], + column_nullability_by_table=keywords["column_nullability_by_table"], + dynamic_families_by_table=keywords["dynamic_families_by_table"], + ), + models=((keywords["query_sql"], keywords["families"]),), + )[0] + + def _append(parity: AnalysisParity, name: str, python: object, native: object) -> None: parity.names.append(name) parity.python.append(python) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 4e9527ea64..1b996f612a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -2,6 +2,7 @@ from __future__ import annotations +import pickle import random from collections import Counter from dataclasses import replace @@ -18,20 +19,23 @@ from sqlbuild.compiler.compile.models import ( CompiledProject, CompileProjectInputs, - DynamicColumnContractProof, ) from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from sqlbuild.rule_engine._helpers.engine.custom_rules import host_project +from sqlbuild.rule_engine._helpers.host.custom_host_pool import host_payload_project from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AnalysisFallbackTestCase, GeneratedAnalysisParityTestCase, + HostPayloadTestCase, SessionFailureTestCase, StandalonePivotProofTestCase, ) from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( AnalysisParity, FailingProvideSession, + NativePivotProofs, analysis_request, compare_analyses, compile_inputs, @@ -97,6 +101,7 @@ def test_given_generated_projects_when_analysing_natively_then_matches_python( assert parity.expression_shapes >= test_case.expected_minimum_expression_shapes assert parity.pivot_proofs >= test_case.expected_minimum_pivot_proofs assert parity.standalone_proofs >= test_case.expected_minimum_pivot_proofs + assert parity.session_proofs >= test_case.expected_minimum_pivot_proofs assert parity.proven_pivots >= test_case.expected_minimum_proven_pivots assert ( kinds["analysis_session:input_enrichment"] @@ -199,12 +204,24 @@ def test_given_session_failure_after_deferrals_when_analysing_then_records_only_ description="a selection that leaves the pivots out of model analysis", analysed_models=frozenset({"orders_list"}), expected_native_proofs=2, + expected_session_answers=[True, True], expected_proven_by_model={ "orders_list": None, "status_amounts": True, "status_passthrough": True, }, - ) + ), + StandalonePivotProofTestCase( + description="no model analysis, so the proofs run without a session", + analysed_models=frozenset(), + expected_native_proofs=2, + expected_session_answers=[False, False], + expected_proven_by_model={ + "orders_list": None, + "status_amounts": True, + "status_passthrough": True, + }, + ), ], ids=lambda case: case.description, ) @@ -215,7 +232,7 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with project_dir=tmp_path / "project", files=pivot_project_files() ) monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) - proofs: list[DynamicColumnContractProof | None] = native_pivot_proofs(monkeypatch=monkeypatch) + recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) project: CompiledProject = assemble_compiled_project( inputs=inputs, @@ -224,13 +241,51 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ) assert ( - all(proof is not None for proof in proofs), - len(proofs), + all(proof is not None for proof in recorded.proofs), + len(recorded.proofs), + recorded.session_answers, { model.name: getattr(model.dynamic_column_contract, "output_proven", None) for model in project.models }, - ) == (True, test_case.expected_native_proofs, test_case.expected_proven_by_model) + ) == ( + True, + test_case.expected_native_proofs, + test_case.expected_session_answers, + test_case.expected_proven_by_model, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + HostPayloadTestCase( + description="a pivot project analysed in one native session", + expected_models=("orders_list", "status_amounts", "status_passthrough"), + ) + ], + ids=lambda case: case.description, +) +def test_given_native_session_when_publishing_rule_host_payload_then_session_is_dropped( + test_case: HostPayloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + inputs: CompileProjectInputs = compile_inputs( + project_dir=tmp_path / "project", files=pivot_project_files() + ) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) + project: CompiledProject = assemble_compiled_project( + inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb") + ) + + payload: CompiledProject = pickle.loads( + pickle.dumps(host_payload_project(host_project(project))) + ) + + assert project.native_session is not None + assert (payload.native_session, tuple(sorted(model.name for model in payload.models))) == ( + None, + test_case.expected_models, + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index 680d93254c..1e010ed519 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -45,3 +45,13 @@ class WindowsEnvironmentTestCase: environment: dict[str, str] seed_schema: str expected_schema: str + + +@dataclass(frozen=True) +class OptOutTestCase: + """Models opting out of required SQL analysis, and which ones parse so Python rejects them.""" + + description: str + models: dict[str, str] + expected_rejected: dict[str, bool] + expected_python_validations: int diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 05d2a1188c..c8b4ded2df 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -15,6 +15,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly import project as project_module from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild.compiler.compile._helpers.diagnostics import sql_analysis_opt_outs from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -38,6 +39,7 @@ "audit_scope_deps", ) _ASSEMBLY_RECORD_PREFIX: str = "project_assembly:" +_OPT_OUT_VALIDATORS: tuple[str, ...] = ("validate_sql_syntax", "validate_hook_sql_syntax") _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, @@ -199,6 +201,38 @@ def assembly_view(project: CompiledProject) -> object: ) +def rejected_opt_outs( + *, + project_dir: Path, + files: dict[str, str], + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[dict[str, bool], int]: + """Attach `files` with `engine`; return each opt-out's rejection and Python validations.""" + + calls: Counter[str] = Counter() + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + with monkeypatch.context() as patch: + _ = [ + patch.setattr( + sql_analysis_opt_outs, + name, + _counted(name=name, function=getattr(sql_analysis_opt_outs, name), calls=calls), + ) + for name in _OPT_OUT_VALIDATORS + ] + inputs: CompileProjectInputs = project_inputs(project_dir=project_dir, files=files) + return ( + { + model_input.model_file.file_path.stem: ( + model_input.rejected_sql_analysis_opt_out is not None + ) + for model_input in inputs.model_inputs + }, + calls.total(), + ) + + def assembly_deferrals(record_dir: Path) -> Counter[str]: """Count the project assembly `site:kind` deferral records below `record_dir`.""" diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index 5b9ef5783e..e88e498591 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -19,6 +19,7 @@ AssemblyDeferralTestCase, DeferredAssemblyTestCase, GeneratedAssemblyParityTestCase, + OptOutTestCase, WindowsEnvironmentTestCase, ) from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( @@ -29,6 +30,7 @@ project_inputs, python_resource_calls, recorded_assembly, + rejected_opt_outs, seed_yml, windows_environ, ) @@ -37,6 +39,11 @@ 'name = "assembly"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n\n' ) _ORDERS_MODEL: str = 'MODEL (description "Orders");\n\nSELECT 1 AS n\n' +_REQUIRED_ANALYSIS_TOML: str = ( + 'name = "assembly"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = ":memory:"\n\n[settings]\nrequire_sql_analysis = true\n' +) +_OPT_OUT_HEADER: str = 'MODEL (\n description "Orders",\n sql_analysis false,\n' _SEED_FILES: dict[str, str] = { "seeds/countries.csv": "n\n1\n", "models/orders.sql": _ORDERS_MODEL, @@ -261,3 +268,47 @@ def test_given_windows_environment_when_assembling_natively_then_env_lookups_mat assert python.seeds[0].destination.schema == test_case.expected_schema assert assembly_view(native) == assembly_view(python) assert assembly_deferrals(record_dir) == Counter() + + +@pytest.mark.parametrize( + "test_case", + [ + OptOutTestCase( + description="parseable, unparseable and unparseable-hook opt-outs", + models={ + "models/parses.sql": f"{_OPT_OUT_HEADER});\n\nSELECT 1 AS n\n", + "models/broken_query.sql": f"{_OPT_OUT_HEADER});\n\nSELECT FROM WHERE (\n", + "models/broken_hook.sql": ( + f'{_OPT_OUT_HEADER} post_hooks [inline_sql("SELECT FROM (")],\n);\n\n' + "SELECT 1 AS n\n" + ), + }, + expected_rejected={"broken_hook": False, "broken_query": False, "parses": True}, + expected_python_validations=7, + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejections_match( + test_case: OptOutTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record_dir: Path = tmp_path / "records" + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + files: dict[str, str] = {"sqlbuild_project.toml": _REQUIRED_ANALYSIS_TOML, **test_case.models} + + python: tuple[dict[str, bool], int] = rejected_opt_outs( + project_dir=tmp_path / "python", + files=files, + engine=CompilerEngine.PYTHON, + monkeypatch=monkeypatch, + ) + native: tuple[dict[str, bool], int] = rejected_opt_outs( + project_dir=tmp_path / "native", + files=files, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + + assert python == (test_case.expected_rejected, test_case.expected_python_validations) + assert native == (test_case.expected_rejected, 0) + assert assembly_deferrals(record_dir) == Counter() diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index 8b65e73896..33c8c913f4 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -100,7 +100,7 @@ "SELECT 1\n" ), }, - expected_unpopulated_trimmed_fields=frozenset(), + expected_unpopulated_trimmed_fields=frozenset({"native_session"}), ), ), ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py index a49bf87317..6fd230b52e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py @@ -10,6 +10,7 @@ class DynamicColumnContractDispatchTestCase: description: str sql_analysis: ModelSqlAnalysis | None + native_proof: DynamicColumnContractProof | None dialect: str families: tuple[SchemaDynamicColumnFamily, ...] expected_proof: DynamicColumnContractProof | None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py index 92ff874493..8addc9e694 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py @@ -44,6 +44,7 @@ ( DynamicColumnContractDispatchTestCase( description="native analysis proof replaces the python proof", + native_proof=None, sql_analysis=ModelSqlAnalysis( polyglot_analysis=_ANALYSIS, placeholders=None, @@ -55,6 +56,7 @@ ), DynamicColumnContractDispatchTestCase( description="analysis without a proof falls back to the python proof", + native_proof=None, sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), dialect="bigquery", families=(_FAMILY,), @@ -62,13 +64,23 @@ ), DynamicColumnContractDispatchTestCase( description="unanalyzed model falls back to the python proof", + native_proof=None, sql_analysis=None, dialect="bigquery", families=(_FAMILY,), expected_proof=_PYTHON_UNSUPPORTED_PROOF, ), + DynamicColumnContractDispatchTestCase( + description="a batched native proof answers an unanalyzed model", + native_proof=_NATIVE_PROOF, + sql_analysis=None, + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_NATIVE_PROOF, + ), DynamicColumnContractDispatchTestCase( description="model without dynamic families has no proof", + native_proof=None, sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), dialect="snowflake", families=(), @@ -82,6 +94,7 @@ def test_given_model_analysis_when_proving_dynamic_contract_then_native_proof_ta ) -> None: proof: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( sql_analysis=test_case.sql_analysis, + native_proof=test_case.native_proof, python_proof=partial( analyze_dynamic_column_contract, query_sql="SELECT * FROM order_amounts", From c51b7c43d00478373c89b2efc39dc8932046e3de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:38:30 +0100 Subject: [PATCH 2/3] fix(native): keep the finished analysis session out of CompiledProject and gate on session proofs --- .../compile/_helpers/assembly/project.py | 1 - .../_helpers/engine/custom_rules.py | 1 - .../compile/test_native_analysis_seams.py | 1 - .../compiler/analysis_session/_test_types.py | 10 +---- .../compiler/analysis_session/helpers.py | 7 +-- .../test_native_model_analysis.py | 44 ++----------------- .../host_payload/test_host_payload_facts.py | 2 +- 7 files changed, 10 insertions(+), 56 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index b05203f0f0..2e925251bf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -498,7 +498,6 @@ def assemble_compiled_project( ), external_sql_reference_resolver=inputs.external_sql_reference_resolver, scope_index=scope_index, - native_session=native_session, ) return complete_semantic_diagnostics( project=project, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 3ce8285d28..b806fb398c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -277,7 +277,6 @@ def host_project(project: CompiledProject) -> CompiledProject: return replace( project, binding_catalog=None, - native_session=None, sql_expansions={}, loaded_macros={}, loader_functions=(), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 1f9ef3f3c5..5df91de88a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -93,7 +93,6 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ len(planned_artifacts) ] assert isinstance(preview_seams["complete_native_semantic_diagnostics"][0], CompiledProject) - assert preview_seams["complete_native_semantic_diagnostics"][0].native_session is not None assert isinstance(preview_seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert (native_run.returncode, preview_run.returncode) == (0, 0) assert report_without_engine(native_run) == report_without_engine(python_run) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 4aca1c7f8d..25a92f2c22 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -48,13 +48,5 @@ class StandalonePivotProofTestCase: description: str analysed_models: frozenset[str] expected_native_proofs: int - expected_session_answers: list[bool] + expected_session_proofs: int expected_proven_by_model: dict[str, bool | None] - - -@dataclass(frozen=True) -class HostPayloadTestCase: - """A natively analysed project that custom-rule hosts receive without its native session.""" - - description: str - expected_models: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 669514450a..9da32670a2 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -307,10 +307,10 @@ def pivot_project_files() -> dict[str, str]: @dataclass class NativePivotProofs: - """Every standalone native pivot proof assembly took, and whether a session answered it.""" + """Every native pivot proof assembly took, and how many a finished session proved.""" proofs: list[DynamicColumnContractProof | None] = field(default_factory=list) - session_answers: list[bool] = field(default_factory=list) + session_proofs: int = 0 def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs: @@ -324,7 +324,8 @@ def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs def recorded(**keywords: Any) -> tuple[DynamicColumnContractProof | None, ...]: proofs: tuple[DynamicColumnContractProof | None, ...] = native_proofs(**keywords) recorded_proofs.proofs.extend(proofs) - recorded_proofs.session_answers.extend([keywords["session"] is not None] * len(proofs)) + proven: int = sum(proof is not None for proof in proofs) + recorded_proofs.session_proofs += proven * (keywords["session"] is not None) return proofs python_proof: Callable[..., DynamicColumnContractProof | None] = ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 1b996f612a..d7aebb0d9b 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -2,7 +2,6 @@ from __future__ import annotations -import pickle import random from collections import Counter from dataclasses import replace @@ -23,12 +22,9 @@ from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR -from sqlbuild.rule_engine._helpers.engine.custom_rules import host_project -from sqlbuild.rule_engine._helpers.host.custom_host_pool import host_payload_project from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AnalysisFallbackTestCase, GeneratedAnalysisParityTestCase, - HostPayloadTestCase, SessionFailureTestCase, StandalonePivotProofTestCase, ) @@ -204,7 +200,7 @@ def test_given_session_failure_after_deferrals_when_analysing_then_records_only_ description="a selection that leaves the pivots out of model analysis", analysed_models=frozenset({"orders_list"}), expected_native_proofs=2, - expected_session_answers=[True, True], + expected_session_proofs=2, expected_proven_by_model={ "orders_list": None, "status_amounts": True, @@ -215,7 +211,7 @@ def test_given_session_failure_after_deferrals_when_analysing_then_records_only_ description="no model analysis, so the proofs run without a session", analysed_models=frozenset(), expected_native_proofs=2, - expected_session_answers=[False, False], + expected_session_proofs=0, expected_proven_by_model={ "orders_list": None, "status_amounts": True, @@ -243,7 +239,7 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with assert ( all(proof is not None for proof in recorded.proofs), len(recorded.proofs), - recorded.session_answers, + recorded.session_proofs, { model.name: getattr(model.dynamic_column_contract, "output_proven", None) for model in project.models @@ -251,42 +247,10 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ) == ( True, test_case.expected_native_proofs, - test_case.expected_session_answers, + test_case.expected_session_proofs, test_case.expected_proven_by_model, ) -@pytest.mark.parametrize( - "test_case", - [ - HostPayloadTestCase( - description="a pivot project analysed in one native session", - expected_models=("orders_list", "status_amounts", "status_passthrough"), - ) - ], - ids=lambda case: case.description, -) -def test_given_native_session_when_publishing_rule_host_payload_then_session_is_dropped( - test_case: HostPayloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - inputs: CompileProjectInputs = compile_inputs( - project_dir=tmp_path / "project", files=pivot_project_files() - ) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) - project: CompiledProject = assemble_compiled_project( - inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb") - ) - - payload: CompiledProject = pickle.loads( - pickle.dumps(host_payload_project(host_project(project))) - ) - - assert project.native_session is not None - assert (payload.native_session, tuple(sorted(model.name for model in payload.models))) == ( - None, - test_case.expected_models, - ) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index 33c8c913f4..8b65e73896 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -100,7 +100,7 @@ "SELECT 1\n" ), }, - expected_unpopulated_trimmed_fields=frozenset({"native_session"}), + expected_unpopulated_trimmed_fields=frozenset(), ), ), ids=lambda case: case.description, From d5c99e840f4a89a12a3aff42d79ce0dd64cc4207 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:45:02 +0100 Subject: [PATCH 3/3] feat(native): feed semantic completion from the finished analysis session Type recovery and completion read each analysed model's output names and lineage from the finished native session instead of payload rows rebuilt from CompiledProject. Models whose analysis Python answered, failed, or whose names a dynamic pivot proof replaced keep the payload. --- .../analysis_session/_helpers/session.rs | 36 +++++++- .../main/finished_fact_models.rs | 8 ++ .../main/finished_model_facts.rs | 11 +++ .../src/assembly/analysis_session/main/mod.rs | 2 + .../src/assembly/analysis_session/models.rs | 12 ++- .../analysis_session/tests/helpers.rs | 39 +++++++-- .../tests/test_analysis_session.rs | 64 +++++++++++++- .../analysis_session/tests/test_types.rs | 7 ++ .../src/assembly/analysis_session/types.rs | 2 + .../_helpers/analysis_session/session.rs | 15 ++++ .../_helpers/semantic_checks/completion.rs | 44 +++++++--- .../bindings/_helpers/semantic_checks/mod.rs | 1 + .../_helpers/semantic_checks/session_facts.rs | 18 ++++ .../_helpers/semantic_checks/type_recovery.rs | 38 ++++++--- .../analysis_corpus/semantic_checks.py | 25 ++++++ src/sqlbuild/_native.pyi | 10 ++- .../compile/_helpers/assembly/project.py | 1 + .../compile/_helpers/diagnostics/recovery.py | 3 + .../semantic_checks/_helpers/completion.py | 33 ++++++-- .../semantic_checks/_helpers/payloads.py | 22 +++++ .../semantic_checks/_helpers/stage.py | 12 ++- .../semantic_checks/_helpers/type_recovery.py | 19 ++++- .../_complete_native_semantic_diagnostics.py | 4 + .../compiler/semantic_checks/_test_types.py | 15 ++++ .../compiler/semantic_checks/helpers.py | 58 ++++++++++++- .../test_native_semantic_checks.py | 83 +++++++++++++++++++ 26 files changed, 531 insertions(+), 51 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index fa9ebdc21f..de54b97e79 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -24,8 +24,8 @@ use crate::assembly::analysis_session::constants::{ }; use crate::assembly::analysis_session::models::{ AnalysisSession, Awaiting, ColumnFact, Deferral, DeferredAnalysis, FinishedSession, - LineageFacts, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, - SessionOutcome, SessionRequest, SessionStep, + LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, + PivotTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::ProjectCatalog; @@ -164,6 +164,12 @@ impl AnalysisSession { .ok_or("the session left a model unanalysed")?; let (schema_additions, analysis_names) = self.catalog.into_changes(); let request: SessionRequest = self.request; + let mut model_facts: HashMap = HashMap::new(); + for (model, outcome) in request.models.iter().zip(&models) { + if let Some(facts) = session_model_facts(&outcome.analysis) { + model_facts.insert(model.name.clone(), facts); + } + } let finished = FinishedSession { tables: PivotTables { dialect: request.dialect, @@ -173,6 +179,7 @@ impl AnalysisSession { families_by_table: request.dynamic_families_by_table, }, pool, + models: model_facts, }; let outcome = SessionOutcome { models, @@ -611,3 +618,28 @@ fn native_answer(model: usize, columns: Vec, has_star: bool) -> Defe binding_validated: false, } } + +/// A compiled model's output names and lineage from a successful analysis with native lineage. +fn session_model_facts(analysis: &ModelAnalysis) -> Option { + let rows: &Vec = match &analysis.lineage { + LineageFacts::Native(rows) | LineageFacts::NativeEnrichment(rows) => rows, + LineageFacts::PythonAnalysis | LineageFacts::PythonEnrichment => return None, + }; + analysis.analysis_succeeded.then(|| SessionModelFacts { + columns: analysis.columns.as_deref().map(column_names), + lineage: rows.iter().map(output_sources).collect(), + }) +} + +fn column_names(columns: &[ColumnFact]) -> Vec { + columns.iter().map(|column| column.name.clone()).collect() +} + +/// One lineage row as `(output column, [(resource name, column name)])`. +fn output_sources(row: &LineageRow) -> (String, Vec<(String, String)>) { + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (_, name, column) in &row.sources { + sources.push((name.clone(), column.clone())); + } + (row.output_column.clone(), sources) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs new file mode 100644 index 0000000000..13692c612d --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_fact_models.rs @@ -0,0 +1,8 @@ +//! The models whose output names and lineage a finished analysis session kept. + +use crate::assembly::analysis_session::models::FinishedSession; + +/// Names of the models whose facts the session kept, in no particular order. +pub fn finished_fact_models(session: &FinishedSession) -> Vec { + session.models.keys().cloned().collect() +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs new file mode 100644 index 0000000000..bac994679b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/finished_model_facts.rs @@ -0,0 +1,11 @@ +//! The output names and lineage a finished analysis session kept for compiled models. + +use crate::assembly::analysis_session::models::{FinishedSession, SessionModelFacts}; + +/// The named model's facts, or None where the session kept none for it. +pub fn finished_model_facts<'a>( + session: &'a FinishedSession, + name: &str, +) -> Option<&'a SessionModelFacts> { + session.models.get(name) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index f50a22f2ca..d3a5e4c347 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -1,5 +1,7 @@ pub mod expression_shapes; pub mod finish_analysis_session; +pub mod finished_fact_models; +pub mod finished_model_facts; pub mod prove_dynamic_contracts; pub mod prove_finished_dynamic_contracts; pub mod provide_deferred_analyses; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 565ccf3544..f5988b2b08 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -7,7 +7,7 @@ use rayon::ThreadPool; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; -use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::assembly::analysis_session::types::{OutputSources, Pairs, Shapes}; use crate::semantic_validation::types::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. @@ -88,11 +88,19 @@ pub struct PivotBatchRequest { pub models: Vec, } -/// A finished session's pivot tables and analysis pool, kept to prove models outside it. +/// A finished session's pivot tables, analysis pool and model facts, kept for later stages. #[derive(Debug)] pub struct FinishedSession { pub(crate) tables: PivotTables, pub(crate) pool: Result, String>, + pub(crate) models: HashMap, +} + +/// A compiled model's output names and lineage from an analysis the session completed natively. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionModelFacts { + pub columns: Option>, + pub lineage: OutputSources, } /// A model's dynamic pivot proof: none declared, left to Python, or proven natively. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 0970aafe27..5705bd4a1b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -1,12 +1,15 @@ +use std::collections::HashMap; + use crate::assembly::analysis_session::_helpers::mappings::catalog_relations; use crate::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; +use crate::assembly::analysis_session::main::finished_model_facts::finished_model_facts; use crate::assembly::analysis_session::main::provide_deferred_analyses::provide_deferred_analyses; use crate::assembly::analysis_session::main::run_analysis_session::run_analysis_session; use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, Deferral, DeferredAnalysis, DynamicFamily, FinishedSession, ModelOutcome, ModelReference, ModelRequest, PivotBatchRequest, PivotModel, - PivotOutcome, PivotTables, SessionRequest, SessionStep, + PivotOutcome, PivotTables, SessionModelFacts, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::tests::test_types::ModelSpec; use crate::assembly::analysis_session::types::{Pairs, Shapes}; @@ -113,11 +116,11 @@ pub(crate) fn started(request: SessionRequest) -> AnalysisSession { start_analysis_session(request, &catalog).expect("the models form no cycle") } -/// Run to completion, answering every deferral with `answer`; return the steps and outcomes. +/// Run to completion, answering every deferral with `answer`: the steps, outcomes and session. pub(crate) fn completed( mut session: AnalysisSession, answer: fn(&Deferral) -> DeferredAnalysis, -) -> (Vec, Vec) { +) -> (Vec, Vec, FinishedSession) { let mut done: bool = false; let steps: Vec = std::iter::from_fn(|| { (!done).then(|| { @@ -131,8 +134,8 @@ pub(crate) fn completed( }) }) .collect(); - let (outcome, _) = finish_analysis_session(session).expect("the session finished"); - (steps, outcome.models) + let (outcome, finished) = finish_analysis_session(session).expect("the session finished"); + (steps, outcome.models, finished) } /// `name type nullability` per column, or `failed` when analysis did not succeed. @@ -205,7 +208,7 @@ pub(crate) fn model_requests(models: &[ModelSpec]) -> Vec { /// Run `models` over the orders request: each step's lines and each model's description. pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec>) { - let (steps, outcomes) = completed( + let (steps, outcomes, _) = completed( started(orders_request(model_requests(models))), empty_answer, ); @@ -215,6 +218,29 @@ pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec Vec { + let (_, _, finished) = completed( + started(orders_request(model_requests(models))), + empty_answer, + ); + models + .iter() + .map(|(name, ..)| { + finished_model_facts(&finished, name).map_or("none".to_owned(), fact_line) + }) + .collect() +} + +fn fact_line(facts: &SessionModelFacts) -> String { + let lineage: Vec = facts + .lineage + .iter() + .map(|(output, sources)| format!("{output}<-{sources:?}")) + .collect(); + format!("{:?} {}", facts.columns, lineage.join(" ")) +} + /// Proofs over typed `raw_orders` and pivot `status_amounts`: the model, then a family-less one. pub(crate) fn pivot_request( dialect: &str, @@ -266,6 +292,7 @@ pub(crate) fn finished_session(tables: PivotTables) -> FinishedSession { FinishedSession { tables, pool: catalog("duckdb", &Vec::new()).analysis_pool(), + models: HashMap::new(), } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs index ef2f2315bf..a2faa129b4 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs @@ -3,11 +3,11 @@ use crate::assembly::analysis_session::main::prove_finished_dynamic_contracts::p use crate::assembly::analysis_session::main::start_analysis_session::start_analysis_session; use crate::assembly::analysis_session::models::PivotOutcome; use crate::assembly::analysis_session::tests::helpers::{ - catalog, failed, finished_session, model_requests, orders_request, pivot_request, proven, - session_lines, + catalog, fact_lines, failed, finished_session, model_requests, orders_request, pivot_request, + proven, session_lines, }; use crate::assembly::analysis_session::tests::test_types::{ - PivotTestCase, SessionTestCase, UnscheduledTestCase, + PivotTestCase, SessionFactsTestCase, SessionTestCase, UnscheduledTestCase, }; #[test] @@ -127,6 +127,64 @@ fn given_models_when_running_the_session_then_defers_and_publishes_as_python_doe } } +#[test] +fn given_finished_sessions_when_reading_model_facts_then_keeps_only_native_successes() { + let test_cases = [ + SessionFactsTestCase { + description: "native lineage keeps output names and sources", + models: &[ + ( + "stg_orders", + "SELECT order_id, amount * 2 AS doubled FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ( + "orders_bad", + "SELECT missing_column FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ), + ], + expected_facts: &[ + "Some([\"order_id\", \"doubled\"]) \ + order_id<-[(\"raw_orders\", \"order_id\")] \ + doubled<-[(\"raw_orders\", \"amount\")]", + "Some([\"missing_column\"]) missing_column<-[(\"raw_orders\", \"missing_column\")]", + ], + }, + SessionFactsTestCase { + description: "a model Python analysed keeps no session facts", + models: &[ + ( + "events", + "SELECT event_id FROM __source(\"raw_events\")", + &["raw_events"], + &[], + ), + ( + "events_mart", + "SELECT event_id, 1 AS one FROM __ref(\"events\")", + &[], + &["events"], + ), + ], + expected_facts: &[ + "Some([\"event_id\"]) event_id<-[(\"raw_events\", \"event_id\")]", + "none", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + fact_lines(test_case.models), + test_case.expected_facts, + "{}", + test_case.description + ); + } +} + #[test] fn given_unschedulable_models_when_starting_then_python_analyses() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index c09b783048..547582f4ff 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -17,6 +17,13 @@ pub(crate) struct SessionTestCase { pub(crate) expected_outcomes: &'static [&'static [&'static str]], } +pub(crate) struct SessionFactsTestCase { + pub(crate) description: &'static str, + pub(crate) models: &'static [ModelSpec], + /// `fact_lines` of each model, in request order. + pub(crate) expected_facts: &'static [&'static str], +} + pub(crate) struct UnscheduledTestCase { pub(crate) description: &'static str, pub(crate) models: &'static [ModelSpec], diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs index 5c36906409..0dbac8f6a5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/types.rs @@ -4,3 +4,5 @@ pub type Pairs = Vec<(String, String)>; /// Ordered relation shapes in Python dict order. pub type Shapes = Vec<(String, Pairs)>; +/// `(output column, [(resource name, column name)])` per output, in lineage order. +pub type OutputSources = Vec<(String, Vec<(String, String)>)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 284a8545d3..59e9dd7a86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::provide_deferred_analyses::provide_deferred_analyses; @@ -124,6 +125,11 @@ pub(crate) struct NativeModelAnalysisSession { } impl NativeModelAnalysisSession { + /// The finished session later compile stages read, once `finish` has succeeded. + pub(crate) fn finished_session(&self) -> Option<&FinishedSession> { + self.finished.as_ref() + } + /// Keep `result`'s value, or end the session and keep its failure for Python's debug log. fn kept(&mut self, result: Result) -> Option { match result { @@ -139,6 +145,15 @@ impl NativeModelAnalysisSession { #[pymethods] impl NativeModelAnalysisSession { + /// Models whose output names and lineage the finished session keeps for later stages. + #[getter] + fn fact_models(&self) -> Vec { + self.finished + .as_ref() + .map(finished_fact_models) + .unwrap_or_default() + } + /// Advance; None means Python must analyse every model, no deferrals means done. fn run(&mut self, py: Python<'_>) -> Option { let mut session: AnalysisSession = self.inner.take()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index c9ac573eaa..c01e4e67d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -8,7 +8,9 @@ use sqlbuild_analysis::semantic_checks::models::{ SemanticLocation, }; +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::semantic_checks::session_facts::session_model_facts; use crate::bindings::_helpers::semantic_checks::type_recovery::{LineageInput, lineage_outputs}; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; @@ -33,8 +35,8 @@ type ModelInput = ( String, String, String, - Vec, - Vec, + Option>, + Option>, Vec, Option, ); @@ -66,14 +68,16 @@ type OutcomeRow = ( Vec, ); -/// Recover, explain and reject opt-outs on `catalog`'s analysis pool. +/// Recover, explain and reject opt-outs, reading facts the payload omits from `session`. #[pyfunction] +#[pyo3(signature = (catalog, request, session=None))] fn complete_semantic_checks( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: RequestInput, + session: Option>, ) -> PyResult { - let request = completion_request(request); + let request = completion_request(request, session.as_deref()).map_err(compiler_error)?; let catalog = &catalog.inner; let outcome = py .compiler_detach(|| complete_semantic_diagnostics(&request, catalog)) @@ -106,13 +110,19 @@ fn location_row(location: SemanticLocation) -> LocationRow { ) } -fn completion_request((dialect, diagnostics, models, shapes): RequestInput) -> CompletionRequest { - CompletionRequest { +fn completion_request( + (dialect, diagnostics, models, shapes): RequestInput, + session: Option<&NativeModelAnalysisSession>, +) -> Result { + Ok(CompletionRequest { dialect, diagnostics: diagnostics.into_iter().map(semantic_diagnostic).collect(), - models: models.into_iter().map(completion_model).collect(), + models: models + .into_iter() + .map(|model| completion_model(model, session)) + .collect::>()?, shapes, - } + }) } fn semantic_diagnostic( @@ -134,8 +144,20 @@ fn semantic_diagnostic( fn completion_model( (name, query_sql, authored_sql, inferred_columns, lineage, binding_codes, opt_out): ModelInput, -) -> CompletionModel { - CompletionModel { + session: Option<&NativeModelAnalysisSession>, +) -> Result { + let inferred_columns: Vec = match inferred_columns { + Some(names) => names, + None => session_model_facts(session, &name)? + .columns + .clone() + .unwrap_or_default(), + }; + let lineage: Vec = match lineage { + Some(lineage) => lineage, + None => session_model_facts(session, &name)?.lineage.clone(), + }; + Ok(CompletionModel { name, query_sql, authored_sql, @@ -143,7 +165,7 @@ fn completion_model( lineage: lineage_outputs(lineage), binding_codes, rejected_opt_out_file: opt_out, - } + }) } fn completed_row(diagnostic: CompletedDiagnostic) -> CompletedRow { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 581ba801c2..1565c82749 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod completion; pub(crate) mod metadata; pub(crate) mod registration; +pub(crate) mod session_facts; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs new file mode 100644 index 0000000000..eaf244fa10 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs @@ -0,0 +1,18 @@ +//! Model output names and lineage the semantic stages read from a finished analysis session. + +use sqlbuild_analysis::assembly::analysis_session::main::finished_model_facts::finished_model_facts; +use sqlbuild_analysis::assembly::analysis_session::models::SessionModelFacts; + +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; + +/// The facts the finished `session` kept for model `name`, which the payload left out. +pub(crate) fn session_model_facts<'a>( + session: Option<&'a NativeModelAnalysisSession>, + name: &str, +) -> Result<&'a SessionModelFacts, String> { + let finished = session + .and_then(NativeModelAnalysisSession::finished_session) + .ok_or("a model asked for session facts without a finished session")?; + finished_model_facts(finished, name) + .ok_or_else(|| format!("the finished session kept no facts for model {name}")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 29d48a1739..801971e2a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -10,7 +10,9 @@ use sqlbuild_analysis::semantic_checks::models::{ }; use sqlbuild_analysis::semantic_checks::types::RevisedBinding; +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::semantic_checks::session_facts::session_model_facts; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; @@ -22,7 +24,7 @@ type ModelInput = ( String, Option>, Vec, - Vec, + Option>, Vec<(usize, String, String, bool)>, Vec<(usize, String, String, Option, Option)>, ); @@ -84,14 +86,16 @@ impl SemanticTypeRecovery { } } -/// Plan type recovery on `catalog`'s analysis pool. +/// Plan type recovery, reading names and lineage from `session` for models with no lineage. #[pyfunction] +#[pyo3(signature = (catalog, request, session=None))] fn plan_semantic_type_recovery( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: RequestInput, + session: Option>, ) -> PyResult { - let request = recovery_request(request); + let request = recovery_request(request, session.as_deref()).map_err(compiler_error)?; let catalog = &catalog.inner; let step = py .compiler_detach(|| plan_type_recovery(&request, catalog)) @@ -109,10 +113,16 @@ pub(crate) fn lineage_outputs(lineage: Vec) -> Vec .collect() } -fn recovery_request((dialect, models, diagnostics): RequestInput) -> TypeRecoveryRequest { - TypeRecoveryRequest { +fn recovery_request( + (dialect, models, diagnostics): RequestInput, + session: Option<&NativeModelAnalysisSession>, +) -> Result { + Ok(TypeRecoveryRequest { dialect, - models: models.into_iter().map(recovery_model).collect(), + models: models + .into_iter() + .map(|model| recovery_model(model, session)) + .collect::>()?, diagnostics: diagnostics .into_iter() .map(|(id, code, is_model, resource_name)| DiagnosticOwner { @@ -122,13 +132,21 @@ fn recovery_request((dialect, models, diagnostics): RequestInput) -> TypeRecover resource_name, }) .collect(), - } + }) } fn recovery_model( (name, query_sql, inferred_columns, references, lineage, bindings, raw_bindings): ModelInput, -) -> RecoveryModel { - RecoveryModel { + session: Option<&NativeModelAnalysisSession>, +) -> Result { + let (inferred_columns, lineage) = match lineage { + Some(lineage) => (inferred_columns, lineage), + None => { + let facts = session_model_facts(session, &name)?; + (facts.columns.clone(), facts.lineage.clone()) + } + }; + Ok(RecoveryModel { name, query_sql, inferred_columns, @@ -153,7 +171,7 @@ fn recovery_model( end, }) .collect(), - } + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py index ddf0404a2c..753027e0f3 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -19,6 +19,13 @@ "amount, status", "amount, status, CAST('2026-01-01' AS TIMESTAMP) AS ordered_at" ) +_INTERMEDIATE_PATH: str = "models/intermediate/int_orders.sql" +_INTERMEDIATE: str = ( + 'MODEL (\n description "Doubled order amounts",\n);\n\n' + "SELECT customer_id, amount * 2 AS doubled_amount\n" + 'FROM __ref("stg_orders")\n' +) + _FUNCTION_PATH: str = "functions/sql/scaled_amount.sql" _SCALED_AMOUNT: str = ( "FUNCTION (\n" @@ -60,6 +67,24 @@ def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="semantic-recovered-uses-through-intermediate", + expected_code="B002", + expected_message="Unknown column 'amonut' in raw_orders", + expected_help="did you mean 'amount'?", + expected_notes=( + "raw_orders has: amount, status, customer_id, order_id", + "1 downstream uses of stg_orders.amount were not checked because of this error", + ), + files={ + **staging_files(FAILURE_BASE_STAGING.replace("amount, status", "amonut, status")), + _INTERMEDIATE_PATH: _INTERMEDIATE, + **mart_body_files( + "SELECT customer_id, SUM(doubled_amount) AS total_amount\n" + 'FROM __ref("int_orders")\nGROUP BY customer_id\n' + ), + }, + ), failure_case( name="semantic-qualified-alias-unknown-column", expected_code="B002", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8bfeb38c3b..dbcdd70b6c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -363,6 +363,8 @@ type _PivotContractRow = tuple[ ] class NativeModelAnalysisSession: + @property + def fact_models(self) -> list[str]: ... def run( self, ) -> ( @@ -481,13 +483,14 @@ def plan_semantic_type_recovery( str, list[str] | None, list[str], - list[tuple[str, list[tuple[str, str]]]], + list[tuple[str, list[tuple[str, str]]]] | None, list[tuple[int, str, str, bool]], list[tuple[int, str, str, int | None, int | None]], ] ], list[tuple[int, str, bool, str | None]], ], + session: NativeModelAnalysisSession | None = None, /, ) -> SemanticTypeRecovery: ... def check_semantic_metadata_rows( @@ -544,14 +547,15 @@ def complete_semantic_checks( str, str, str, - list[str], - list[tuple[str, list[tuple[str, str]]]], + list[str] | None, + list[tuple[str, list[tuple[str, str]]]] | None, list[str], str | None, ] ], list[tuple[str, list[tuple[str, str]]]], ], + session: NativeModelAnalysisSession | None = None, /, ) -> tuple[ str | None, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 2e925251bf..c30fde0930 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -503,6 +503,7 @@ def assemble_compiled_project( project=project, profile=profile, resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, + native_session=native_session, binding_results={ name: analysis.polyglot_analysis.binding_diagnostics for name, analysis in model_sql_analysis_by_name.items() diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index 521430a59e..eb225ef54b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -5,6 +5,7 @@ import re from collections import Counter from dataclasses import replace +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( @@ -44,6 +45,7 @@ def complete_semantic_diagnostics( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool = True, + native_session: Any | None = None, ) -> CompiledProject: """Recover type facts before metadata checks, then explain retained root diagnostics.""" if native_stage_enabled(NativeStage.SEMANTIC_CHECKS): @@ -52,6 +54,7 @@ def complete_semantic_diagnostics( profile=profile, binding_results=binding_results, resource_sql_analysis=resource_sql_analysis, + session=native_session, ) if native_project is not None: return native_project diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py index a54b887900..8afb26b9d3 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py @@ -28,7 +28,11 @@ def complete_native_diagnostics( - *, project: CompiledProject, catalog: object + *, + project: CompiledProject, + catalog: object, + session: Any | None, + session_models: frozenset[str], ) -> CompiledProject | None: """Return the recovered, explained and opt-out-checked project, or None to use Python.""" @@ -40,10 +44,15 @@ def complete_native_diagnostics( request: Any = ( project.sql_analysis_dialect, [_diagnostic_payload(item=item, ids=ids) for item in project.diagnostics], - [_model_payload(model=model, lineage=recovers) for model in project.models], + [ + _model_payload(model=model, lineage=recovers, from_session=model.name in session_models) + for model in project.models + ], [(name, list(columns.items())) for name, columns in shapes.items()], ) - deferral, completed, model_bindings, order = _native.complete_semantic_checks(catalog, request) + deferral, completed, model_bindings, order = _native.complete_semantic_checks( + catalog, request, session + ) if deferral is not None: record_semantic_deferral(kind=deferral, site=COMPLETION_DEFERRAL_SITE) return None @@ -98,14 +107,14 @@ def _diagnostic_payload( ) -def _model_payload(*, model: CompiledModel, lineage: bool) -> tuple[Any, ...]: +def _model_payload(*, model: CompiledModel, lineage: bool, from_session: bool) -> tuple[Any, ...]: opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out return ( model.name, model.query_sql, model.authored_sql, - [column.name for column in model.inferred_columns or ()], - lineage_payload(model) if lineage else [], + None if from_session else [column.name for column in model.inferred_columns or ()], + _lineage(model=model, lineage=lineage, from_session=from_session), [item.code for item in model.binding_diagnostics], None if opt_out is None else opt_out.path.name, ) @@ -151,3 +160,15 @@ def _opt_out( notes=(note,), help=help_text, ) + + +def _lineage( + *, model: CompiledModel, lineage: bool, from_session: bool +) -> list[tuple[str, list[tuple[str, str]]]] | None: + """The model's lineage rows; None asks for the session's, and only recovery reads them.""" + + if not lineage: + return [] + if from_session: + return None + return lineage_payload(model) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py index b05811fc97..76d8e40a60 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py @@ -2,11 +2,14 @@ from __future__ import annotations +from typing import Any, cast + from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledModel, CompiledProject, CompilerDiagnostic, + DynamicColumnContractProof, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic @@ -17,6 +20,25 @@ def lineage_payload(model: CompiledModel) -> list[tuple[str, list[tuple[str, str return [_output_payload(output) for output in model.fast_lineage_columns or ()] +def session_fact_models(*, project: CompiledProject, session: Any | None) -> frozenset[str]: + """Models whose output names and lineage are the finished session's, so need no payload.""" + + if session is None: + return frozenset() + kept: frozenset[str] = frozenset(cast(list[str], session.fact_models)) + return frozenset( + model.name + for model in project.models + if model.name in kept and not _output_proven(model.dynamic_column_contract) + ) + + +def _output_proven(proof: DynamicColumnContractProof | None) -> bool: + """Whether a dynamic proof replaced the analysed output names.""" + + return proof is not None and proof.output_proven + + def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[str, str]]]: return ( output.output_column, diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py index 1468d8c319..922ca72ff2 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import replace +from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs @@ -10,6 +11,7 @@ from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics +from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types from sqlbuild.compiler.semantic_checks.constants import ( COMPLETION_DEFERRAL_SITE, @@ -24,6 +26,7 @@ def completed_semantic_project( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool, + session: Any | None, ) -> CompiledProject | None: """Python's completed project, or None where any native step hands the stage back.""" @@ -31,8 +34,13 @@ def completed_semantic_project( record_semantic_deferral(kind=NATIVE_SEMANTIC_NO_CATALOG, site=COMPLETION_DEFERRAL_SITE) return None catalog: object = project.binding_catalog.native + session_models: frozenset[str] = session_fact_models(project=project, session=session) recovered: CompiledProject | None = recover_native_output_types( - project=project, binding_results=binding_results, catalog=catalog + project=project, + binding_results=binding_results, + catalog=catalog, + session=session, + session_models=session_models, ) if recovered is None: return None @@ -49,4 +57,6 @@ def completed_semantic_project( return complete_native_diagnostics( project=replace(recovered, diagnostics=(*recovered.diagnostics, *metadata)), catalog=catalog, + session=session, + session_models=session_models, ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py index c4ade28bdb..d433974d15 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py @@ -39,6 +39,8 @@ def recover_native_output_types( project: CompiledProject, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], catalog: object, + session: Any | None, + session_models: frozenset[str], ) -> CompiledProject | None: """Return `recover_output_types`'s project, or None where Python must recover the types.""" @@ -49,7 +51,13 @@ def recover_native_output_types( request: Any = ( project.sql_analysis_dialect, [ - _model_payload(model=model, binding_results=binding_results, ids=ids, raw_ids=raw_ids) + _model_payload( + model=model, + binding_results=binding_results, + ids=ids, + raw_ids=raw_ids, + from_session=model.name in session_models, + ) for model in project.models ], [ @@ -62,7 +70,9 @@ def recover_native_output_types( for item in project.diagnostics ], ) - recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery(catalog, request) + recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery( + catalog, request, session + ) if recovery.status == NATIVE_TYPE_RECOVERY_UNCHANGED: return project if recovery.status == NATIVE_TYPE_RECOVERY_DEFERRED: @@ -120,6 +130,7 @@ def _model_payload( binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], ids: dict[CompilerDiagnostic, int], raw_ids: dict[SqlBindingDiagnostic, int], + from_session: bool, ) -> tuple[Any, ...]: raws: tuple[SqlBindingDiagnostic, ...] = ( binding_results.get(model.name, ()) if model.binding_diagnostics else () @@ -128,10 +139,10 @@ def _model_payload( model.name, model.query_sql, None - if model.inferred_columns is None + if from_session or model.inferred_columns is None else [column.name for column in model.inferred_columns], [reference.ref_name for reference in model.references], - lineage_payload(model), + None if from_session else lineage_payload(model), [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], [(raw_ids[raw], raw.code, raw.message, raw.start, raw.end) for raw in raws], ) diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py index 762329e0a7..ca8f3e64ce 100644 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Any + from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project @@ -14,6 +16,7 @@ def complete_native_semantic_diagnostics( profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], resource_sql_analysis: bool, + session: Any | None = None, ) -> CompiledProject | None: """Return the completed project, or None where Python must complete its diagnostics.""" @@ -22,4 +25,5 @@ def complete_native_semantic_diagnostics( profile=profile, binding_results=binding_results, resource_sql_analysis=resource_sql_analysis, + session=session, ) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py index 31dd072d29..10a8a6c3d3 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py @@ -40,3 +40,18 @@ class GeneratedMetadataParityTestCase: dialects: tuple[str, ...] expected_minimum_native: int expected_minimum_families: dict[str, int] + + +@dataclass(frozen=True) +class SessionCompletionTestCase: + """Seeded preview compiles whose completion reads analysed models from the native session.""" + + description: str + corpus: str + seed: int + count: int + model_count: int + expected_session_models: int + expected_payload_models: int + expected_proven_outputs: int + expected_minimum_codes: dict[str, int] diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 97ed5ccb0d..8cbb77988d 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,6 +13,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project +import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( @@ -26,10 +27,14 @@ from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics +from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( complete_native_semantic_diagnostics, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( + generated_analysis_files, +) _RELATIONS: dict[str, tuple[str, ...]] = { '__source("raw_orders")': ( @@ -81,6 +86,7 @@ class SemanticInputs: profile: ExpressionInferenceProfile binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]] resource_sql_analysis: bool + native_session: Any | None = None def _typo(rng: random.Random, column: str) -> str: @@ -173,9 +179,13 @@ def generated_semantic_files( def captured_semantic_inputs( - *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch + *, + project_dir: Path, + files: dict[str, str], + monkeypatch: pytest.MonkeyPatch, + engine: str = "python", ) -> SemanticInputs: - """Compile with the Python engine and keep the semantic completion stage's inputs.""" + """Compile with `engine` and keep the semantic completion stage's inputs.""" for relative_path, contents in files.items(): path: Path = project_dir / relative_path @@ -188,7 +198,7 @@ def capture(**arguments: Any) -> CompiledProject: return complete_semantic_diagnostics(**arguments) with monkeypatch.context() as patch: - patch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + patch.setenv(COMPILER_ENGINE_ENV_VAR, engine) patch.setattr(assembly_project, "complete_semantic_diagnostics", capture) _ = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), @@ -248,9 +258,51 @@ def native_completion(inputs: SemanticInputs) -> CompiledProject | None: profile=inputs.profile, binding_results=inputs.binding_results, resource_sql_analysis=inputs.resource_sql_analysis, + session=inputs.native_session, ) +def session_corpus_files(*, corpus: str, rng: random.Random, model_count: int) -> dict[str, str]: + """One generated project from the failing-semantics or the analysis-session corpus.""" + + generators: dict[str, Callable[[], dict[str, str]]] = { + "semantic": lambda: generated_semantic_files( + rng=rng, model_count=model_count, require_analysis=False + ), + "analysis": lambda: generated_analysis_files(rng=rng, model_count=model_count), + } + return generators[corpus]() + + +def proven_output_count(project: CompiledProject) -> int: + """How many models a dynamic pivot proof gave their output names.""" + + return sum( + model.dynamic_column_contract is not None and model.dynamic_column_contract.output_proven + for model in project.models + ) + + +def without_session(inputs: SemanticInputs) -> SemanticInputs: + """The same stage inputs with every model's output names and lineage in the payload.""" + + return replace(inputs, native_session=None) + + +def record_session_models(*, monkeypatch: pytest.MonkeyPatch) -> list[frozenset[str]]: + """Keep each completion's models whose output names and lineage the session supplies.""" + + selections: list[frozenset[str]] = [] + + def recorded(**arguments: Any) -> frozenset[str]: + selected: frozenset[str] = session_fact_models(**arguments) + selections.append(selected) + return selected + + monkeypatch.setattr(semantic_stage, "session_fact_models", recorded) + return selections + + def completion_view(project: CompiledProject) -> tuple[object, ...]: """Everything the stage decides: diagnostics and each model's binding and type facts.""" diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 96d98e0bbb..b597aa098c 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -14,6 +14,7 @@ from tests.integration.src.sqlbuild.compiler.semantic_checks._test_types import ( DeferredSemanticTestCase, GeneratedSemanticParityTestCase, + SessionCompletionTestCase, ) from tests.integration.src.sqlbuild.compiler.semantic_checks.helpers import ( SemanticInputs, @@ -24,9 +25,13 @@ generated_semantic_files, native_completion, note_kinds, + proven_output_count, python_completion, record_native_statuses, + record_session_models, + session_corpus_files, with_dialect, + without_session, ) _NOTE_KINDS: tuple[str, ...] = ("downstream output uses", "downstream uses of", " has: ") @@ -154,5 +159,83 @@ def test_given_deferred_projects_when_completing_then_python_completes_and_recor assert deferral_records(record_dir) == test_case.expected_kinds +@pytest.mark.parametrize( + "test_case", + [ + SessionCompletionTestCase( + description="failing semantics corpus", + corpus="semantic", + seed=20261010, + count=6, + model_count=12, + expected_session_models=44, + expected_payload_models=28, + expected_proven_outputs=0, + expected_minimum_codes={"B002": 1, "downstream output uses": 1}, + ), + SessionCompletionTestCase( + description="analysis session corpus with pivots, stars and set operations", + corpus="analysis", + seed=20261011, + count=4, + model_count=16, + expected_session_models=67, + expected_payload_models=29, + expected_proven_outputs=11, + expected_minimum_codes={"B002": 1}, + ), + ], + ids=lambda case: case.description, +) +def test_given_preview_compiles_when_completing_from_the_session_then_matches_payload_and_python( + test_case: SessionCompletionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + captured: list[SemanticInputs] = [ + captured_semantic_inputs( + project_dir=tmp_path / f"project_{index}", + files=session_corpus_files( + corpus=test_case.corpus, rng=rng, model_count=test_case.model_count + ), + monkeypatch=monkeypatch, + engine="native-preview", + ) + for index in range(test_case.count) + ] + selections: list[frozenset[str]] = record_session_models(monkeypatch=monkeypatch) + codes: Counter[str] = Counter() + python_views: list[tuple[object, ...]] = [] + payload_views: list[tuple[object, ...]] = [] + session_views: list[tuple[object, ...]] = [] + for inputs in captured: + python: CompiledProject = python_completion(inputs=inputs, monkeypatch=monkeypatch) + codes.update(item.code for item in python.diagnostics) + codes.update(note_kinds(python, _NOTE_KINDS)) + python_views.append(completion_view(python)) + payload_views.append(completion_view(completed_natively(without_session(inputs)))) + session_views.append(completion_view(completed_natively(inputs))) + session_models: int = sum(len(selected) for selected in selections) + model_count: int = sum(len(inputs.project.models) for inputs in captured) + + assert all(inputs.native_session is not None for inputs in captured) + assert session_views == python_views + assert payload_views == python_views + assert ( + session_models, + model_count - session_models, + sum(proven_output_count(inputs.project) for inputs in captured), + ) == ( + test_case.expected_session_models, + test_case.expected_payload_models, + test_case.expected_proven_outputs, + ) + assert { + code: min(codes[code], minimum) + for code, minimum in test_case.expected_minimum_codes.items() + } == test_case.expected_minimum_codes + + if __name__ == "__main__": pytest.main([__file__, "-vv"])