From 804418b1db38fcf6efc73b06e09d13711288348f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:04:52 +0100 Subject: [PATCH 1/6] perf: read native records directly in compile-phase consumers Compile-phase consumers (analysis session requests, declared relations, source keys, lineage counts, SQL test assembly, scope index, model loop) now read native model, source and compiled-model records directly instead of materialising Python record objects. Generic record helpers resolve specs by type pointer. Output is byte-identical; main-thread instructions on the dense 10k cold compile drop by about 10G (13.5%). --- .../_helpers/analysis/binding_catalog.rs | 12 + .../analysis_session/model_analyses.rs | 162 +++++++---- .../analysis_session/profile_inputs.rs | 17 +- .../_helpers/analysis_session/session.rs | 2 +- .../compile_driver/assembly_analysis.rs | 59 +++- .../compile_driver/integration_loaders.rs | 76 +++--- .../compiled_model_analysis.rs | 10 + .../compiled_project/compiled_model_view.rs | 5 + .../_helpers/compiled_project/loop_classes.rs | 19 +- .../compiled_project/model_input_view.rs | 10 + .../compiled_project/model_loop_host.rs | 9 +- .../_helpers/compiled_project/schema_facts.rs | 8 +- .../_helpers/compiled_project/source_loop.rs | 35 ++- .../_helpers/compiled_project/source_views.rs | 38 +++ .../_helpers/discovery/project_facts.rs | 14 +- .../_helpers/discovery/sql_test_blocks.rs | 28 +- .../_helpers/lineage/column_lineage_graph.rs | 86 ++++-- .../_helpers/lineage/project_lineage.rs | 139 ++++++---- .../_helpers/planner_selection/unit_tests.rs | 11 +- .../project_assembly/declared_relations.rs | 256 +++++++++++++++--- .../bindings/_helpers/records/constants.rs | 2 + .../src/bindings/_helpers/records/fields.rs | 116 +++++++- .../bindings/_helpers/records/registration.rs | 7 +- .../bindings/_helpers/rules/early_sql_lint.rs | 4 +- .../bindings/_helpers/rules/relation_keys.rs | 76 +++++- .../bindings/_helpers/scopes/scope_index.rs | 71 ++++- .../_helpers/semantic_shapes/shapes.rs | 47 +++- .../_helpers/sql_test_glue/assembly.rs | 34 ++- fensu.toml | 15 +- .../compiler/compiled_project/_test_types.py | 9 + .../compiled_project/test_native_sources.py | 39 +++ 31 files changed, 1093 insertions(+), 323 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 0938071139..00b92ba4c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -1,5 +1,7 @@ //! The compile-lifetime binding catalog: known relation shapes around one native project catalog. +use std::sync::atomic::{AtomicBool, Ordering}; + use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, @@ -50,12 +52,19 @@ pub(crate) struct BindingCatalog { pub(crate) expression_shapes: Py, #[pyo3(get)] pub(crate) native: Py, + /// Whether binding preparation has added relations to `schemas`. + pub(crate) extended: AtomicBool, } /// A catalog's dialect, case mode, known functions and known types. pub(crate) type CatalogKey = (String, bool, Vec, Vec); impl BindingCatalog { + /// Whether `schemas` holds exactly the relations it was created with and later given. + pub(crate) fn unextended(&self) -> bool { + !self.extended.load(Ordering::Relaxed) + } + /// A catalog for `key` that starts with `relations`. pub(crate) fn create( py: Python<'_>, @@ -80,6 +89,7 @@ impl BindingCatalog { quoted_ignore_case, analysis_shapes: PyDict::new(py).unbind(), expression_shapes: PyDict::new(py).unbind(), + extended: AtomicBool::new(false), native: Py::new(py, native)?, }) } @@ -103,6 +113,7 @@ impl BindingCatalog { } } if !additions.is_empty() { + self.extended.store(true, Ordering::Relaxed); let relations: Relations = conversions::relations(additions.as_any())?; self.native.borrow_mut(py).inner.update_relations(relations); } @@ -207,6 +218,7 @@ impl BindingCatalog { analysis_shapes: self.analysis_shapes.bind(py).copy()?.unbind(), expression_shapes: self.expression_shapes.bind(py).copy()?.unbind(), native: Py::new(py, native)?, + extended: AtomicBool::new(true), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 04d020dd94..81960c08d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -2,23 +2,27 @@ use std::path::PathBuf; -use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; +use crate::bindings::_helpers::project_assembly::declared_relations::DeclaredShapes; + use crate::bindings::_helpers::analysis_session::profile_inputs::{ case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::session::{ - CacheStatsRow, NativeModelAnalysisSession, RequestRow, start_model_analysis_session, + CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, + start_model_analysis_session, }; use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; use crate::bindings::_helpers::compiled_project::model_input_view::{ - AttachedSchema, attached_schema, input_model_name, + AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, + input_reference_rows, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -37,7 +41,7 @@ const STORE_FILE_NAME: &str = "model-analyses.bin"; const STORE_VERSION: &str = "native-model-analysis-store-v1"; /// What one compile's model analysis reads: the analysed models and the relation facts. -pub(crate) struct ModelAnalysisRequest<'py> { +pub(crate) struct ModelAnalysisRequest<'a, 'py> { pub(crate) model_inputs: Vec>, pub(crate) profile: Bound<'py, PyAny>, pub(crate) column_types: Bound<'py, PyAny>, @@ -47,6 +51,8 @@ pub(crate) struct ModelAnalysisRequest<'py> { pub(crate) rich_type_inference: bool, /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. pub(crate) cache: Bound<'py, PyAny>, + /// Native rows of the three declaration maps, when the caller built them alongside. + pub(crate) declared: Option<&'a DeclaredShapes>, } /// Each analysed model's position in the finished session, those that succeeded, and the session. @@ -59,7 +65,7 @@ pub(crate) struct ModelAnalyses<'py> { /// Each model's analysis by name and the finished session; a native internal failure raises. pub(crate) fn analyze_models<'py>( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, ) -> PyResult> { let py: Python<'py> = request.profile.py(); if request.model_inputs.is_empty() { @@ -71,7 +77,7 @@ pub(crate) fn analyze_models<'py>( } let catalog: Bound<'py, PyAny> = native_field(&request.profile, "binding_catalog")?; let (rules, adapter_rules) = nullability_rules(&request.profile)?; - let session_request: RequestRow = session_request(request, &catalog, rules)?.extract()?; + let session_request: RequestRow = session_request(request, &catalog, rules)?; let (cache, adapter) = match adapter_rules { Some((adapter, nullability)) => (None, Some((adapter, nullability))), None => (analysis_store(request)?, None), @@ -90,36 +96,71 @@ pub(crate) fn analyze_models<'py>( /// The session request row, in the order the native session reads it. fn session_request<'py>( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, rules: Vec<(String, String)>, -) -> PyResult> { - let py: Python<'py> = catalog.py(); +) -> PyResult { let profile: &Bound<'py, PyAny> = &request.profile; - let mut families: Vec<(String, Vec<_>)> = Vec::new(); + let mut families: Vec<(String, Vec)> = Vec::new(); for item in mapping_items(&request.dynamic_families)?.try_iter()? { let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } - let items: Vec> = vec![ - profile_dialect(profile)?.into_bound_py_any(py)?, - case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)? - .into_bound_py_any(py)?, - return_types(profile)?.into_bound_py_any(py)?, - rules.into_bound_py_any(py)?, - request.rich_type_inference.into_bound_py_any(py)?, - shape_rows(&request.column_types)?.into_bound_py_any(py)?, - shape_rows(&request.nullability)?.into_bound_py_any(py)?, - shape_rows(&request.complete_schemas)?.into_bound_py_any(py)?, - shape_rows(&native_field(catalog, "schemas")?)?.into_bound_py_any(py)?, - families.into_bound_py_any(py)?, - model_rows(py, &request.model_inputs)?, - ]; - Ok(PyTuple::new(py, items)?.into_any()) + let (types, nullability, complete, schemas): (Shapes, Shapes, Shapes, Shapes) = + match declared_rows(request, catalog)? { + Some(rows) => rows, + None => ( + shape_rows(&request.column_types)?, + shape_rows(&request.nullability)?, + shape_rows(&request.complete_schemas)?, + shape_rows(&native_field(catalog, "schemas")?)?, + ), + }; + Ok(( + profile_dialect(profile)?, + case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)?, + return_types(profile)?, + Some(rules), + request.rich_type_inference, + types, + nullability, + complete, + schemas, + families, + model_rows(profile.py(), &request.model_inputs)?, + )) +} + +/// The caller's native declaration rows when they still match the maps and catalog they mirror. +fn declared_rows( + request: &ModelAnalysisRequest<'_, '_>, + catalog: &Bound<'_, PyAny>, +) -> PyResult> { + let Some(declared) = request.declared else { + return Ok(None); + }; + let Ok(binding) = catalog.downcast::() else { + return Ok(None); + }; + let complete: &Shapes = declared.complete.rows(); + let unchanged: bool = binding.borrow().unextended() + && binding.borrow().schemas.bind(catalog.py()).len() == complete.len() + && request.complete_schemas.len()? == complete.len() + && request.column_types.len()? == declared.types.rows().len() + && request.nullability.len()? == declared.nullability.rows().len(); + if !unchanged { + return Ok(None); + } + Ok(Some(( + declared.types.rows().clone(), + declared.nullability.rows().clone(), + complete.clone(), + complete.clone(), + ))) } /// The analysis store's path and environment, or None where Python bypasses its cache. -fn analysis_store(request: &ModelAnalysisRequest<'_>) -> PyResult> { +fn analysis_store(request: &ModelAnalysisRequest<'_, '_>) -> PyResult> { let context: &Bound<'_, PyAny> = &request.cache; if context.is_none() { return Ok(None); @@ -138,7 +179,7 @@ fn analysis_store(request: &ModelAnalysisRequest<'_>) -> PyResult( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, session: &Bound<'py, NativeModelAnalysisSession>, ) -> PyResult> { @@ -249,25 +290,15 @@ fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<( } /// Each model's native request row, in analysis order. -fn model_rows<'py>( - py: Python<'py>, - model_inputs: &[Bound<'py, PyAny>], -) -> PyResult> { +fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyResult> { let enforced: Bound<'py, PyAny> = enum_member(py, RecordId::ContractPolicy, "ENFORCED")?; let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; - let mut rows: Vec> = Vec::with_capacity(model_inputs.len()); + let mut rows: Vec = Vec::with_capacity(model_inputs.len()); for model_input in model_inputs { let values: Bound<'py, PyAny> = native_field(model_input, "config")?.getattr("values")?; let get = |key: &str| mapping_get(&values, key); - let query_sql: Bound<'py, PyAny> = PyString::new( - py, - &analysis_cursor_sql( - py, - &native_field(model_input, "query_sql")?.extract::()?, - &get("cursor_type")?, - )?, - ) - .into_any(); + let query_sql: String = + analysis_cursor_sql(py, &input_query_sql(model_input)?, &get("cursor_type")?)?; let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { @@ -276,14 +307,7 @@ fn model_rows<'py>( placeholders.push((key.str()?.extract()?, value.str()?.extract()?)); } } - let mut references: Vec<(String, String)> = Vec::new(); - for row in native_field(model_input, "reference_rows")?.try_iter()? { - let row: Bound<'py, PyAny> = row?; - references.push(( - row.get_item(0)?.str()?.extract()?, - row.get_item(1)?.extract()?, - )); - } + let references: Vec<(String, String)> = analysed_references(model_input)?; let schema: Option> = attached_schema(model_input)?; let recovers: bool = get("contract")?.eq(&enforced)? || schema @@ -301,18 +325,36 @@ fn model_rows<'py>( Some(schema) => family_rows(&schema.dynamic_columns)?, None => Vec::new(), }; - let row: Vec> = vec![ - input_model_name(model_input)?.into_bound_py_any(py)?, + rows.push(( + input_model_name(model_input)?, query_sql, - placeholders.into_bound_py_any(py)?, - references.into_bound_py_any(py)?, - recovers.into_bound_py_any(py)?, - snapshot_columns.into_bound_py_any(py)?, - families.into_bound_py_any(py)?, - ]; - rows.push(PyTuple::new(py, row)?.into_any()); + placeholders, + references, + recovers, + snapshot_columns, + families, + )); + } + Ok(rows) +} + +/// Each reference's `(kind, name)`; a native input answers from its record. +fn analysed_references(model_input: &Bound<'_, PyAny>) -> PyResult> { + if model_input.is_instance_of::() { + return Ok(input_reference_rows(model_input)? + .into_iter() + .map(|(kind, name, _, _)| (kind, name)) + .collect()); + } + let mut references: Vec<(String, String)> = Vec::new(); + for row in native_field(model_input, "reference_rows")?.try_iter()? { + let row: Bound<'_, PyAny> = row?; + references.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); } - Ok(PyList::new(py, rows)?.into_any()) + Ok(references) } /// `str(value or default)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 98391124cc..7348587e6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -64,16 +64,21 @@ pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { let mut rows: Shapes = Vec::new(); for item in mapping_items(shapes)?.try_iter()? { let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; - let mut pairs: Pairs = Vec::new(); - for column in mapping_items(&shape)?.try_iter()? { - let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; - pairs.push((column, value.str()?.extract()?)); - } - rows.push((name, pairs)); + rows.push((name, text_pairs(&shape)?)); } Ok(rows) } +/// `{column: value}` as ordered pairs, each value as `str(value)`. +pub(crate) fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { + let mut pairs: Pairs = Vec::new(); + for column in mapping_items(shape)?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + Ok(pairs) +} + /// A `{key: value}` text mapping's items in order. pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { mapping_items(mapping)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index ae07bd9735..6d7a78e232 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -48,7 +48,7 @@ type LineageItem = (String, u8, u8, Vec<(String, String, String)>); /// `(name, pivot column, value column, aggregate, type, name pattern)`. pub(crate) type FamilyRow = (String, String, String, String, String, Option); /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -type ModelRow = ( +pub(crate) type ModelRow = ( String, String, Pairs, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 0dd7aef323..60ec1d1a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -1,14 +1,17 @@ //! Project assembly's inference profile, binding catalog, source shapes and model analysis. +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; +use crate::bindings::_helpers::analysis_session::profile_inputs::text_pairs; use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; @@ -20,10 +23,11 @@ use crate::bindings::_helpers::compile_driver::stage_work::{ EXPRESSION_SHAPES, MODEL_ANALYSIS, count_stage, }; use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; +use crate::bindings::_helpers::compiled_project::source_views::read_source_view; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ - DeclaredRelations, declared_relations, + DeclaredRelations, DeclaredShapes, declared_relations, }; use crate::bindings::_helpers::records::construct::{enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -47,6 +51,8 @@ pub(crate) struct AnalysisContext<'py> { pub(crate) dynamic_families: Bound<'py, PyAny>, /// The model analyses, or `None` when SQL analysis does not run. pub(crate) analyses: Option>, + /// The native rows of `column_types`, `nullability` and `complete_schemas`. + pub(crate) declared: DeclaredShapes, } /// Whether `model_input` is selected for analysis by `names` (`None` selects every model). @@ -87,7 +93,7 @@ pub(crate) fn analysis_context<'py>( &native_field(inputs, "effective_settings")?.getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let (column_types, nullability, complete_schemas): DeclaredRelations<'py> = + let (column_types, nullability, complete_schemas, declared): DeclaredRelations<'py> = declared_relations(inputs.clone())?; let (column_types, nullability, complete_schemas): ( Bound<'py, PyAny>, @@ -109,9 +115,11 @@ pub(crate) fn analysis_context<'py>( complete_schemas, dynamic_families, analyses: None, + declared, }; if context.sql_analysis_enabled.is_truthy()? { - record_expression_shapes(request, &context)?; + let declared: DeclaredShapes = std::mem::take(&mut context.declared); + context.declared = record_expression_shapes(request, &context, declared)?; } let catalog: Bound<'py, PyAny> = native_field(&context.profile, "binding_catalog")?; native_field(&catalog, "native")? @@ -252,15 +260,15 @@ fn function_return_types<'py>( fn record_expression_shapes<'py>( request: &AssemblyRequest, context: &AnalysisContext<'py>, -) -> PyResult<()> { + mut declared: DeclaredShapes, +) -> PyResult { let py: Python<'py> = context.profile.py(); let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let mut sources: Vec> = Vec::new(); let mut expressions: Vec> = Vec::new(); for source_input in native_field(inputs, "source_inputs")?.try_iter()? { let source_input: Bound<'py, PyAny> = source_input?; - let expression: Bound<'py, PyAny> = - native_field(&source_input, "source_entry")?.getattr("expression")?; + let expression: Bound<'py, PyAny> = source_expression(&source_input)?; if expression.is_truthy()? { sources.push(source_input); expressions.push(expression); @@ -295,8 +303,7 @@ fn record_expression_shapes<'py>( if shape.is_none() { continue; } - let name: Bound<'py, PyAny> = - native_field(source_input, "source_entry")?.getattr("name")?; + let name: Bound<'py, PyAny> = source_name(source_input)?; let declared_types: Bound<'py, PyAny> = mapping_get_or(&context.column_types, &name, PyDict::new(py))?; let declared_shape: Bound<'py, PyDict> = PyDict::new(py); @@ -316,9 +323,40 @@ fn record_expression_shapes<'py>( mapping_get_or(&context.nullability, &name, PyDict::new(py))?; nullability.set_item(&column, mapping_get_or(&known, &column, &unknown)?)?; } - context.nullability.set_item(&name, nullability)?; + context.nullability.set_item(&name, &nullability)?; + let key: String = name.extract()?; + let shape_pairs: Pairs = text_pairs(declared_shape.as_any())?; + declared.complete.set(key.clone(), shape_pairs.clone()); + declared.types.set(key.clone(), shape_pairs); + declared + .nullability + .set(key, text_pairs(nullability.as_any())?); + } + Ok(declared) +} + +/// `source_input.source_entry.expression`; a native source answers from its attached record. +fn source_expression<'py>(source_input: &Bound<'py, PyAny>) -> PyResult> { + match read_source_view(source_input, |source, py| { + source + .attached() + .expression + .as_deref() + .into_bound_py_any(py) + }) { + Some(expression) => expression, + None => native_field(source_input, "source_entry")?.getattr("expression"), + } +} + +/// `source_input.source_entry.name`; a native source answers from its attached record. +fn source_name<'py>(source_input: &Bound<'py, PyAny>) -> PyResult> { + match read_source_view(source_input, |source, py| { + Ok(PyString::new(py, &source.attached().name).into_any()) + }) { + Some(name) => name, + None => native_field(source_input, "source_entry")?.getattr("name"), } - Ok(()) } /// Each selected model's native analysis, timed as `model_analysis_ms`, when SQL analysis runs. @@ -366,6 +404,7 @@ fn model_analyses<'py>( dynamic_families: context.dynamic_families.clone(), rich_type_inference: rich, cache, + declared: Some(&context.declared), }) .map(Some) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 4169bc2566..1d1d6a7af9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -35,48 +35,50 @@ pub(crate) fn integration_loader_functions<'py>( py: Python<'py>, source_files: &Bound<'py, PyAny>, ) -> PyResult> { + let mut declared: Vec<(String, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for source_file in source_files.try_iter()? { + let source_file: Bound<'py, PyAny> = source_file?; + for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { + let source_entry: Bound<'py, PyAny> = source_entry?; + let loader: Bound<'py, PyAny> = native_field(&source_entry, "integration_loader")?; + if loader.is_none() { + continue; + } + let kind: String = native_field(&loader, "kind")?.extract()?; + declared.push((kind, source_file.clone(), source_entry)); + } + } + if declared.is_empty() { + return Ok(PyTuple::empty(py).into_any()); + } let external: Bound<'py, PyAny> = enum_member(py, RecordId::LoaderConnectionMode, "EXTERNAL")?; let mut loaders: Vec> = Vec::new(); for integration in &INTEGRATIONS { let kind: String = host::entry(py, integration.kind_constant)?.extract()?; let factory: Bound<'py, PyAny> = host::entry(py, integration.loader_factory)?; - for source_file in source_files.try_iter()? { - let source_file: Bound<'py, PyAny> = source_file?; - for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { - let source_entry: Bound<'py, PyAny> = source_entry?; - let declared: Bound<'py, PyAny> = - native_field(&source_entry, "integration_loader")?; - if declared.is_none() - || native_field(&declared, "kind")?.extract::()? != kind - { - continue; - } - let name: String = integration_loader_name( - &kind, - &native_field(&source_entry, "name")?.extract::()?, - ); - let function: Bound<'py, PyAny> = call_with( - &factory, - &[ - ("source_entry", source_entry), - ("name", name.clone().into_pyobject(py)?.into_any()), - ], - )?; - loaders.push(new_record_with( - py, - RecordId::DiscoveredLoaderFunction, - &[ - ("file_path", native_field(&source_file, "file_path")?), - ( - "relative_path", - native_field(&source_file, "relative_path")?, - ), - ("name", name.into_pyobject(py)?.into_any()), - ("function", function), - ("connection_mode", external.clone()), - ], - )?); - } + for (_, source_file, source_entry) in declared.iter().filter(|loader| loader.0 == kind) { + let name: String = integration_loader_name( + &kind, + &native_field(source_entry, "name")?.extract::()?, + ); + let function: Bound<'py, PyAny> = call_with( + &factory, + &[ + ("source_entry", source_entry.clone()), + ("name", name.clone().into_pyobject(py)?.into_any()), + ], + )?; + loaders.push(new_record_with( + py, + RecordId::DiscoveredLoaderFunction, + &[ + ("file_path", native_field(source_file, "file_path")?), + ("relative_path", native_field(source_file, "relative_path")?), + ("name", name.into_pyobject(py)?.into_any()), + ("function", function), + ("connection_mode", external.clone()), + ], + )?); } } Ok(PyTuple::new(py, loaders)?.into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 83db2ce746..8871aa9d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -195,6 +195,16 @@ impl AnalysisRecord { .map(|lineage| lineage.clone_ref(py)) } + /// The number of upstream sources across the model's lineage rows, or `None` without lineage. + pub(crate) fn lineage_edge_count(&self) -> Option { + match &self.lineage { + None => None, + Some(LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows)) => { + Some(rows.iter().map(|row| row.sources.len()).sum()) + } + } + } + /// The model's `CompilerDiagnostic` tuple, built once. pub(crate) fn binding_tuple( &self, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index a9f7b3142f..cd7cf04bb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -735,6 +735,11 @@ impl NativeCompiledModel { }) } + /// The upstream source count of the model's lineage facts, `None` without them. + pub(crate) fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) + } + #[getter] pub(crate) fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index fb9f23ff34..3660b9432b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -389,17 +389,14 @@ pub(crate) fn model_loop_classes<'py>( )?; let record: Bound<'py, PyDict> = dict( py, - vec![ - ("replace", imported(py, "dataclasses", "replace")?), - ( - "declaration_rows", - pyo3::wrap_pyfunction!( - crate::bindings::_helpers::compiled_project::records::declaration_rows, - py - )? - .into_any(), - ), - ], + vec![( + "declaration_rows", + pyo3::wrap_pyfunction!( + crate::bindings::_helpers::compiled_project::records::declaration_rows, + py + )? + .into_any(), + )], )?; dict( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 9b56f17be5..bc37f09b55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -535,6 +535,16 @@ pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(record.query_sql.to_string()) + }); + } + native_field(model_input, "query_sql")?.extract() +} + /// Each reference of `model_input` as `(kind, name, package, call argument count)`. pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = model_input.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 79036fdaeb..38a437789d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; use crate::bindings::_helpers::render::declaration_expansion::{ ExpandedDeclarations, ScannedReference, span_rows, }; @@ -103,8 +103,6 @@ pub(crate) struct FinishedModel<'py> { #[derive(FromPyObject)] #[pyo3(from_item_all)] pub(crate) struct RecordClasses<'py> { - /// `dataclasses.replace`. - replace: Bound<'py, PyAny>, /// `declaration_rows`, the facts rows of the model's local enums and constants. declaration_rows: Bound<'py, PyAny>, } @@ -159,10 +157,7 @@ pub(crate) fn finished_record<'py>( .extract()?; let replaced: Bound<'py, PyDict> = PyDict::new(py); replaced.set_item(VALUES_ARGUMENT, &stripped)?; - let config: Bound<'py, PyAny> = inputs - .classes - .replace - .call((&finished.config,), Some(&replaced))?; + let config: Bound<'py, PyAny> = replaced_keywords(&finished.config, &replaced)?; let mut declaration_usages: Vec> = unique(py, usages)?; declaration_usages.extend(finished.named_usages); let (macro_sql, macro_spans, macro_deps, macro_usages) = match finished.macros { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs index d72ac55f30..611a222050 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs @@ -1,6 +1,6 @@ //! Native facts of declared schema columns and families, read once from their Python specs. -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, record_attribute_opt}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiled_project::models::{ DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, @@ -116,7 +116,11 @@ pub(crate) fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { /// An attribute's value, `None` when it is missing or `None`. fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - Ok(spec.getattr_opt(name)?.filter(|value| !value.is_none())) + let value: Option> = match record_attribute_opt(spec, name) { + Some(value) => value?, + None => spec.getattr_opt(name)?, + }; + Ok(value.filter(|value| !value.is_none())) } fn optional<'py, T: pyo3::FromPyObject<'py>>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index a665075ef2..4f3e3d1204 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -1,7 +1,7 @@ //! `build_source_inputs`: every discovered source attached natively into the compiled project. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; @@ -415,8 +415,39 @@ fn expanded_audit( replaced_spec(audit, &changes) } +/// `dataclasses.replace(value, **changes)`, or `value` itself when every change keeps its field. fn replaced_spec(value: &Bound<'_, PyAny>, changes: &Bound<'_, PyDict>) -> PyResult> { - replaced(value.py(), value, changes) + for (name, changed) in changes.iter() { + let current: Bound<'_, PyAny> = native_field(value, &name.extract::()?)?; + if !same_value(¤t, &changed)? { + return replaced(value.py(), value, changes); + } + } + Ok(value.clone().unbind()) +} + +/// Whether `changed` is `current` or an equal value of the same type, element by element. +fn same_value(current: &Bound<'_, PyAny>, changed: &Bound<'_, PyAny>) -> PyResult { + if current.is(changed) { + return Ok(true); + } + if !current.get_type().is(changed.get_type()) { + return Ok(false); + } + if let (Ok(current), Ok(changed)) = + (current.downcast::(), changed.downcast::()) + { + if current.len() != changed.len() { + return Ok(false); + } + for (left, right) in current.iter().zip(changed.iter()) { + if !left.is(&right) { + return Ok(false); + } + } + return Ok(true); + } + current.eq(changed) } /// `validate_source_expression_syntax`: the expression parses as a FROM target. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 59d85a8209..48690285f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -5,15 +5,18 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; +use crate::bindings::_helpers::rules::relation_keys::audit_key_rows; const SOURCE_RESOURCE: &str = "source"; @@ -43,9 +46,16 @@ pub(crate) struct AttachedSource { pub(crate) usages: Py, } +/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. +pub(crate) struct SourceKeyFacts { + pub(crate) columns: Vec, + pub(crate) audits: Vec<(String, Option)>, +} + /// One attached source held by the compiled project. pub(crate) struct SourceRecord { attached: AttachedSource, + key_facts: PyOnceLock, /// The namespace native project assembly resolved, once it ran. namespace: Option, attached_entry: PyOnceLock>, @@ -57,6 +67,7 @@ impl SourceRecord { pub(crate) fn new(attached: AttachedSource) -> Self { Self { attached, + key_facts: PyOnceLock::new(), namespace: None, attached_entry: PyOnceLock::new(), assembled_entry: PyOnceLock::new(), @@ -69,6 +80,16 @@ impl SourceRecord { &self.attached } + /// The source's declared column facts and audit key rows, read once from its expanded specs. + pub(crate) fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { + self.key_facts.get_or_try_init(py, || { + Ok(SourceKeyFacts { + columns: python_column_facts(self.attached.columns.bind(py).as_any(), None)?, + audits: audit_key_rows(self.attached.audits.bind(py).as_any())?, + }) + }) + } + /// Keep the namespace native project assembly resolved for the source. pub(crate) fn set_namespace(&mut self, namespace: SourceNamespace) { self.namespace = Some(namespace); @@ -120,6 +141,23 @@ pub(crate) fn replaced( Ok(replaced_keywords(value, changes)?.unbind()) } +/// Read the source record behind a native source view, or `None` for any other object. +pub(crate) fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + let py: Python<'py> = view.py(); + if let Ok(input) = view.cast::() { + let input: &NativeSourceInput = input.get(); + return Some(read_source(&input.owner, input.index, py, read)); + } + let Ok(compiled) = view.cast::() else { + return None; + }; + let compiled: &NativeCompiledSource = compiled.get(); + Some(read_source(&compiled.owner, compiled.index, py, read)) +} + fn read_source<'py, T>( owner: &Py, index: usize, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 66ae0dd634..fe0176e6ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -1,6 +1,7 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -95,7 +96,7 @@ fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult, name: &str) -> PyResult { - native_field(&native_field(owner, name)?, "stem")?.extract() + path_stem(&native_field(owner, name)?) } /// `(resource.name, str(resource.relative_path))`. @@ -106,8 +107,15 @@ fn named(resource: &Bound<'_, PyAny>) -> PyResult { }) } -/// `(file.file_path.stem, str(file.relative_path))`. +/// `(file.file_path.stem, str(file.relative_path))`; a native model file answers from its texts. fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { + if let Ok(model_file) = file.cast::() { + let texts: ModelFileTexts = model_file.get().path_texts(file.py())?; + return Ok(NamedPath { + name: texts.name, + path: texts.relative_path, + }); + } Ok(NamedPath { name: stem(file, "file_path")?, path: text(file, "relative_path")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 55c95a0e56..6f636c6d51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -1,7 +1,12 @@ //! SQL `TEST(...)` blocks validated natively and built as discovery records. +use std::collections::HashMap; +use std::sync::Mutex; + use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_model_config::sql_tests::main::sql_test_block::sql_test_block; use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_names_failure; @@ -53,6 +58,22 @@ fn case_record<'py>( ) } +/// `SqlTestMode(mode)`, the member each mode resolves to looked up once per process. +fn test_mode<'py>(py: Python<'py>, mode: &'static str) -> PyResult> { + static MODES: PyOnceLock>>> = PyOnceLock::new(); + let modes: &Mutex>> = + MODES.get_or_init(py, || Mutex::new(HashMap::new())); + let mut known = modes + .lock() + .map_err(|_| PyRuntimeError::new_err("SQL test mode cache is poisoned"))?; + if let Some(member) = known.get(mode) { + return Ok(member.bind(py).clone()); + } + let member: Bound<'py, PyAny> = host::entry(py, FacadeHost::SqlTestMode)?.call1((mode,))?; + let _ = known.insert(mode, member.clone().unbind()); + Ok(member) +} + fn block_record<'py>( py: Python<'py>, (index, native): (usize, &NativeBlock<'py>), @@ -86,10 +107,7 @@ fn block_record<'py>( keywords.set_item("header_values", &native.0)?; keywords.set_item("sql_body", native.1.as_str())?; keywords.set_item("name", block.name)?; - keywords.set_item( - "mode", - host::entry(py, FacadeHost::SqlTestMode)?.call1((block.mode,))?, - )?; + keywords.set_item("mode", test_mode(py, block.mode)?)?; keywords.set_item("parameters", PyTuple::new(py, parameters)?)?; keywords.set_item("cases", PyTuple::new(py, cases)?)?; keywords.set_item("cursor_start", block.cursor_start)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index ba0d85caeb..91df11f629 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -18,10 +18,32 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; +/// Where a compact model's facts are: given, or projected from its compiled model on first read. +pub(crate) enum CompactColumns { + Facts(Py), + /// A compiled model whose `fast_lineage_columns` are the facts, and their upstream count. + Model { + model: Py, + edges: usize, + }, +} + /// One compact model's facts and root-star flag. -struct CompactModel { - columns: Py, - has_star: bool, +pub(crate) struct CompactModel { + pub(crate) columns: CompactColumns, + pub(crate) has_star: bool, +} + +impl CompactModel { + /// The model's lineage facts as Python objects. + fn facts<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.columns { + CompactColumns::Facts(columns) => Ok(columns.bind(py).clone()), + CompactColumns::Model { model, .. } => { + native_field(model.bind(py), "fast_lineage_columns") + } + } + } } /// The Python objects one edge record materialises from. @@ -103,30 +125,12 @@ impl ProjectColumnLineage { let _ = compact.insert( name.extract()?, CompactModel { - columns: columns.unbind(), + columns: CompactColumns::Facts(columns.unbind()), has_star, }, ); } - let mut edge_counts: HashMap = HashMap::new(); - for name in &model_order { - if let Some(model) = compact.get(name) { - let count: usize = fact_edge_count(&classes, model.columns.bind(cls.py()))?; - let _ = edge_counts.insert(name.clone(), count); - } - } - for item in mapping_items(models)?.try_iter()? { - let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; - let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); - } - Ok(Self { - models: models.clone().unbind(), - compact, - model_order, - edges: None, - records: None, - edge_counts, - }) + Self::compacted(&classes, models, compact, model_order) } /// Return model lineage, expanding compact model facts on first access. @@ -144,7 +148,7 @@ impl ProjectColumnLineage { continue; } let lineage: Bound<'_, PyAny> = - model_from_facts(&classes, name, compact.columns.bind(py), compact.has_star)?; + model_from_facts(&classes, name, &compact.facts(py)?, compact.has_star)?; models.set_item(name, lineage)?; } slf.compact.clear(); @@ -275,6 +279,38 @@ impl ProjectColumnLineage { } impl ProjectColumnLineage { + /// Indexed project lineage of built `models` and `compact` facts, in `model_order`. + pub(crate) fn compacted( + classes: &LineageClasses<'_>, + models: &Bound<'_, PyAny>, + compact: HashMap, + model_order: Vec, + ) -> PyResult { + let py: Python<'_> = models.py(); + let mut edge_counts: HashMap = HashMap::new(); + for name in &model_order { + if let Some(model) = compact.get(name) { + let count: usize = match &model.columns { + CompactColumns::Facts(columns) => fact_edge_count(classes, columns.bind(py))?, + CompactColumns::Model { edges, .. } => *edges, + }; + let _ = edge_counts.insert(name.clone(), count); + } + } + for item in mapping_items(models)?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); + } + Ok(Self { + models: models.clone().unbind(), + compact, + model_order, + edges: None, + records: None, + edge_counts, + }) + } + /// Build fast edge records from the compact facts, or the expanded models, in model order. fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { if self.records.is_some() { @@ -285,7 +321,7 @@ impl ProjectColumnLineage { let models: Bound<'_, PyAny> = self.models.bind(py).clone(); for name in &self.model_order { if let Some(compact) = self.compact.get(name) { - builder.push_facts(classes, name, compact.columns.bind(py))?; + builder.push_facts(classes, name, &compact.facts(py)?)?; continue; } let model: Bound<'_, PyAny> = mapping_get(&models, name)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 8254b952d8..49c73bacfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -1,5 +1,7 @@ //! Project column lineage in FAST or RICH mode, and direct semantic uses, from a project view. +use std::collections::HashMap; + use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; @@ -18,7 +20,10 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::star_flags; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::lineage::column_lineage_graph::{ + CompactColumns, CompactModel, ProjectColumnLineage, +}; use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_lineage_event}; use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, @@ -139,11 +144,18 @@ fn selected_models<'py>( /// One model's FAST lineage work: its compact facts and whether its root star expands. struct FastModel<'py> { name: String, - facts: Option>, + facts: FastFacts<'py>, has_star: bool, expands_star: bool, } +/// A model's lineage facts: none, Python facts, or a compiled model's native facts and edge count. +enum FastFacts<'py> { + Missing, + Python(Bound<'py, PyAny>), + Native(Bound<'py, PyAny>, usize), +} + fn fast_lineage<'py>( classes: &LineageClasses<'py>, project: &Bound<'py, PyAny>, @@ -154,9 +166,22 @@ fn fast_lineage<'py>( let mut work: Vec> = Vec::with_capacity(models.len()); let mut requests: Vec = Vec::new(); for model in models { - let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; let (has_star, star_resolved): (bool, bool) = star_flags(model)?; let expands_star: bool = has_star && !star_resolved; + let native_edges: Option = match model.downcast::() { + Ok(view) if !expands_star => view.get().lineage_edge_count(py)?, + _ => None, + }; + if let Some(edges) = native_edges { + work.push(FastModel { + name: native_field(model, "name")?.extract()?, + facts: FastFacts::Native(model.clone(), edges), + has_star, + expands_star, + }); + continue; + } + let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; if facts.is_none() { requests.push(FastLineageModel::Parse { query_sql: native_field(model, "query_sql")?.extract()?, @@ -170,7 +195,11 @@ fn fast_lineage<'py>( } work.push(FastModel { name: native_field(model, "name")?.extract()?, - facts: (!facts.is_none()).then_some(facts), + facts: if facts.is_none() { + FastFacts::Missing + } else { + FastFacts::Python(facts) + }, has_star, expands_star, }); @@ -196,56 +225,70 @@ fn fast_lineage<'py>( }; let mut outcomes = outcomes.into_iter(); let built: Bound<'py, PyDict> = PyDict::new(py); - let compact: Bound<'py, PyDict> = PyDict::new(py); + let mut compact: HashMap = HashMap::new(); let mut model_order: Vec = Vec::new(); for model in work { - if let Some(mut columns) = model.facts { - if model.expands_star { - let star_columns: Vec = outcomes - .next() - .map(|outcome| outcome.into_parts().1) - .unwrap_or_default(); - let mut expanded: Vec> = - columns.try_iter()?.collect::>()?; - for column in &star_columns { - expanded.push(classes.column_fact(column)?); + let columns: CompactColumns = match model.facts { + FastFacts::Native(view, edges) => CompactColumns::Model { + model: view.unbind(), + edges, + }, + FastFacts::Python(mut columns) => { + if model.expands_star { + let star_columns: Vec = outcomes + .next() + .map(|outcome| outcome.into_parts().1) + .unwrap_or_default(); + let mut expanded: Vec> = + columns.try_iter()?.collect::>()?; + for column in &star_columns { + expanded.push(classes.column_fact(column)?); + } + columns = PyTuple::new(py, expanded)?.into_any(); } - columns = PyTuple::new(py, expanded)?.into_any(); + CompactColumns::Facts(columns.unbind()) } - compact.set_item(&model.name, (columns, model.has_star))?; - model_order.push(model.name); - continue; - } - match outcomes.next() { - Some(FastLineageOutcome::Built { columns, has_star }) => { - let lineage_columns: Vec> = columns - .iter() - .map(|column| classes.column(column, &classes.unknown_nullability)) - .collect::>()?; - built.set_item( - &model.name, - classes.model(&model.name, lineage_columns, has_star)?, - )?; - model_order.push(model.name); - } - Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( - py, - "fast column lineage parse failed; falling back", - &model.name, - Some(&detail), - )?, - Some(FastLineageOutcome::UnknownDialect(detail)) => { - return Err(unknown_dialect_error(py, &detail)); + FastFacts::Missing => { + match outcomes.next() { + Some(FastLineageOutcome::Built { columns, has_star }) => { + let lineage_columns: Vec> = columns + .iter() + .map(|column| classes.column(column, &classes.unknown_nullability)) + .collect::>()?; + built.set_item( + &model.name, + classes.model(&model.name, lineage_columns, has_star)?, + )?; + model_order.push(model.name); + } + Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( + py, + "fast column lineage parse failed; falling back", + &model.name, + Some(&detail), + )?, + Some(FastLineageOutcome::UnknownDialect(detail)) => { + return Err(unknown_dialect_error(py, &detail)); + } + _ => {} + } + continue; } - _ => {} - } + }; + let _ = compact.insert( + model.name.clone(), + CompactModel { + columns, + has_star: model.has_star, + }, + ); + model_order.push(model.name); } - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("models", built)?; - fields.set_item("compact_models", compact)?; - fields.set_item("model_order", PyTuple::new(py, model_order)?)?; - py.get_type::() - .call_method("from_fast_facts", (), Some(&fields)) + Bound::new( + py, + ProjectColumnLineage::compacted(classes, built.as_any(), compact, model_order)?, + ) + .map(Bound::into_any) } /// The output column names of compact facts or plain column facts. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index c08d164649..926ac60583 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -10,9 +10,10 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use crate::bindings::_helpers::records::construct::new_record_with; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use crate::bindings::_helpers::records::ids::RecordId; const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ @@ -34,7 +35,7 @@ const RESOURCE_NODES: [&str; 4] = [ ]; fn stem(path: &Bound<'_, PyAny>) -> PyResult { - native_field(path, "stem")?.extract() + path_stem(path) } fn unit_test_names(discovered: &Bound<'_, PyAny>) -> PyResult> { @@ -57,7 +58,11 @@ fn resource_names(discovered: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); for field in RESOURCE_PATH_FILES { for file in native_field(discovered, field)?.try_iter()? { - names.insert(stem(&native_field(&file?, "file_path")?)?); + let file: Bound<'_, PyAny> = file?; + names.insert(match file.cast::() { + Ok(native) => native.get().path_texts(file.py())?.name, + Err(_) => stem(&native_field(&file, "file_path")?)?, + }); } } for (field, entries) in RESOURCE_ENTRY_FILES { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index e2c2c9a375..effacd0eb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -1,15 +1,20 @@ //! What each model, seed, source and table function declares about its columns before analysis. +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use std::collections::HashMap; + +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_input_view::{ attached_columns, attached_schema, }; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::records::construct::{enum_member, record_class}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compiled_project::source_views::{AttachedSource, read_source_view}; +use crate::bindings::_helpers::records::construct::enum_member; +use crate::bindings::_helpers::records::fields::{enum_member_value, field as native_field}; use crate::bindings::_helpers::records::ids::RecordId; const UNKNOWN_TYPE: &str = "UNKNOWN"; @@ -19,10 +24,152 @@ const CONTRACT_KEY: &str = "contract"; const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const NON_NULL_MEMBER: &str = "NON_NULL"; const UNKNOWN_MEMBER: &str = "UNKNOWN"; +/// `str(None)`, the text of an exact declaration without a type. +const NONE_TEXT: &str = "None"; /// The three per-relation declaration maps the analysis starts from. -pub(crate) type DeclaredRelations<'py> = - (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); +pub(crate) type DeclaredRelations<'py> = ( + Bound<'py, PyDict>, + Bound<'py, PyDict>, + Bound<'py, PyDict>, + DeclaredShapes, +); + +/// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. +#[derive(Debug, Clone, Default)] +pub(crate) struct ShapeRows { + rows: Shapes, + positions: HashMap, +} + +impl ShapeRows { + /// `shapes[name] = pairs`, keeping an existing name's position as a dict does. + pub(crate) fn set(&mut self, name: String, pairs: Pairs) { + match self.positions.get(&name) { + Some(&position) => self.rows[position].1 = pairs, + None => { + let _ = self.positions.insert(name.clone(), self.rows.len()); + self.rows.push((name, pairs)); + } + } + } + + /// The rows in insertion order. + pub(crate) fn rows(&self) -> &Shapes { + &self.rows + } +} + +/// `(column, value)` pairs as a dict keeps them: each column once, at its first position, last value. +pub(crate) fn dict_pairs(items: impl IntoIterator) -> Pairs { + let mut pairs: Pairs = Vec::new(); + let mut positions: HashMap = HashMap::new(); + for (column, value) in items { + match positions.get(&column) { + Some(&position) => pairs[position].1 = value, + None => { + let _ = positions.insert(column.clone(), pairs.len()); + pairs.push((column, value)); + } + } + } + pairs +} + +/// The native rows of the declared types, nullability and authoritative schemas. +#[derive(Debug, Clone, Default)] +pub(crate) struct DeclaredShapes { + pub(crate) types: ShapeRows, + pub(crate) nullability: ShapeRows, + pub(crate) complete: ShapeRows, +} + +/// The Python declaration maps being built: declared types, nullability and authoritative schemas. +type DeclaredMaps<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); + +/// The native rows of the maps being built and the nullability texts they hold. +struct Declared { + shapes: DeclaredShapes, + non_null_text: &'static str, + unknown_text: &'static str, +} + +impl Declared { + fn new() -> PyResult { + let text = |member: &str| -> PyResult<&'static str> { + enum_member_value(RecordId::InferredNullability.name(), member).ok_or_else(|| { + PyRuntimeError::new_err(format!("InferredNullability has no member {member}")) + }) + }; + Ok(Self { + shapes: DeclaredShapes::default(), + non_null_text: text(NON_NULL_MEMBER)?, + unknown_text: text(UNKNOWN_MEMBER)?, + }) + } + + /// Record `name`'s declared types and declared nullability. + fn declare<'py>( + &mut self, + maps: &DeclaredMaps<'py>, + name: &Bound<'py, PyAny>, + columns: &[SchemaColumnFact], + ) -> PyResult<()> { + let py: Python<'py> = name.py(); + let key: String = name.extract()?; + maps.0.set_item(name, column_types(py, columns, false)?)?; + self.shapes + .types + .set(key.clone(), type_pairs(columns, false)); + maps.1.set_item(name, column_nullability(py, columns)?)?; + let pairs: Pairs = dict_pairs(columns.iter().map(|column| { + let text: &str = if non_null(column) { + self.non_null_text + } else { + self.unknown_text + }; + (column.name.clone(), text.to_owned()) + })); + self.shapes.nullability.set(key, pairs); + Ok(()) + } + + /// Record `name`'s declared types as an authoritative schema. + fn complete<'py>( + &mut self, + maps: &DeclaredMaps<'py>, + name: &Bound<'py, PyAny>, + (columns, exact): (&[SchemaColumnFact], bool), + ) -> PyResult<()> { + maps.2 + .set_item(name, column_types(name.py(), columns, exact)?)?; + self.shapes + .complete + .set(name.extract()?, type_pairs(columns, exact)); + Ok(()) + } +} + +/// Each column's declared type text as `str` reads the declared types map. +fn type_pairs(columns: &[SchemaColumnFact], exact: bool) -> Pairs { + dict_pairs(columns.iter().map(|column| { + let text: &str = match column.data_type.as_deref() { + Some(declared) if exact || !declared.is_empty() => declared, + None if exact => NONE_TEXT, + _ => UNKNOWN_TYPE, + }; + (column.name.clone(), text.to_owned()) + })) +} + +/// Whether a column is declared non-null: `nullable` false or a `not_null` audit. +fn non_null(column: &SchemaColumnFact) -> bool { + column.nullable == Some(false) + || column + .audit_definitions + .iter() + .any(|definition| definition == NOT_NULL_AUDIT_NAME) +} /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. fn column_types<'py>( @@ -45,19 +192,16 @@ fn column_types<'py>( /// Each column's declared nullability: non-null when `nullable` is false or a `not_null` audit. fn column_nullability<'py>( + py: Python<'py>, columns: &[SchemaColumnFact], - nullability: &Bound<'py, PyAny>, ) -> PyResult> { - let facts: Bound<'py, PyDict> = PyDict::new(nullability.py()); - let non_null_member: Bound<'py, PyAny> = nullability.getattr(NON_NULL_MEMBER)?; - let unknown_member: Bound<'py, PyAny> = nullability.getattr(UNKNOWN_MEMBER)?; + let facts: Bound<'py, PyDict> = PyDict::new(py); + let non_null_member: Bound<'py, PyAny> = + enum_member(py, RecordId::InferredNullability, NON_NULL_MEMBER)?; + let unknown_member: Bound<'py, PyAny> = + enum_member(py, RecordId::InferredNullability, UNKNOWN_MEMBER)?; for column in columns { - let non_null: bool = column.nullable == Some(false) - || column - .audit_definitions - .iter() - .any(|definition| definition == NOT_NULL_AUDIT_NAME); - let state: &Bound<'py, PyAny> = if non_null { + let state: &Bound<'py, PyAny> = if non_null(column) { &non_null_member } else { &unknown_member @@ -72,24 +216,35 @@ fn table_function_name(function: &Bound<'_, PyAny>) -> PyResult { Ok(format!("{TABLE_FUNCTION_PREFIX}{name}")) } +/// Record one source's declared types and nullability, and its schema when its contract is enforced. +fn declare_source<'py>( + (declared, maps): (&mut Declared, &DeclaredMaps<'py>), + (name, columns, enforced): (&Bound<'py, PyAny>, &[SchemaColumnFact], bool), +) -> PyResult<()> { + declared.declare(maps, name, columns)?; + if enforced && !columns.is_empty() { + declared.complete(maps, name, (columns, false))?; + } + Ok(()) +} + /// Declared column types, declared nullability and the authoritative binding schemas. pub(crate) fn declared_relations<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let nullability: Bound<'py, PyAny> = record_class(py, RecordId::InferredNullability)?; - let types: Bound<'py, PyDict> = PyDict::new(py); - let nulls: Bound<'py, PyDict> = PyDict::new(py); - let complete: Bound<'py, PyDict> = PyDict::new(py); + let maps: DeclaredMaps<'py> = (PyDict::new(py), PyDict::new(py), PyDict::new(py)); + let mut declared: Declared = Declared::new()?; for model in native_field(&inputs, "model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let Some(schema) = attached_schema(&model)? else { continue; }; - let (name, columns): (String, Vec) = - (schema.name, attached_columns(&model)?); - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + let (name, columns): (Bound<'py, PyAny>, Vec) = ( + PyString::new(py, &schema.name).into_any(), + attached_columns(&model)?, + ); + declared.declare(&maps, &name, &columns)?; let contract: Option> = native_field(&native_field(&model, "config")?, "values")? .downcast_into::()? @@ -99,7 +254,7 @@ pub(crate) fn declared_relations<'py>( None => false, }; if enforced && !columns.is_empty() && !schema.dynamic_columns.is_truthy()? { - complete.set_item(&name, column_types(py, &columns, false)?)?; + declared.complete(&maps, &name, (&columns, false))?; } } let mut seeds: Vec<(Bound<'py, PyAny>, Vec)> = Vec::new(); @@ -108,24 +263,34 @@ pub(crate) fn declared_relations<'py>( let name: Bound<'py, PyAny> = native_field(&entry, "name")?; let columns: Vec = python_column_facts(&native_field(&entry, "columns")?, None)?; - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + declared.declare(&maps, &name, &columns)?; seeds.push((name, columns)); } for source in native_field(&inputs, "source_inputs")?.try_iter()? { - let entry: Bound<'py, PyAny> = native_field(&source?, "source_entry")?; + let source: Bound<'py, PyAny> = source?; + let native = read_source_view(&source, |record, py| { + let attached: &AttachedSource = record.attached(); + let name: Bound<'py, PyAny> = PyString::new(py, &attached.name).into_any(); + let enforced: bool = attached.contract.as_deref() == Some(ENFORCED_CONTRACT); + declare_source( + (&mut declared, &maps), + (&name, &record.key_facts(py)?.columns, enforced), + ) + }); + if let Some(outcome) = native { + outcome?; + continue; + } + let entry: Bound<'py, PyAny> = native_field(&source, "source_entry")?; let name: Bound<'py, PyAny> = native_field(&entry, "name")?; let columns: Vec = python_column_facts(&native_field(&entry, "columns")?, None)?; - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; - if entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)? && !columns.is_empty() { - complete.set_item(&name, column_types(py, &columns, false)?)?; - } + let enforced: bool = entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)?; + declare_source((&mut declared, &maps), (&name, &columns, enforced))?; } for (name, columns) in seeds { if !columns.is_empty() { - complete.set_item(name, column_types(py, &columns, false)?)?; + declared.complete(&maps, &name, (&columns, false))?; } } for function in native_field(&inputs, "sql_function_inputs")?.try_iter()? { @@ -135,8 +300,13 @@ pub(crate) fn declared_relations<'py>( if columns.is_empty() { continue; } - let name: String = table_function_name(&function)?; - types.set_item(&name, column_types(py, &columns, true)?)?; + let name: Bound<'py, PyAny> = + PyString::new(py, &table_function_name(&function)?).into_any(); + maps.0.set_item(&name, column_types(py, &columns, true)?)?; + declared + .shapes + .types + .set(name.extract()?, type_pairs(&columns, true)); let unknown: Bound<'py, PyDict> = PyDict::new(py); for column in &columns { unknown.set_item( @@ -144,8 +314,18 @@ pub(crate) fn declared_relations<'py>( enum_member(py, RecordId::InferredNullability, UNKNOWN_MEMBER)?, )?; } - nulls.set_item(&name, unknown)?; - complete.set_item(&name, column_types(py, &columns, true)?)?; + maps.1.set_item(&name, unknown)?; + let unknown_text: &str = declared.unknown_text; + declared.shapes.nullability.set( + name.extract()?, + dict_pairs( + columns + .iter() + .map(|column| (column.name.clone(), unknown_text.to_owned())), + ), + ); + declared.complete(&maps, &name, (&columns, true))?; } - Ok((types, nulls, complete)) + let (types, nulls, complete) = maps; + Ok((types, nulls, complete, declared.shapes)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d537996df5..65b19fa0c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -165,6 +165,8 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ CONTRACT_RECORDS, ]; +/// `str(PurePath("."))`, the path whose name and stem are empty. +pub(crate) const CURRENT_DIRECTORY_PATH: &str = "."; /// Records whose bound values are normalized or validated before the record is created. pub(crate) const RECORD_FINISHERS: &[&[(&str, RecordFinish)]] = &[ &[("SourceFreshnessObservation", normalize_observation)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index a3001a18c4..f1f5086168 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -13,20 +13,22 @@ use pyo3::types::{ PyType, PyTypeMethods, }; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::constants::{ - CACHED_PROPERTIES, CLASS_PROPERTIES, ENUMS, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, - ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + CACHED_PROPERTIES, CLASS_METHODS, CLASS_PROPERTIES, CURRENT_DIRECTORY_PATH, ENUMS, + FROZEN_INSTANCE_ERROR, NATIVE_MODULE, ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, + RECORDS, }; use crate::bindings::_helpers::records::construct::{enum_member, new_record}; use crate::bindings::_helpers::records::getters::getter_field; use crate::bindings::_helpers::records::models::{ EnumSpec, FieldDefault, NativeRecord, RecordSpec, }; -use crate::bindings::_helpers::records::types::ClassProperty; +use crate::bindings::_helpers::records::types::{ClassProperty, RecordFinish}; /// How a test run treats `field` falling back to `getattr` on a native class: unset outside tests. pub(crate) const NATIVE_FIELD_STRICT_ENV_VAR: &str = "SQLBUILD_NATIVE_FIELD_STRICT"; @@ -38,13 +40,31 @@ const ENUM_NAME_ATTRIBUTE: &str = "name"; /// Native string enums by class object, for `.value` and `.name` reads answered from their members. static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); +/// Each registered record class's spec, by class object. +static REGISTERED_SPECS: PyOnceLock> = PyOnceLock::new(); + /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) } +/// Keep each registered record class's spec, so records of that exact class find it natively. +pub(crate) fn register_specs( + py: Python<'_>, + classes: HashMap, +) -> PyResult<()> { + REGISTERED_SPECS.get_or_init(py, || classes); + Ok(()) +} + /// The spec registered for `cls`, read from its class attribute. pub(crate) fn record_spec(cls: &Bound<'_, PyType>) -> PyResult<&'static RecordSpec> { + if let Some(spec) = REGISTERED_SPECS + .get(cls.py()) + .and_then(|specs| specs.get(&(cls.as_ptr() as usize))) + { + return Ok(spec); + } let Ok(index) = cls.getattr(RECORD_SPEC_ATTRIBUTE) else { return Err(PyTypeError::new_err(format!( "{} cannot be instantiated directly", @@ -84,6 +104,43 @@ pub(crate) fn field<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( + value: &Bound<'py, PyAny>, + name: &str, +) -> Option>>> { + let Ok(cell) = value.cast::() else { + return None; + }; + if name.starts_with('_') { + return None; + } + let record: &NativeRecord = cell.get(); + if let Some(found) = record + .spec + .fields + .iter() + .position(|field| field.name == name) + .and_then(|index| record.values.get(index)) + { + return Some(Ok(Some(found.bind(value.py()).clone()))); + } + match record_properties().get(&(record.spec.name, name)) { + Some(&(property, cached_name)) => Some( + property_value(cell, property, cached_name) + .map(|found| Some(found.into_bound(value.py()))), + ), + None if CLASS_METHODS + .iter() + .flat_map(|group| group.iter()) + .any(|(class, method, _)| *class == record.spec.name && *method == name) => + { + None + } + None => Some(Ok(None)), + } +} + /// A strict-mode setting: record fallbacks to a file, or fail on any fallback missing from an allowlist. enum StrictFields { Record(String, Mutex>), @@ -373,16 +430,24 @@ pub(crate) fn build_values( }; values.push(value.unbind()); } - match RECORD_FINISHERS - .iter() - .flat_map(|group| group.iter()) - .find(|(record, _)| *record == name) - { - Some((_, finish)) => finish(py, values), + match record_finishers().get(name) { + Some(finish) => finish(py, values), None => Ok(values), } } +/// Each finished record's finisher, by class name. +fn record_finishers() -> &'static HashMap<&'static str, RecordFinish> { + static FINISHERS: OnceLock> = OnceLock::new(); + FINISHERS.get_or_init(|| { + RECORD_FINISHERS + .iter() + .flat_map(|group| group.iter()) + .map(|&(record, finish)| (record, finish)) + .collect() + }) +} + fn default_value<'py>( py: Python<'py>, record: &str, @@ -417,7 +482,7 @@ pub(crate) fn compared_values<'py>( record: &Bound<'py, NativeRecord>, ) -> PyResult> { let py: Python<'py> = record.py(); - let spec: &RecordSpec = record_spec(&record.get_type())?; + let spec: &RecordSpec = record.get().spec; let values = spec .fields .iter() @@ -432,7 +497,7 @@ pub(crate) fn compared_values<'py>( pub(crate) fn repr_text(record: &Bound<'_, NativeRecord>) -> PyResult { let py: Python<'_> = record.py(); let cls: Bound<'_, PyType> = record.get_type(); - let spec: &RecordSpec = record_spec(&cls)?; + let spec: &RecordSpec = record.get().spec; let mut parts: Vec = Vec::with_capacity(spec.fields.len()); for (field, value) in spec.fields.iter().zip(record.get().values.iter()) { if field.shown { @@ -460,7 +525,7 @@ pub(crate) fn ordered_comparison( op: CompareOp, ) -> PyResult> { let py: Python<'_> = slf.py(); - let name: &str = record_spec(&slf.get_type())?.name; + let name: &str = slf.get().spec.name; let ordered: bool = ORDERED_RECORDS.iter().any(|group| group.contains(&name)); if !ordered || !other.get_type().is(slf.get_type()) { return Ok(py.NotImplemented()); @@ -512,3 +577,30 @@ pub(crate) fn path_posix(path: &Bound<'_, PyAny>) -> PyResult { let text: String = path.str()?.to_string_lossy().into_owned(); Ok(String::from_utf8_lossy(&pure_path_posix(std::path::Path::new(&text))).into_owned()) } + +/// The value of the native string enum `enum_name`'s member `member`. +pub(crate) fn enum_member_value(enum_name: &str, member: &str) -> Option<&'static str> { + ENUMS + .iter() + .flat_map(|group| group.iter()) + .find(|spec| spec.name == enum_name)? + .members + .iter() + .find(|(name, _)| *name == member) + .map(|(_, value)| *value) +} + +/// `path.stem` of a `pathlib` path, spelled natively from its text. +pub(crate) fn path_stem(path: &Bound<'_, PyAny>) -> PyResult { + let posix: String = path_posix_text(path)?; + if posix == CURRENT_DIRECTORY_PATH { + return Ok(String::new()); + } + Ok(model_file_stem(&posix).to_owned()) +} + +/// `path.as_posix()` of a `pathlib` path, its text with the platform separator spelled `/`. +pub(crate) fn path_posix_text(path: &Bound<'_, PyAny>) -> PyResult { + let text: Bound<'_, PyString> = path.str()?; + Ok(text.to_str()?.replace(std::path::MAIN_SEPARATOR, "/")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 4f69782b9d..7fcecb19ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -13,8 +13,10 @@ use crate::bindings::_helpers::records::constants::{ use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, }; -use crate::bindings::_helpers::records::fields::all_specs; +use std::collections::HashMap; + use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{all_specs, register_specs}; use crate::bindings::_helpers::records::models::{ BoundNativeMethod, EnumSpec, ErrorSpec, NativeMethod, NativeProperty, NativeRecord, RecordAttribute, RecordSpec, @@ -47,14 +49,17 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add(spec.name, error_class(&parent, (index, spec))?)?; } let dataclasses: Bound<'_, PyModule> = py.import(DATACLASSES_MODULE)?; + let mut registered: HashMap = HashMap::new(); for (index, spec) in all_specs().enumerate() { let class: Bound<'_, PyAny> = record_class(&metaclass, &base, (index, spec))?; + let _ = registered.insert(class.as_ptr() as usize, spec); class.setattr( DATACLASS_FIELDS_ATTRIBUTE, dataclass_fields(&dataclasses, spec)?, )?; module.add(spec.name, class)?; } + register_specs(py, registered)?; for (class, name, property) in CLASS_PROPERTIES.iter().flat_map(|group| group.iter()) { let descriptor: Py = Py::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 06568f07bc..f1bf047ba8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -34,7 +34,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; use crate::bindings::_helpers::rules::relation_keys::{ - RelationKeyRow, RelationKeys, relation_keys, + RelationKeyRow, RelationKeys, relation_keys, source_key_input, }; use crate::bindings::_helpers::rules::rule_cache::stored_entries; use crate::bindings::_helpers::rules::rules_config::rules_config_of; @@ -122,7 +122,7 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> = Vec::new(); for source in record_field(inputs, "source_inputs")?.try_iter()? { - sources.push(record_field(&source?, "source_entry")?); + sources.push(source_key_input(&source?)?); } let keys: RelationKeys = relation_keys( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index d5f429d621..989f88060c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -10,6 +10,7 @@ use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; +use crate::bindings::_helpers::compiled_project::source_views::{SourceKeyFacts, read_source_view}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_inputs::model_key_row; @@ -52,19 +53,7 @@ pub(crate) fn relation_key_row( column.nullable == Some(false), )); } - for audit in declared_audits.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let argument: Option> = native_field(&audit, "arguments")? - .downcast::()? - .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; - audits.push(( - native_field(&audit, "definition_name")?.extract()?, - argument - .filter(pyo3::types::PyAnyMethods::is_instance_of::) - .map(|value| value.extract()) - .transpose()?, - )); - } + audits = audit_key_rows(declared_audits)?; } Ok(( kind.to_owned(), @@ -81,6 +70,61 @@ pub(crate) fn relation_key_row( )) } +/// Each audit spec's `(definition name, column argument when it is text)`. +pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { + let mut rows: Vec<(String, Option)> = Vec::new(); + for audit in audits.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let argument: Option> = native_field(&audit, "arguments")? + .downcast::()? + .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; + rows.push(( + native_field(&audit, "definition_name")?.extract()?, + argument + .filter(pyo3::types::PyAnyMethods::is_instance_of::) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) +} + +/// A source's key row from its native view's declared facts, or `None` for any other object. +pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option> { + read_source_view(view, |source, py| { + let facts: &SourceKeyFacts = source.key_facts(py)?; + Ok(( + RELATION_KIND_SOURCE.to_owned(), + source.attached().name.clone(), + String::new(), + String::new(), + String::new(), + false, + None, + facts + .columns + .iter() + .map(|column| { + ( + column.name.clone(), + column.audit_definitions.clone(), + column.nullable == Some(false), + ) + }) + .collect(), + facts.audits.clone(), + )) + }) +} + +/// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. +pub(crate) fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + if read_source_view(source, |_, _| Ok(())).is_some() { + return Ok(source.clone()); + } + native_field(source, "source_entry") +} + /// `str(values.get(key, ""))`. fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { match values.get_item(key)? { @@ -149,6 +193,10 @@ pub(crate) fn relation_keys<'py>( for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { let entry: Bound<'py, PyAny> = entry?; + if let Some(row) = source_view_key_row(&entry) { + rows.push(row?); + continue; + } let name: String = native_field(&entry, "name")?.extract()?; rows.push(entry_relation_key_row( kind, @@ -201,7 +249,7 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult> = Vec::new(); for source in native_field(project, "sources")?.try_iter()? { - sources.push(native_field(&source?, "source_entry")?); + sources.push(source_key_input(&source?)?); } relation_keys( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 334bfbf364..84db425ada 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -20,9 +20,10 @@ use sqlbuild_scopes::scope_index::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_values; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::relationship_scan::{ RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, @@ -94,7 +95,7 @@ fn optional_text(item: &Bound<'_, PyAny>, name: &str) -> PyResult } fn stem(path: &Bound<'_, PyAny>) -> PyResult { - native_field(path, "stem")?.extract() + path_stem(path) } /// `value or fallback` with Python truthiness. @@ -109,6 +110,14 @@ fn or_else<'py>( } } +/// `value` when it is a non-empty string. +fn header_text(value: &Bound<'_, PyAny>) -> Option { + value + .extract::() + .into_iter() + .find(|name| !name.is_empty()) +} + /// `value if isinstance(value, str) and value else path.stem`. fn name_or_stem(value: &Bound<'_, PyAny>, path: &Bound<'_, PyAny>) -> PyResult { if let Ok(name) = value.extract::() @@ -144,6 +153,16 @@ impl<'py> Walk<'py> { kind: ResourceKind, name: String, location: (&Bound<'py, PyAny>, ResourceRoot), + ) -> PyResult<()> { + let (path, root) = location; + self.add_resource_at(kind, name, (text(path)?, root)) + } + + fn add_resource_at( + &mut self, + kind: ResourceKind, + name: String, + location: (String, ResourceRoot), ) -> PyResult<()> { let (path, root) = location; self.resource_identities @@ -154,7 +173,7 @@ impl<'py> Walk<'py> { self.resource_roots.push((kind, root)); self.inputs.resources.push(ResourceInput { identity: ResourceIdentity { kind, name }, - path: text(path)?, + path, root, }); Ok(()) @@ -289,13 +308,37 @@ impl<'py> Walk<'py> { fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for model_file in items(inputs, "model_files")? { - let path: Bound<'py, PyAny> = native_field(&model_file, "relative_path")?; - let model_name: String = name_or_stem(&header_name(&model_file)?, &path)?; - self.add_resource( + let header: Bound<'py, PyAny> = header_name(&model_file)?; + let (model_name, path, folder): (String, String, String) = + match model_file.cast::() { + Ok(native) => { + let native: &NativeModelFile = native.get(); + let texts: ModelFileTexts = native.path_texts(model_file.py())?; + let name: String = match header_text(&header) { + Some(name) => name, + None => texts.name, + }; + (name, texts.relative_path, native.folder(model_file.py())?) + } + Err(_) => { + let path: Bound<'py, PyAny> = native_field(&model_file, "relative_path")?; + ( + name_or_stem(&header, &path)?, + text(&path)?, + text(&native_field(&path, "parent")?)?, + ) + } + }; + self.add_resource_at( ResourceKind::Model, model_name.clone(), - (&path, ResourceRoot::Kind), + (path.clone(), ResourceRoot::Kind), )?; + let enums: Vec> = items(&model_file, "enum_declarations")?; + let constants: Vec> = items(&model_file, "constant_declarations")?; + if enums.is_empty() && constants.is_empty() { + continue; + } let owner: Bound<'py, PyAny> = self.classes.model(RecordId::ResourceIdentity)?.call1(( self.classes @@ -305,7 +348,7 @@ impl<'py> Walk<'py> { let private: Bound<'py, PyAny> = self .classes .member(RecordId::ScopeKind, ScopeKind::Private.as_str())?; - for declaration in items(&model_file, "enum_declarations")? { + for declaration in enums { let identity: Bound<'py, PyAny> = self.declaration_identity( DeclarationKind::Enum, &native_field(&declaration, "name")?, @@ -313,9 +356,9 @@ impl<'py> Walk<'py> { )?; let metadata = [None, Some(self.enum_metadata(&declaration)?), None]; let values = self.values(identity, private.clone(), metadata); - self.add_private(values, &path)?; + self.add_private(values, (&path, &folder))?; } - for declaration in items(&model_file, "constant_declarations")? { + for declaration in constants { let identity: Bound<'py, PyAny> = self.declaration_identity( DeclarationKind::Constant, &native_field(&declaration, "name")?, @@ -323,7 +366,7 @@ impl<'py> Walk<'py> { )?; let metadata = [None, None, Some(self.constant_metadata(&declaration)?)]; let values = self.values(identity, private.clone(), metadata); - self.add_private(values, &path)?; + self.add_private(values, (&path, &folder))?; } } Ok(()) @@ -332,15 +375,15 @@ impl<'py> Walk<'py> { fn add_private( &mut self, values: DeclarationValues<'py>, - path: &Bound<'py, PyAny>, + (path, folder): (&str, &str), ) -> PyResult<()> { self.add_declaration( values, Placement { - path: text(path)?, + path: path.to_owned(), ownership_root: Some(PRIVATE_ROOT.to_owned()), root_fallback: PRIVATE_ROOT.to_owned(), - owning_path: Some(text(&native_field(path, "parent")?)?), + owning_path: Some(folder.to_owned()), }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 08bd61504e..6ea7f7db18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -13,7 +13,10 @@ use crate::bindings::_helpers::analysis::model_reads::{ use crate::bindings::_helpers::analysis::normalization::normalize_analysis_sql; use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_expression_source_shapes; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compiled_project::compiled_model_view::{DeclaredRow, InferredRow}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + DeclaredRow, InferredRow, NativeCompiledModel, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::enum_member; @@ -34,11 +37,21 @@ const TABLE_FUNCTION_KIND: &str = "table_fn"; /// The relation names `(kind, name, package, count)` reference rows bind, UDFs excluded. pub(crate) fn relation_row_names<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { - let mut names: Vec = Vec::new(); + let mut pairs: Vec<(String, String)> = Vec::new(); for row in rows.try_iter()? { let row: Bound<'py, PyAny> = row?; - let kind: String = row.get_item(0)?.str()?.extract()?; - let name: String = row.get_item(1)?.extract()?; + pairs.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); + } + Ok(relation_names(pairs)) +} + +/// The relations `(kind, name)` reference rows bind, each once, table functions by binding name. +fn relation_names(rows: Vec<(String, String)>) -> Vec { + let mut names: Vec = Vec::new(); + for (kind, name) in rows { if kind == UDF_KIND { continue; } @@ -51,7 +64,23 @@ pub(crate) fn relation_row_names<'py>(rows: &Bound<'py, PyAny>) -> PyResult) -> PyResult<(Vec, bool)> { + if let Ok(view) = model.downcast::() { + let rows: Vec = view.get().reference_rows(model.py())?; + let referenced: bool = !rows.is_empty(); + let names: Vec = relation_names( + rows.into_iter() + .map(|(kind, name, _, _)| (kind, name)) + .collect(), + ); + return Ok((names, referenced)); + } + let rows: Bound<'_, PyAny> = native_field(model, "reference_rows")?; + Ok((relation_row_names(&rows)?, rows.is_truthy()?)) } /// `{name: shapes.get(name, {})}` for each relation a model's reference rows bind. @@ -61,7 +90,7 @@ pub(crate) fn model_inputs<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let inputs: Bound<'py, PyDict> = PyDict::new(py); - for name in relation_row_names(&native_field(model, "reference_rows")?)? { + for name in model_relation_names(model)?.0 { let shape: Bound<'py, PyAny> = match shapes.get_item(&name)? { Some(shape) => shape, None => PyDict::new(py).into_any(), @@ -293,8 +322,7 @@ fn model_shapes<'py>( for model in &pending { let declared: Option> = declared_rows(model)?; let inferred: Option> = inferred_rows(model)?; - let rows: Bound<'py, PyAny> = native_field(model, "reference_rows")?; - let relations: Vec = relation_row_names(&rows)?; + let (relations, referenced): (Vec, bool) = model_relation_names(model)?; let contract: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")? .call_method1("get", ("contract",))?; @@ -317,8 +345,7 @@ fn model_shapes<'py>( } let (has_star, star_resolved): (bool, bool) = star_flags(model)?; let closes: bool = inferred.as_ref().is_some_and(|rows| !rows.is_empty()) - && (!has_star - || (star_resolved && rows.is_truthy()? && all_known(&relations, shapes)?)); + && (!has_star || (star_resolved && referenced && all_known(&relations, shapes)?)); if let (true, Some(inferred)) = (closes, &inferred) { let inputs: Bound<'py, PyDict> = model_inputs(model, shapes)?; let columns: Bound<'py, PyDict> = typed_columns(py, inferred)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 55b7a7de1f..9dcd6a4f87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -13,7 +13,10 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::records::fields::{ + field as native_field, path_posix_text, path_stem, +}; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; @@ -69,6 +72,29 @@ pub(crate) fn assemble_compiled_sql_tests( } fn assembly_model(model: &Bound<'_, PyAny>) -> PyResult { + if let Ok(native) = model.cast::() { + return native.get().read(model.py(), |record, _| { + let unscanned_macro_source = if record.macro_deps.is_empty() { + Some(record.macro_source_sql.to_string()).filter(|sql| sql.contains(MACRO_TOKEN)) + } else { + None + }; + Ok(SqlTestAssemblyModel { + name: record.name.clone(), + macro_deps: record.macro_deps.clone(), + unscanned_macro_source, + references: record + .references + .iter() + .map(|(kind, name, package, _)| SqlTestAssemblyReference { + kind: kind.clone(), + name: name.clone(), + package: package.clone(), + }) + .collect(), + }) + }); + } let macro_deps: Vec = native_field(model, "macro_deps")?.extract()?; let unscanned_macro_source = if macro_deps.is_empty() { Some(native_field(model, "macro_source_sql")?.extract::()?) @@ -123,9 +149,9 @@ fn assembly_test(test: &Bound<'_, PyAny>) -> PyResult { }; Ok(SqlTestAssemblyTest { block_name: native_field(&test_block, "name")?.extract()?, - file_stem: native_field(&native_field(&test_file, "file_path")?, "stem")?.extract()?, - relative_path: relative_path.call_method0("as_posix")?.extract()?, - relative_stem: native_field(&relative_path, "stem")?.extract()?, + file_stem: path_stem(&native_field(&test_file, "file_path")?)?, + relative_path: path_posix_text(&relative_path)?, + relative_stem: path_stem(&relative_path)?, contents: native_field(&test_file, "contents")?.extract()?, block_sql: native_field(&test_block, "sql_body")?.extract()?, block_index: native_field(&test_block, "test_index")?.extract()?, diff --git a/fensu.toml b/fensu.toml index d9bfe33c54..356ddab89e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -607,7 +607,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs=4", @@ -616,7 +616,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=5", @@ -790,7 +790,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", @@ -929,7 +929,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", @@ -1044,7 +1044,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs=1", @@ -1151,7 +1151,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=3", @@ -1220,10 +1220,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 681d5d2618..bfb3ca0eab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -85,6 +85,15 @@ class NativeSourceAttachmentTestCase: expected_usages: tuple[str, ...] +@dataclass(frozen=True) +class NativeSourceKeysTestCase: + """Sources whose declared keys native views answer as their attached entries do.""" + + description: str + source_yml: str + expected_keys: dict[str, tuple[tuple[str, ...], ...]] + + @dataclass(frozen=True) class NativeSourceErrorTestCase: """A source the native loop rejects while attaching it.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py index 694f8f2939..409c1e5f39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py @@ -11,6 +11,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeSourceAttachmentTestCase, NativeSourceErrorTestCase, + NativeSourceKeysTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( column_metadata, @@ -99,6 +100,44 @@ def test_given_source_when_compiling_then_native_project_holds_its_attached_entr ) +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceKeysTestCase( + description="column and table unique audits with templated column metadata", + source_yml=( + "sources:\n" + " - name: raw_orders\n" + " audits:\n - unique:\n column: order_id\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n nullable: false\n" + " description: Order in ${region}.\n" + " - name: customer_id\n type: INTEGER\n" + " audits:\n - unique\n - not_null\n" + " - name: raw_events\n" + " columns:\n - name: event_id\n type: INTEGER\n" + ), + expected_keys={"source:raw_orders": (("customer_id",), ("order_id",))}, + ), + ], + ids=lambda case: case.description, +) +def test_given_keyed_sources_when_deriving_relation_keys_then_native_views_match_entries( + test_case: NativeSourceKeysTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: CompiledProjectView + project, _ = compile_source_project( + project_dir=tmp_path, source_yml=test_case.source_yml, monkeypatch=monkeypatch + ) + + from_entries: dict[str, tuple[tuple[str, ...], ...]] = _native.declared_relation_keys( + model_rows=(), seeds=(), sources=tuple(source.source_entry for source in project.sources) + ) + compiled: dict[str, tuple[tuple[str, ...], ...]] = _native.compiled_relation_keys(project) + assert from_entries == test_case.expected_keys + assert {identity: compiled[identity] for identity in from_entries} == from_entries + + @pytest.mark.parametrize( "test_case", [ From 920a7c398756e22dc3006317ac6c62fc29a3b7c1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:58:56 +0100 Subject: [PATCH 2/6] perf: emit JSON text into one growing buffer Serialising the 85 MB dense compile report copied every nested container's text into its parent and allocated a string per escaped character. Thread one buffer through the emitter instead; the output is byte-identical. --- .../src/json/_helpers/emitter.rs | 115 +++++++++++------- .../src/json/_helpers/strings.rs | 93 ++++++++------ 2 files changed, 122 insertions(+), 86 deletions(-) diff --git a/crates/sqlbuild-core/src/json/_helpers/emitter.rs b/crates/sqlbuild-core/src/json/_helpers/emitter.rs index 3a275b282b..1b2f1dd35b 100644 --- a/crates/sqlbuild-core/src/json/_helpers/emitter.rs +++ b/crates/sqlbuild-core/src/json/_helpers/emitter.rs @@ -1,7 +1,7 @@ //! Serialize a [`JsonValue`] in the layout of one Python serializer. use crate::json::_helpers::floats::{orjson_text, python_repr}; -use crate::json::_helpers::strings::{StringEscaping, json_string, json_utf16_string}; +use crate::json::_helpers::strings::{StringEscaping, with_json_string, with_json_utf16_string}; use crate::json::constants::{ ORJSON_MAX_CONTAINER_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, STDLIB_MAX_CONTAINER_DEPTH, @@ -48,9 +48,14 @@ impl<'options> Layout<'options> { } } - fn newline(&self, level: usize) -> String { - self.indent - .map_or_else(String::new, |indent| format!("\n{}", indent.repeat(level))) + fn with_newline(&self, mut output: String, level: usize) -> String { + if let Some(indent) = self.indent { + output.push('\n'); + for _ in 0..level { + output.push_str(indent); + } + } + output } fn integer(&self, value: &JsonInteger) -> Result { @@ -94,67 +99,85 @@ impl<'options> Layout<'options> { counted && level >= limit } - fn value(&self, value: &JsonValue, level: usize) -> Result { + fn with_value( + &self, + mut output: String, + value: &JsonValue, + level: usize, + ) -> Result { if self.too_deep(value, level) { return Err(JsonEmitError::NestingTooDeep); } match value { - JsonValue::Null => Ok("null".to_owned()), - JsonValue::Bool(flag) => Ok(flag.to_string()), - JsonValue::Integer(number) => self.integer(number), - JsonValue::Float(number) => self.float(*number), - JsonValue::String(text) => Ok(json_string(text, self.escaping)), - JsonValue::Utf16(units) => json_utf16_string(units, self.escaping), - JsonValue::Array(items) => { - let members = items - .iter() - .map(|item| self.value(item, level + 1)) - .collect::, _>>()?; - Ok(self.container(('[', ']'), members, level)) + JsonValue::Null => output.push_str("null"), + JsonValue::Bool(flag) => output.push_str(if *flag { "true" } else { "false" }), + JsonValue::Integer(number) => output.push_str(&self.integer(number)?), + JsonValue::Float(number) => output.push_str(&self.float(*number)?), + JsonValue::String(text) => return Ok(with_json_string(output, text, self.escaping)), + JsonValue::Utf16(units) => { + return with_json_utf16_string(output, units, self.escaping); } - JsonValue::Object(entries) => self.object(entries, level), + JsonValue::Array(items) => return self.with_array(output, items, level), + JsonValue::Object(entries) => return self.with_object(output, entries, level), + } + Ok(output) + } + + /// `output` followed by the text before the member at `index` of a container at `level`. + fn with_separator(&self, mut output: String, index: usize, level: usize) -> String { + if index > 0 { + output.push_str(self.item_separator); + } + self.with_newline(output, level + 1) + } + + fn with_array( + &self, + mut output: String, + items: &[JsonValue], + level: usize, + ) -> Result { + if items.is_empty() { + output.push_str("[]"); + return Ok(output); } + output.push('['); + for (index, item) in items.iter().enumerate() { + output = self.with_separator(output, index, level); + output = self.with_value(output, item, level + 1)?; + } + output = self.with_newline(output, level); + output.push(']'); + Ok(output) } - fn object( + fn with_object( &self, + mut output: String, entries: &[(String, JsonValue)], level: usize, ) -> Result { + if entries.is_empty() { + output.push_str("{}"); + return Ok(output); + } let mut ordered: Vec<&(String, JsonValue)> = entries.iter().collect(); if self.sort_keys { ordered.sort_by(|left, right| left.0.cmp(&right.0)); } - let members = ordered - .into_iter() - .map(|(key, item)| { - Ok(format!( - "{}{}{}", - json_string(key, self.escaping), - self.key_separator, - self.value(item, level + 1)? - )) - }) - .collect::, _>>()?; - Ok(self.container(('{', '}'), members, level)) - } - - fn container(&self, brackets: (char, char), members: Vec, level: usize) -> String { - if members.is_empty() { - return format!("{}{}", brackets.0, brackets.1); + output.push('{'); + for (index, (key, item)) in ordered.into_iter().enumerate() { + output = self.with_separator(output, index, level); + output = with_json_string(output, key, self.escaping); + output.push_str(self.key_separator); + output = self.with_value(output, item, level + 1)?; } - let opening = self.newline(level + 1); - let separator = format!("{}{opening}", self.item_separator); - let closing = self.newline(level); - format!( - "{}{opening}{}{closing}{}", - brackets.0, - members.join(&separator), - brackets.1 - ) + output = self.with_newline(output, level); + output.push('}'); + Ok(output) } } pub(crate) fn emit(value: &JsonValue, dialect: &JsonDialect) -> Result { - Layout::new(dialect).value(value, 0) + Layout::new(dialect).with_value(String::new(), value, 0) } diff --git a/crates/sqlbuild-core/src/json/_helpers/strings.rs b/crates/sqlbuild-core/src/json/_helpers/strings.rs index 9152d34258..76c45d8a18 100644 --- a/crates/sqlbuild-core/src/json/_helpers/strings.rs +++ b/crates/sqlbuild-core/src/json/_helpers/strings.rs @@ -1,5 +1,7 @@ //! JSON string escaping of `json.dumps` and orjson. +use std::fmt::Write; + use crate::json::constants::FIRST_PRINTABLE_ASCII; use crate::json::errors::JsonEmitError; @@ -12,67 +14,78 @@ pub(crate) enum StringEscaping { Unicode, } -fn unicode_escape(unit: u32) -> String { - format!("\\u{unit:04x}") +fn with_unicode_escape(mut output: String, unit: u32) -> String { + let _ = write!(output, "\\u{unit:04x}"); + output } -fn escaped(character: char, escaping: StringEscaping) -> String { +fn with_escaped(mut output: String, character: char, escaping: StringEscaping) -> String { match character { - '"' => "\\\"".to_owned(), - '\\' => "\\\\".to_owned(), - '\n' => "\\n".to_owned(), - '\r' => "\\r".to_owned(), - '\t' => "\\t".to_owned(), - '\u{8}' => "\\b".to_owned(), - '\u{c}' => "\\f".to_owned(), - ' '..='~' => character.to_string(), - _ if character < FIRST_PRINTABLE_ASCII => unicode_escape(u32::from(character)), - _ if escaping == StringEscaping::Unicode => character.to_string(), - _ => character - .encode_utf16(&mut [0_u16; 2]) - .iter() - .map(|unit| unicode_escape(u32::from(*unit))) - .collect(), + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + '\u{8}' => output.push_str("\\b"), + '\u{c}' => output.push_str("\\f"), + ' '..='~' => output.push(character), + _ if character < FIRST_PRINTABLE_ASCII => { + return with_unicode_escape(output, u32::from(character)); + } + _ if escaping == StringEscaping::Unicode => output.push(character), + _ => { + for unit in character.encode_utf16(&mut [0_u16; 2]).iter() { + output = with_unicode_escape(output, u32::from(*unit)); + } + } } + output +} + +/// Whether `byte` is printable ASCII that a JSON string carries unescaped. +fn plain(byte: u8) -> bool { + (b' '..=b'~').contains(&byte) && byte != b'"' && byte != b'\\' } -/// `text` as a quoted JSON string. -pub(crate) fn json_string(text: &str, escaping: StringEscaping) -> String { - let mut output = String::with_capacity(text.len() + 2); +/// `output` followed by `text` as a quoted JSON string. +pub(crate) fn with_json_string(mut output: String, text: &str, escaping: StringEscaping) -> String { + output.reserve(text.len() + 2); output.push('"'); - for character in text.chars() { - match character { - ' '..='~' if character != '"' && character != '\\' => output.push(character), - _ => output.push_str(&escaped(character, escaping)), - } + let mut rest: &str = text; + while !rest.is_empty() { + let run: usize = rest + .bytes() + .position(|byte| !plain(byte)) + .unwrap_or(rest.len()); + output.push_str(&rest[..run]); + rest = &rest[run..]; + let Some(character) = rest.chars().next() else { + break; + }; + output = with_escaped(output, character, escaping); + rest = &rest[character.len_utf8()..]; } output.push('"'); output } -/// UTF-16 `units` as a quoted JSON string; `ensure_ascii` escapes a lone surrogate as `\uXXXX`. -pub(crate) fn json_utf16_string( +/// `output` followed by UTF-16 `units` as a quoted JSON string; `ensure_ascii` escapes lone surrogates. +pub(crate) fn with_json_utf16_string( + mut output: String, units: &[u16], escaping: StringEscaping, ) -> Result { - let mut output = String::with_capacity(units.len() + 2); + output.reserve(units.len() + 2); output.push('"'); for decoded in char::decode_utf16(units.iter().copied()) { - match (decoded, escaping) { - (Ok(character), _) => output.push_str(&escaped_or_plain(character, escaping)), + output = match (decoded, escaping) { + (Ok(character), _) => with_escaped(output, character, escaping), (Err(lone), StringEscaping::Ascii) => { - output.push_str(&unicode_escape(u32::from(lone.unpaired_surrogate()))); + with_unicode_escape(output, u32::from(lone.unpaired_surrogate())) } (Err(_), StringEscaping::Unicode) => return Err(JsonEmitError::LoneSurrogate), - } + }; } output.push('"'); Ok(output) } - -fn escaped_or_plain(character: char, escaping: StringEscaping) -> String { - match character { - ' '..='~' if character != '"' && character != '\\' => character.to_string(), - _ => escaped(character, escaping), - } -} From 0e9947f5c6949d791e4ca6420373b58a960ef23f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:59:04 +0100 Subject: [PATCH 3/6] perf: compute compile JSON semantic coverage once The --json report computed the selected models' semantic coverage twice: once for the stderr notice and again for semantic_checks_partial. Compute it once and hand the notice to the caller before the report is assembled. --- .../_helpers/compile_command/report.rs | 81 +++++++++++-------- .../_helpers/compile_driver/command.rs | 13 ++- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 480d6346bb..929f554d35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -176,15 +176,7 @@ impl<'py> ReportInputs<'py> { } fn semantic_notice(&self, reader: &SurrogateText) -> PyResult> { - let mut reasons: Vec<(String, Vec)> = Vec::new(); - for (name, values) in self.semantic_coverage()?.iter() { - let values: Vec = values - .try_iter()? - .map(|value| reader.text(&value?)) - .collect::>()?; - reasons.push((reader.text(&name)?, values)); - } - Ok(notice_text(&reasons)) + coverage_notice(&self.semantic_coverage()?, reader) } /// `written.target_dir / "compiled"` relative to its `target` directory, POSIX style. @@ -396,27 +388,20 @@ pub(crate) fn format_compile_text<'py>( reader.python(py, &compile_text(&report)) } -/// The partial semantic-check notice of the selected models, printed beside `--json` output. -pub(crate) fn json_semantic_notice<'py>( - graph: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, - sql_validation_enabled: bool, -) -> PyResult>> { - let py: Python<'py> = graph.py(); - let reader: SurrogateText = SurrogateText::default(); - let notice: Option = ReportInputs { - project: native_field(graph, "project")?, - graph: graph.clone(), - written: py.None().into_bound(py), - manifest: false, - lineage: py.None().into_bound(py), - diagnostics: Vec::new(), - lineage_mode: String::new(), - selected_keys: selected_keys.clone(), - sql_validation_enabled, +/// The partial semantic-check notice of these coverage reasons, or `None` when complete. +fn coverage_notice( + coverage: &Bound<'_, PyDict>, + reader: &SurrogateText, +) -> PyResult> { + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for (name, values) in coverage.iter() { + let values: Vec = values + .try_iter()? + .map(|value| reader.text(&value?)) + .collect::>()?; + reasons.push((reader.text(&name)?, values)); } - .semantic_notice(&reader)?; - notice.map(|notice| reader.python(py, ¬ice)).transpose() + Ok(notice_text(&reasons)) } fn dict<'py>( @@ -929,9 +914,37 @@ pub(crate) fn format_compile_json( py: Python<'_>, options: Option>, ) -> PyResult { - let arguments: KeywordArguments<'_> = + compile_json(py, options, None::) -> PyResult<()>>) +} + +/// `format_compile_json`, first handing the selected models' partial-check notice to `notice`. +pub(crate) fn format_compile_json_with_notice<'py>( + py: Python<'py>, + options: Option>, + notice: impl FnOnce(Bound<'py, PyString>) -> PyResult<()>, +) -> PyResult { + compile_json(py, options, Some(notice)) +} + +fn compile_json<'py>( + py: Python<'py>, + options: Option>, + notice: Option) -> PyResult<()>>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "format_compile_json", options, REPORT_OPTIONS)?; - let inputs: ReportInputs<'_> = ReportInputs::read(&arguments)?; + let inputs: ReportInputs<'py> = ReportInputs::read(&arguments)?; + let coverage: Option> = match notice { + Some(notice) => { + let coverage: Bound<'py, PyDict> = inputs.semantic_coverage()?; + let reader: SurrogateText = SurrogateText::default(); + if let Some(text) = coverage_notice(&coverage, &reader)? { + notice(reader.python(py, &text)?)?; + } + Some(coverage) + } + None => None, + }; let project: &Bound<'_, PyAny> = &inputs.project; let mut has_errors: bool = false; let mut diagnostics: Vec> = Vec::new(); @@ -1019,7 +1032,11 @@ pub(crate) fn format_compile_json( ("diagnostics", PyList::new(py, diagnostics)?.into_any()), ( "semantic_checks_partial", - inputs.semantic_coverage()?.into_any(), + match coverage { + Some(coverage) => coverage, + None => inputs.semantic_coverage()?, + } + .into_any(), ), ("compile_timings", arguments.value("timings_ms")?), ("lineage_mode", any(py, inputs.lineage_mode.as_str())?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 61fa9bf4f9..6a554300b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -8,7 +8,7 @@ use std::rc::Rc; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ - format_compile_json, format_compile_text, json_semantic_notice, + format_compile_json_with_notice, format_compile_text, }; use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_compile; use crate::bindings::_helpers::compile_command::reuse_recording::{ @@ -435,16 +435,15 @@ fn record_output<'py>( ]; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); let rendered: Bound<'py, PyString> = if native_field(request, "json_output")?.is_truthy()? { - let notice: Option> = - json_semantic_notice(&graph, &selected_keys, validation_enabled.is_truthy()?)?; - if let Some(notice) = notice { - output.call_method1("print_stderr_line", (notice,))?; - } arguments.push(("timings_ms", timings.clone().into_any())); arguments.push(("sql_validation_enabled", validation_enabled)); + let print_notice = |notice: Bound<'py, PyString>| -> PyResult<()> { + output.call_method1("print_stderr_line", (notice,))?; + Ok(()) + }; PyString::new( py, - &format_compile_json(py, Some(keywords(py, &arguments)?))?, + &format_compile_json_with_notice(py, Some(keywords(py, &arguments)?), print_notice)?, ) } else { let use_color: bool = From 0c057bc1c717cc0a12012da956367f788c2d2202 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:59:04 +0100 Subject: [PATCH 4/6] perf: write compile artifact groups behind the queueing work write_static_target alternated between queueing a group on the main thread and waiting for its files to be written. Model, function and audit groups now go to one writer thread in order while the next group is queued; the SQL test group's synchronous flush waits for them, so records, stale cleanup and the manifest still see every file on disk. The writer stops at the first failure and no later group is written. --- .../compile_command/artifact_batch.rs | 51 +++++++++++++- .../_helpers/compile_command/target_writer.rs | 6 +- .../_helpers/compile_outputs/artifacts.rs | 68 +++++++++++++++++++ .../compile/test_compile_target_writer.py | 32 +++++++++ 4 files changed, 151 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs index fae39e9dc2..1140c789d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -7,14 +7,21 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_command::reuse_recording::{ ArtifactRecorder, artifact_written, artifact_written_at, }; +use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timing; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; -use crate::bindings::_helpers::compile_outputs::artifacts::write_compiled_artifacts; +use crate::bindings::_helpers::compile_outputs::artifacts::{ + BackgroundArtifactWriter, write_compiled_artifacts, +}; + +const PHYSICAL_WRITE_PHASE: &str = "physical_write_ms"; /// The artifacts of one write group, written together by `flush`. pub(crate) struct ArtifactBatch<'a> { check_existing: bool, recorder: ArtifactRecorder<'a>, files: Vec<(PathBuf, Vec)>, + /// Earlier groups, still being written in the background. + writer: Option, } /// `contents` with platform line endings. @@ -48,6 +55,7 @@ impl<'a> ArtifactBatch<'a> { check_existing, recorder, files: Vec::new(), + writer: None, } } @@ -77,14 +85,51 @@ impl<'a> ArtifactBatch<'a> { Ok(()) } - /// Write every queued artifact natively and empty the queue. + /// Write every queued artifact natively, after any background group, and empty the queue. pub(crate) fn flush(&mut self, py: Python<'_>) -> PyResult<()> { + self.finish_pending(py)?; if self.files.is_empty() { return Ok(()); } let files: Vec<(PathBuf, Vec)> = std::mem::take(&mut self.files); - record_compile_timing("physical_write_ms", || { + record_compile_timing(PHYSICAL_WRITE_PHASE, || { write_compiled_artifacts(py, files, self.check_existing).map(|_| ()) }) } + + /// Write the queued artifacts in the background, after every earlier background group. + pub(crate) fn flush_in_background(&mut self, py: Python<'_>) -> PyResult<()> { + if self.files.is_empty() { + return Ok(()); + } + if self.writer.is_none() { + self.writer = BackgroundArtifactWriter::start(); + } + let Some(writer) = &self.writer else { + return self.flush(py); + }; + writer.send(std::mem::take(&mut self.files), self.check_existing); + Ok(()) + } + + /// Wait for the background groups, raising the first write error. + fn finish_pending(&mut self, py: Python<'_>) -> PyResult<()> { + let Some(writer) = self.writer.take() else { + return Ok(()); + }; + let elapsed = writer.finish(py)?; + add_compile_timing( + PHYSICAL_WRITE_PHASE, + u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX), + ); + Ok(()) + } +} + +impl Drop for ArtifactBatch<'_> { + fn drop(&mut self) { + if let Some(writer) = self.writer.take() { + writer.wait(); + } + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index c65aeee020..356dad5877 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -611,7 +611,7 @@ pub(crate) fn write_static_target<'py>( )?; managed.push(path); } - batch.flush(py)?; + batch.flush_in_background(py)?; for function in native_field(project, "functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; if native_field(&function, "destination")? @@ -628,7 +628,7 @@ pub(crate) fn write_static_target<'py>( batch.queue_sql(&path, &sql)?; managed.push(path); } - batch.flush(py)?; + batch.flush_in_background(py)?; for audit in native_field(project, "audits")?.try_iter()? { let audit: Bound<'py, PyAny> = audit?; let path: Bound<'py, PyAny> = compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; @@ -638,7 +638,7 @@ pub(crate) fn write_static_target<'py>( )?; managed.push(path); } - batch.flush(py)?; + batch.flush_in_background(py)?; let (test_paths, diagnostics) = write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; managed.extend(test_paths); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs index b1718b837f..5c717f8e0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs @@ -1,7 +1,11 @@ //! Write, clean and publish compiled artifacts natively, raising the errors Python raised. use std::collections::HashSet; +use std::panic::resume_unwind; use std::path::PathBuf; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; use pyo3::exceptions::PyOSError; use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; @@ -17,6 +21,11 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; type PublishedRow = (String, Vec<(String, String)>); +/// One write group's `(path, contents)` artifacts and whether unchanged files are kept. +type ArtifactGroup = (Vec<(PathBuf, Vec)>, bool); + +const WRITER_THREAD: &str = "sqlbuild-artifact-writer"; + /// Write each `(path, contents)` artifact; returns how many were written and left unchanged. pub(crate) fn write_compiled_artifacts( py: Python<'_>, @@ -31,6 +40,65 @@ pub(crate) fn write_compiled_artifacts( }) } +/// Artifact groups written one after another on their own thread, stopping at the first error. +pub(crate) struct BackgroundArtifactWriter { + groups: Sender, + handle: JoinHandle<(Result<(), ArtifactError>, Duration)>, +} + +/// Write each group in order until one fails; returns that failure and the time spent writing. +fn write_groups(groups: &Receiver) -> (Result<(), ArtifactError>, Duration) { + let mut elapsed: Duration = Duration::ZERO; + for (files, check_existing) in groups { + let started: Instant = Instant::now(); + let written: Result = + write_artifacts(&files, check_existing); + elapsed += started.elapsed(); + if let Err(error) = written { + return (Err(error), elapsed); + } + } + (Ok(()), elapsed) +} + +impl BackgroundArtifactWriter { + /// Start the writer thread, or `None` when no thread could start. + pub(crate) fn start() -> Option { + let (groups, received) = channel(); + let spawned = std::thread::Builder::new() + .name(WRITER_THREAD.to_owned()) + .spawn(move || write_groups(&received)); + match spawned { + Ok(handle) => Some(Self { groups, handle }), + Err(_) => None, + } + } + + /// Queue one group behind the groups already sent; a writer that stopped drops it. + pub(crate) fn send(&self, files: Vec<(PathBuf, Vec)>, check_existing: bool) { + let _ = self.groups.send((files, check_existing)); + } + + /// Wait without the GIL for every sent group; returns the time spent writing. + pub(crate) fn finish(self, py: Python<'_>) -> PyResult { + let Self { groups, handle } = self; + drop(groups); + compiler_guard(|| { + let joined = py.detach(|| handle.join()); + let (outcome, elapsed) = joined.unwrap_or_else(|payload| resume_unwind(payload)); + outcome.map_err(|error| artifact_error(py, error))?; + Ok(elapsed) + }) + } + + /// Wait for every sent group, discarding the outcome. + pub(crate) fn wait(self) { + let Self { groups, handle } = self; + drop(groups); + let _ = handle.join(); + } +} + /// Delete unmanaged compiled files and empty directories; returns how many files were removed. pub(crate) fn remove_stale_artifacts( py: Python<'_>, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 335c170747..c6234a1aa5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -334,6 +334,38 @@ def test_given_invalid_utf8_artifact_when_writing_again_then_existing_decode_err ) +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="blocked model directory stops later groups", expected_message="staging" + ) + ], + ids=lambda case: case.description, +) +def test_given_unwritable_model_artifact_when_writing_static_target_then_later_groups_are_not_written( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + project: CompiledProjectView = build_static_target_writer_project() + target_dir: Path = tmp_path / "target" + blocker: Path = target_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory\n", encoding="utf-8") + + with pytest.raises(OSError, match=test_case.expected_message): + write_static_compile_target( + target_dir=target_dir, + adapter=DuckDbAdapter(), + project=project, + ) + + assert blocker.read_text(encoding="utf-8") == "not a directory\n" + assert not (target_dir / "compiled" / "functions").exists() + assert not (target_dir / "compiled" / "audits").exists() + assert not (target_dir / "compiled" / "tests").exists() + + @pytest.mark.parametrize( "test_case", (TargetWriterCacheTestCase(description="unchanged artifact reuse", expected_builder_calls=0),), From 5d552e08065ed2fbf28073cc1122eb6876e259c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:59:05 +0100 Subject: [PATCH 5/6] perf: leave compile state for process exit in the launched CLI The launched sqlbuild-python process exits right after its command, yet each compile spent about half a second freeing the project graph and collecting it during interpreter finalization. The launcher now asks compiles to keep that state alive and freeze it out of collections; in-process entries still free it normally. --- .../_helpers/compile_driver/command.rs | 30 ++++++++++-- src/sqlbuild/_native.pyi | 1 + src/sqlbuild/cli/entry/main/launched.py | 5 +- .../src/sqlbuild/cli/entry/_test_types.py | 9 ++++ .../src/sqlbuild/cli/entry/helpers.py | 10 ++++ .../cli/entry/test_launched_teardown.py | 48 +++++++++++++++++++ 6 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 6a554300b0..5942689a4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -1,7 +1,10 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyString, PyTuple, +}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use std::rc::Rc; @@ -36,6 +39,23 @@ use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const BYPASS_OUTCOME: &str = "bypass"; +/// Compile state kept alive for process exit to reclaim, once a launcher defers teardown. +static RETAINED_UNTIL_EXIT: PyOnceLock> = PyOnceLock::new(); + +/// Let compiles in this process leave their project state for process exit to reclaim. +#[pyfunction] +fn defer_compile_teardown_to_exit(py: Python<'_>) { + let _ = RETAINED_UNTIL_EXIT.get_or_init(py, || PyList::empty(py).unbind()); +} + +/// Keep `value` alive until the process exits when teardown is deferred. +fn retain_until_exit(value: &Bound<'_, PyAny>) -> PyResult<()> { + match RETAINED_UNTIL_EXIT.get(value.py()) { + Some(retained) => retained.bind(value.py()).append(value), + None => Ok(()), + } +} + /// What one full compile reports for reuse recording. struct CommandOutcome<'py> { exit_code: i64, @@ -87,8 +107,9 @@ pub(crate) fn run_compile_command<'py>( (&output, &status), total_start, ); - if was_enabled { - gc.call_method0("enable")?; + let deferred: bool = RETAINED_UNTIL_EXIT.get(py).is_some(); + if deferred || was_enabled { + gc.call_method0(if deferred { "freeze" } else { "enable" })?; } exit_code } @@ -168,6 +189,7 @@ fn recorded_compile<'py>( ), ], )?; + retain_until_exit(&manager)?; within(&manager, |prepared_artifacts| { let recorders: Recorders<'py> = Recorders { detailed_timings: Rc::clone(detailed_timings), @@ -248,6 +270,7 @@ fn compiled_with_status<'py>( py, &analysis_options(request, &project_dir, recorders, status)?, )?; + retain_until_exit(&analysis)?; let rules_failed: bool = rules_failed(&analysis)?; let skip_write: bool = native_field(request, "profile_flags")? .getattr("skip_write")? @@ -460,5 +483,6 @@ fn record_output<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(defer_compile_teardown_to_exit, module)?)?; module.add_function(wrap_pyfunction!(run_compile_command, module)?) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9fcce9f310..959e55faa1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4268,6 +4268,7 @@ def build_compile_inputs( resolved_connection: dict[str, object] | None = None, no_cache: bool = False, ) -> CompileInputsView: ... +def defer_compile_teardown_to_exit() -> None: ... def run_compile_command( request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None ) -> int: ... diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index b544413b74..4925ae9ab4 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -11,13 +11,14 @@ def main() -> int: - """Run the CLI with the `sqb` arguments, setting the allocator purge delay first.""" + """Run the CLI with the `sqb` arguments in a process that exits once the command returns.""" os.environ.setdefault( NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS ) - from sqlbuild._native import LAUNCH_RECORD_OPTION + from sqlbuild._native import LAUNCH_RECORD_OPTION, defer_compile_teardown_to_exit + _ = defer_compile_teardown_to_exit() arguments: list[str] = sys.argv[1:] offered: bool = bool(arguments) and arguments[0].startswith(LAUNCH_RECORD_OPTION) sys.argv = [sys.argv[0], *arguments[int(offered) :]] diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 71438db67b..2f6143377e 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -73,3 +73,12 @@ class SelectorCompileTestCase: description: str args: tuple[str, ...] expected_model: bytes + + +@dataclass(frozen=True) +class LaunchedTeardownTestCase: + """One compile command line and the reuse hits its repeated launched run reports.""" + + description: str + args: tuple[str, ...] + expected_hits: int diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index f43dbaea72..e81c78ba3a 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -189,3 +189,13 @@ def startup_install(directory: Path) -> tuple[Path, Path, Path]: startup: Path = site_packages / "catalog.pth" _ = startup.write_text(str(old) + "\n", encoding="utf-8") return program, startup, new + + +def compiled_files(project_dir: Path) -> dict[str, bytes]: + """Every compiled SQL artifact under the project's target, by relative path.""" + + compiled: Path = project_dir / "target" / "compiled" + return { + path.relative_to(compiled).as_posix(): path.read_bytes() + for path in sorted(compiled.rglob("*.sql")) + } diff --git a/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py b/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py new file mode 100644 index 0000000000..ef18282ea5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py @@ -0,0 +1,48 @@ +import sys +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import LaunchedTeardownTestCase +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + compile_timings, + compiled_files, + run, + without_compile_timings, + write_project, +) + +IN_PROCESS_ENTRY: str = ( + "import sys\nfrom sqlbuild.cli.entry.main.entry import main\nsys.exit(main(sys.argv[1:]))\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + LaunchedTeardownTestCase( + description="json report", args=("compile", "--json"), expected_hits=1 + ) + ], + ids=lambda case: case.description, +) +def test_given_launched_cli_when_compiling_then_output_matches_entry_and_cache_is_reused( + test_case: LaunchedTeardownTestCase, tmp_path: Path +) -> None: + launched_dir: Path = write_project(tmp_path / "launched") + entry_dir: Path = write_project(tmp_path / "entry") + + launched: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=launched_dir) + entry: LauncherRun = run( + Path(sys.executable), args=("-c", IN_PROCESS_ENTRY, *test_case.args), cwd=entry_dir + ) + repeated: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=launched_dir) + + assert launched[0] == entry[0] == repeated[0] == 0 + assert without_compile_timings(launched[1]) == without_compile_timings(entry[1]) + assert compiled_files(launched_dir) == compiled_files(entry_dir) + assert compiled_files(launched_dir) + assert compile_timings(repeated[1])["project_reuse_hits"] == test_case.expected_hits + assert without_compile_timings(repeated[1]) == without_compile_timings(launched[1]) From ddedd86d3062477e7a7c61610447e9767235cc07 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:40:58 +0100 Subject: [PATCH 6/6] fix: report background artifact write errors first and stop on interrupt When a step after a background group failed, the artifact batch discarded the writer's outcome on drop, so a later error could mask an earlier OSError and defeat the staged path's direct-write fallback. Errors from the queueing section now wait for the writer and report its write error first, as the synchronous writes did. Waiting for the writer releases the GIL and checks for signals. An interrupt, or any non-Exception error, stops the writer after the group it is writing, skips groups not yet started, and then propagates. --- .../compile_command/artifact_batch.rs | 22 +++- .../_helpers/compile_command/target_writer.rs | 59 +++++----- .../_helpers/compile_outputs/artifacts.rs | 103 ++++++++++++++++-- .../cli/commands/main/compile/helpers.py | 48 ++++++++ .../compile/test_compile_target_writer.py | 92 ++++++++++++++++ 5 files changed, 284 insertions(+), 40 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs index 1140c789d3..a0bcf70fb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -2,7 +2,8 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use crate::bindings::_helpers::compile_command::reuse_recording::{ ArtifactRecorder, artifact_written, artifact_written_at, @@ -112,6 +113,23 @@ impl<'a> ArtifactBatch<'a> { Ok(()) } + /// `result`, except that a background group's earlier write error wins and interrupts stop it. + pub(crate) fn settle(&mut self, py: Python<'_>, result: PyResult) -> PyResult { + let error: PyErr = match result { + Ok(value) => return Ok(value), + Err(error) => error, + }; + let Some(writer) = self.writer.take() else { + return Err(error); + }; + if !error.is_instance_of::(py) { + writer.stop(py); + return Err(error); + } + writer.finish(py)?; + Err(error) + } + /// Wait for the background groups, raising the first write error. fn finish_pending(&mut self, py: Python<'_>) -> PyResult<()> { let Some(writer) = self.writer.take() else { @@ -129,7 +147,7 @@ impl<'a> ArtifactBatch<'a> { impl Drop for ArtifactBatch<'_> { fn drop(&mut self) { if let Some(writer) = self.writer.take() { - writer.wait(); + Python::attach(|py| writer.stop(py)); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 356dad5877..51a6ac15b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -612,35 +612,38 @@ pub(crate) fn write_static_target<'py>( managed.push(path); } batch.flush_in_background(py)?; - for function in native_field(project, "functions")?.try_iter()? { - let function: Bound<'py, PyAny> = function?; - if native_field(&function, "destination")? - .getattr("qualified_name")? - .is_none() - { - continue; + let queued: PyResult> = (|| { + for function in native_field(project, "functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + if native_field(&function, "destination")? + .getattr("qualified_name")? + .is_none() + { + continue; + } + let sql: String = function_sql(adapter, &function)?; + let path: Bound<'py, PyAny> = compiled.div(function_output_path( + &native_field(&function, "relative_path")?, + &native_field(&function, "language")?, + )?)?; + batch.queue_sql(&path, &sql)?; + managed.push(path); } - let sql: String = function_sql(adapter, &function)?; - let path: Bound<'py, PyAny> = compiled.div(function_output_path( - &native_field(&function, "relative_path")?, - &native_field(&function, "language")?, - )?)?; - batch.queue_sql(&path, &sql)?; - managed.push(path); - } - batch.flush_in_background(py)?; - for audit in native_field(project, "audits")?.try_iter()? { - let audit: Bound<'py, PyAny> = audit?; - let path: Bound<'py, PyAny> = compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; - batch.queue_sql( - &path, - &native_field(&audit, "sql_body")?.extract::()?, - )?; - managed.push(path); - } - batch.flush_in_background(py)?; - let (test_paths, diagnostics) = - write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; + batch.flush_in_background(py)?; + for audit in native_field(project, "audits")?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let path: Bound<'py, PyAny> = + compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; + batch.queue_sql( + &path, + &native_field(&audit, "sql_body")?.extract::()?, + )?; + managed.push(path); + } + batch.flush_in_background(py)?; + write_static_tests(target_dir, (adapter, project), prepared, &mut batch) + })(); + let (test_paths, diagnostics) = batch.settle(py, queued)?; managed.extend(test_paths); if remove_stale_files { let mut paths: Vec = managed_models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs index 5c717f8e0b..8dcc012f25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs @@ -3,7 +3,9 @@ use std::collections::HashSet; use std::panic::resume_unwind; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Condvar, Mutex, PoisonError}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; @@ -25,6 +27,7 @@ type PublishedRow = (String, Vec<(String, String)>); type ArtifactGroup = (Vec<(PathBuf, Vec)>, bool); const WRITER_THREAD: &str = "sqlbuild-artifact-writer"; +const WRITER_POLL: Duration = Duration::from_millis(50); /// Write each `(path, contents)` artifact; returns how many were written and left unchanged. pub(crate) fn write_compiled_artifacts( @@ -43,13 +46,23 @@ pub(crate) fn write_compiled_artifacts( /// Artifact groups written one after another on their own thread, stopping at the first error. pub(crate) struct BackgroundArtifactWriter { groups: Sender, + /// Set to skip every group not yet started; the group being written still completes. + stop: Arc, + /// Marked once the writer returns, so a wait can check for signals meanwhile. + finished: Arc, handle: JoinHandle<(Result<(), ArtifactError>, Duration)>, } -/// Write each group in order until one fails; returns that failure and the time spent writing. -fn write_groups(groups: &Receiver) -> (Result<(), ArtifactError>, Duration) { +/// Write each group in order until one fails or a stop is requested between groups. +fn write_groups( + groups: &Receiver, + stop: &AtomicBool, +) -> (Result<(), ArtifactError>, Duration) { let mut elapsed: Duration = Duration::ZERO; for (files, check_existing) in groups { + if stop.load(Ordering::SeqCst) { + break; + } let started: Instant = Instant::now(); let written: Result = write_artifacts(&files, check_existing); @@ -65,11 +78,23 @@ impl BackgroundArtifactWriter { /// Start the writer thread, or `None` when no thread could start. pub(crate) fn start() -> Option { let (groups, received) = channel(); + let finished: Arc = Arc::new(WriterDone::default()); + let marker: DoneMarker = DoneMarker(Arc::clone(&finished)); + let stop: Arc = Arc::new(AtomicBool::new(false)); + let stopping: Arc = Arc::clone(&stop); let spawned = std::thread::Builder::new() .name(WRITER_THREAD.to_owned()) - .spawn(move || write_groups(&received)); + .spawn(move || { + let _marker: DoneMarker = marker; + write_groups(&received, &stopping) + }); match spawned { - Ok(handle) => Some(Self { groups, handle }), + Ok(handle) => Some(Self { + groups, + stop, + finished, + handle, + }), Err(_) => None, } } @@ -79,10 +104,20 @@ impl BackgroundArtifactWriter { let _ = self.groups.send((files, check_existing)); } - /// Wait without the GIL for every sent group; returns the time spent writing. + /// Wait without the GIL for every sent group; a signal stops it after the current group. pub(crate) fn finish(self, py: Python<'_>) -> PyResult { - let Self { groups, handle } = self; + let Self { + groups, + stop, + finished, + handle, + } = self; drop(groups); + if let Err(interrupt) = wait_for_writer(py, &finished) { + stop.store(true, Ordering::SeqCst); + let _ = py.detach(|| handle.join()); + return Err(interrupt); + } compiler_guard(|| { let joined = py.detach(|| handle.join()); let (outcome, elapsed) = joined.unwrap_or_else(|payload| resume_unwind(payload)); @@ -91,12 +126,60 @@ impl BackgroundArtifactWriter { }) } - /// Wait for every sent group, discarding the outcome. - pub(crate) fn wait(self) { - let Self { groups, handle } = self; + /// Stop after the group being written and wait for it without the GIL, ignoring the outcome. + pub(crate) fn stop(self, py: Python<'_>) { + let Self { + groups, + stop, + finished, + handle, + } = self; + stop.store(true, Ordering::SeqCst); drop(groups); - let _ = handle.join(); + drop(finished); + let _ = py.detach(|| handle.join()); + } +} + +/// Whether the writer thread has returned, with a condition variable to wait on it. +#[derive(Default)] +struct WriterDone { + finished: Mutex, + changed: Condvar, +} + +/// Marks the writer done when the writer thread returns or unwinds. +struct DoneMarker(Arc); + +impl Drop for DoneMarker { + fn drop(&mut self) { + *self + .0 + .finished + .lock() + .unwrap_or_else(PoisonError::into_inner) = true; + self.0.changed.notify_all(); + } +} + +impl WriterDone { + /// Whether the writer returned within `timeout`. + fn finished_within(&self, timeout: Duration) -> bool { + let finished = self.finished.lock().unwrap_or_else(PoisonError::into_inner); + let (finished, _) = self + .changed + .wait_timeout_while(finished, timeout, |finished| !*finished) + .unwrap_or_else(PoisonError::into_inner); + *finished + } +} + +/// Wait without the GIL until the writer returns, checking for signals between waits. +fn wait_for_writer(py: Python<'_>, finished: &WriterDone) -> PyResult<()> { + while !py.detach(|| finished.finished_within(WRITER_POLL)) { + py.check_signals()?; } + Ok(()) } /// Delete unmanaged compiled files and empty directories; returns how many files were removed. diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 17410ca9a1..bad4b27454 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -609,3 +609,51 @@ def staged_then_published( _ = artifacts.planning_diagnostics() before_publish(target) return artifacts.publish(target_dir=target, manifest=None) + + +class FunctionRenderFailingAdapter(DuckDbAdapter): + """A DuckDB adapter whose function rendering fails after the model group is handed off.""" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + raise ValueError(f"function rendering failed for {destination}") + + +class FunctionRenderInterruptedAdapter(DuckDbAdapter): + """A DuckDB adapter interrupted while rendering a function after the model group.""" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + raise KeyboardInterrupt(destination) + + +class BlockedModelStaging(StagingEveryProject): + """Stage artifacts into a staging tree whose model directory is a file.""" + + def create_staging_directory(self, staging_dir: Path) -> None: + super().create_staging_directory(staging_dir) + blocker: Path = staging_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + _ = blocker.write_text("not a directory\n", encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index c6234a1aa5..e2321f4467 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import time from dataclasses import replace from pathlib import Path @@ -35,6 +36,9 @@ TargetWriterTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( + BlockedModelStaging, + FunctionRenderFailingAdapter, + FunctionRenderInterruptedAdapter, build_cached_target_writer_project, build_missing_mock_target_writer_project, build_static_target_writer_project, @@ -44,6 +48,7 @@ from tests.unit.src.sqlbuild.compiler.manifest.helpers import build_test_project SQL_TEST_ARTIFACT_PLANS: str = "plan_compiled_sql_test_artifacts" +INTERRUPT_RETURN_SECONDS: float = 10.0 SQL_TEST_MODEL_CHAINS: str = "sql_test_model_chain_names_by_key" @@ -366,6 +371,93 @@ def test_given_unwritable_model_artifact_when_writing_static_target_then_later_g assert not (target_dir / "compiled" / "tests").exists() +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="the model write error wins over a later rendering error", + expected_message="staging", + ) + ], + ids=lambda case: case.description, +) +def test_given_model_write_failure_and_later_error_when_writing_static_target_then_os_error_wins( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + blocker: Path = target_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory\n", encoding="utf-8") + + with pytest.raises(OSError, match=test_case.expected_message): + write_static_compile_target( + target_dir=target_dir, + adapter=FunctionRenderFailingAdapter(), + project=build_static_target_writer_project(), + ) + + assert not (target_dir / "compiled" / "functions").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="interrupt after the model group", expected_message="compiled/functions" + ) + ], + ids=lambda case: case.description, +) +def test_given_interrupt_after_model_group_when_writing_static_target_then_later_groups_are_not_written( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + started: float = time.monotonic() + + with pytest.raises(KeyboardInterrupt): + write_static_compile_target( + target_dir=target_dir, + adapter=FunctionRenderInterruptedAdapter(), + project=build_static_target_writer_project(), + ) + + assert time.monotonic() - started < INTERRUPT_RETURN_SECONDS + assert not (target_dir / test_case.expected_message).exists() + assert not (target_dir / "compiled" / "audits").exists() + assert not (target_dir / "compiled" / "tests").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="staged model write failure falls back despite a later error", + expected_message="compiled/models/staging/orders.sql", + ) + ], + ids=lambda case: case.description, +) +def test_given_staged_model_write_failure_and_later_error_when_publishing_then_staging_falls_back( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + + with BlockedModelStaging(enabled=True) as artifacts: + artifacts.start( + project=build_static_target_writer_project(), + adapter=FunctionRenderFailingAdapter(), + target_dir=target_dir, + stage_artifacts=True, + ) + written: WrittenTarget | None = artifacts.publish(target_dir=target_dir, manifest=None) + + assert written is None + assert not (target_dir / test_case.expected_message).exists() + + @pytest.mark.parametrize( "test_case", (TargetWriterCacheTestCase(description="unchanged artifact reuse", expected_builder_calls=0),),