From b74e57f5499e444335377f4b762556fd58f707b3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:15:52 +0100 Subject: [PATCH 01/11] refactor(planner): call the last planner helpers directly and count planner work natively Planning, migration fingerprints, identities, SQL-test planning and table types no longer go through _native module lookups. Tests that replaced those functions now observe native work counters (planner_identity_work_info, migration_fingerprint_work_info), drive planning through project and adapter inputs, or inject table-type drift at the executor hand-off. Freshness errors, cost scopes, progress reporters and the event writer are constructed directly. SQL-test planning and chain requests are Rust structs; their records and pyfunctions are removed. --- .../bindings/_helpers/check/check_command.rs | 73 ++++++----- .../_helpers/cli/progress_reporters.rs | 8 +- .../_helpers/compile_driver/plan_result.rs | 57 +++++---- .../_helpers/compiler_records/constants.rs | 29 ----- .../_helpers/cost_telemetry/context.rs | 5 +- .../bindings/_helpers/freshness/planning.rs | 31 ++--- .../_helpers/freshness/publication.rs | 55 +++++---- .../bindings/_helpers/freshness/support.rs | 16 ++- .../_helpers/janitor/janitor_phases.rs | 85 +++++++------ .../_helpers/planner_changes/detect.rs | 2 +- .../_helpers/planner_entries/test_planning.rs | 21 ++-- .../_helpers/planner_identity/identities.rs | 31 +++-- .../planner_migrations/fingerprint.rs | 24 +++- .../_helpers/planner_phases/identities.rs | 108 +++++++++------- .../_helpers/planner_phases/models.rs | 10 ++ .../planner_phases/output_assembly.rs | 44 +++---- .../_helpers/planner_phases/registration.rs | 4 + .../planner_strategy/fixture_context.rs | 15 ++- .../planner_strategy/fixture_reads.rs | 19 +-- .../planner_warehouse/fingerprint_cache.rs | 65 +++++++--- .../_helpers/planner_warehouse/mod.rs | 1 + .../_helpers/planner_warehouse/models.rs | 13 ++ .../planner_warehouse/registration.rs | 4 + .../src/bindings/_helpers/records/ids.rs | 10 -- .../_helpers/result_output/event_writer.rs | 2 +- .../_helpers/sql_test_glue/plan_driver.rs | 108 ++++++---------- .../_helpers/sql_test_glue/plan_rows.rs | 115 +++++++++--------- .../_helpers/sql_test_glue/planning.rs | 72 ++++------- .../_helpers/sql_test_glue/registration.rs | 3 +- src/sqlbuild/_native.pyi | 75 +----------- .../compile/test_native_analysis_seams.py | 9 +- .../commands/main/model_migrations/helpers.py | 35 ++---- .../model_migrations/test_fingerprint_work.py | 16 +-- .../planner/main/test_build_execution_plan.py | 22 +--- .../src/sqlbuild/executor/build/helpers.py | 27 ++-- .../executor/build/test_table_type_routing.py | 13 +- .../compiler/pipeline/main/_test_types.py | 4 +- .../compiler/pipeline/main/helpers.py | 28 +++++ .../main/test_compile_phase_timings.py | 69 +++++------ .../compiler/planner/classes/helpers.py | 38 ++---- .../test_migration_fingerprint_cache.py | 36 ++---- .../planner/classes/test_sql_test_planning.py | 110 +++-------------- .../classes/test_sql_test_planning_exit.py | 29 ++++- 43 files changed, 685 insertions(+), 856 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 5597092321..e3f5a1d425 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -43,9 +43,10 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; +use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const GRAPH_OPERATOR_CODE: &str = "C681"; @@ -131,8 +132,8 @@ fn selected_check_names<'py>( graph: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); - let select: Bound<'py, PyAny> = native_field(request, "select")?; - let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; + let select: Bound<'py, PyAny> = native_field(&request, "select")?; + let exclude: Bound<'py, PyAny> = native_field(&request, "exclude")?; let selectors: Vec> = select .try_iter()? .chain(exclude.try_iter()?) @@ -158,7 +159,7 @@ fn selected_check_names<'py>( PyFrozenSet::empty(py)?.into_any() }; let mut names: Vec> = Vec::new(); - for node in native_field(graph, "nodes")?.try_iter()? { + for node in native_field(&graph, "nodes")?.try_iter()? { let node: Bound<'py, PyAny> = node?; let name: Bound<'py, PyAny> = native_field(&node, "name")?; if native_field(&node, "kind")?.eq(&check_kind)? && !excluded.contains(&name)? { @@ -175,16 +176,16 @@ fn compile_pipeline<'py>( (graph, check_names): (&Bound<'py, PyAny>, &Bound<'py, PyFrozenSet>), ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")?; + let discovered: Bound<'py, PyAny> = native_field(&invocation, "discovered_inputs")?; let required_refs: Bound<'py, PyAny> = call_with( - &native_field(graph, "selected_sql_refs")?, + &native_field(&graph, "selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; let closure: Bound<'py, PyAny> = object( py, transitive_closure_many( check_names.as_any(), - &native_field(graph, "upstream_deps")?, + &native_field(&graph, "upstream_deps")?, false, None, )?, @@ -222,22 +223,19 @@ fn compile_pipeline<'py>( } let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver( - native_field(invocation, "effective_project_dir")?, + native_field(&invocation, "effective_project_dir")?, discovered.clone(), )?; let options: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompilePipelineOptions)?, &[ - ("selected_target", native_field(request, "selected_target")?), - ( - "no_sql_validation", - native_field(request, "no_sql_validation")?, - ), + ("selected_target", native_field(&request, "selected_target")?), + ("no_sql_validation", native_field(&request, "no_sql_validation")?), ( "connection_config", - native_field(invocation, "connection_config")?, + native_field(&invocation, "connection_config")?, ), - ("cli_vars", native_field(request, "cli_vars")?), + ("cli_vars", native_field(&request, "cli_vars")?), ("external_sql_reference_resolver", resolver), ], )?; @@ -245,7 +243,7 @@ fn compile_pipeline<'py>( py, &[ ("discovered_inputs", discovered), - ("adapter", native_field(invocation, "adapter")?), + ("adapter", native_field(&invocation, "adapter")?), ("options", options), ("selected_keys", keys.clone()), ("relation_keys", keys), @@ -292,8 +290,8 @@ fn python_selection<'py>( let _py: Python<'py> = request.py(); let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; reject_unit_test_selectors( - &native_field(request, "select")?, - &native_field(request, "exclude")?, + &native_field(&request, "select")?, + &native_field(&request, "exclude")?, discovered, )?; let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; @@ -305,8 +303,8 @@ fn prepare<'py>( invocation: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")?; - let adapter: Bound<'py, PyAny> = native_field(invocation, "adapter")?; + let discovered: Bound<'py, PyAny> = native_field(&invocation, "discovered_inputs")?; + let adapter: Bound<'py, PyAny> = native_field(&invocation, "adapter")?; let (planning_graph, planning_names) = python_selection(request, &discovered)?; let pipeline: Bound<'py, PyAny> = compile_pipeline(request, invocation, (&planning_graph, &planning_names))?; @@ -389,10 +387,10 @@ fn node_runtime<'py>( let py: Python<'py> = invocation.py(); let project: Bound<'py, PyAny> = native_field(&prepared.pipeline, "project")?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ - ("adapter", native_field(invocation, "adapter")?), + ("adapter", native_field(&invocation, "adapter")?), ( "connection_config", - native_field(invocation, "connection_config")?, + native_field(&invocation, "connection_config")?, ), ("connection", connection.clone()), ("run_id", native_field(&project, "run_id")?), @@ -420,7 +418,7 @@ fn record_run_state<'py>( source_map: &Bound<'py, PyAny>, ) -> PyResult<()> { record_python_run_state_results( - &native_field(invocation, "discovered_inputs")?, + &native_field(&invocation, "discovered_inputs")?, &run_state, (&python_results, load_results.as_ref()), Some(source_map), @@ -442,7 +440,7 @@ fn run_checks<'py>( let source_map: Bound<'py, PyAny> = native_field(&plan_output, "source_map")?; let callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::IngressCallbacks)?, - &[("use_color", native_field(invocation, "use_color")?)], + &[("use_color", native_field(&invocation, "use_color")?)], )?; let ingress: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecuteIngressPythonLoaderNodes)?, @@ -481,8 +479,8 @@ fn run_checks<'py>( &source_map, )?; let read_side: Bound<'py, PyAny> = run_read_side_dependencies(&ReadSideInputs { - adapter: native_field(invocation, "adapter")?, - connection_config: native_field(invocation, "connection_config")?, + adapter: native_field(&invocation, "adapter")?, + connection_config: native_field(&invocation, "connection_config")?, connection: connection.clone(), pipeline_result: prepared.pipeline.clone(), python_graph: prepared.python_graph.clone(), @@ -568,11 +566,11 @@ fn execute<'py>( providers: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = invocation.py(); - let adapter: Bound<'py, PyAny> = native_field(invocation, "adapter")?; + let adapter: Bound<'py, PyAny> = native_field(&invocation, "adapter")?; let connection: Bound<'py, PyAny> = - adapter.call_method1("connect", (native_field(invocation, "connection_config")?,))?; + adapter.call_method1("connect", (native_field(&invocation, "connection_config")?,))?; let event_writer: Bound<'py, PyAny> = - registry::native(py, NativeAttr::ExecutionEventWriter)?.call0()?; + Bound::new(py, ExecutionEventWriter::new(py, None)?)?.into_any(); let executed: PyResult>> = (|| { let projector: Bound<'py, PyAny> = current_native_progress_projector(py)?; if !projector.is_none() { @@ -611,9 +609,9 @@ fn complete<'py>( (prepared, results): (&Prepared<'py>, &[Bound<'py, PyAny>]), ) -> PyResult { let py: Python<'py> = request.py(); - let stream: Bound<'py, PyAny> = native_field(invocation, "progress_stream")?; + let stream: Bound<'py, PyAny> = native_field(&invocation, "progress_stream")?; let style: CliStyle = CliStyle { - use_color: native_field(invocation, "use_color")?.extract()?, + use_color: native_field(&invocation, "use_color")?.extract()?, }; write_results( &stream, @@ -639,8 +637,8 @@ fn complete<'py>( write_execution_json_output( py, PyString::new(py, &results_json(results)?).into_any(), - native_field(request, "json_output")?.is_truthy()?, - native_field(request, "json_output_path")?, + native_field(&request, "json_output")?.is_truthy()?, + native_field(&request, "json_output_path")?, )?, )?; Ok(check_exit_code(&texts)) @@ -675,11 +673,8 @@ fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = (|| { - let results: Vec> = execute( - &invocation, - &prepared, - &native_field(&session, "providers")?, - )?; + let results: Vec> = + execute(&invocation, &prepared, &native_field(&session, "providers")?)?; complete(request, &invocation, (&prepared, &results)) })(); let closed: PyResult> = session.call_method0("close"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs index 3e519738d2..ac240e891f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -48,7 +48,7 @@ impl ConnectionProgressReporter { impl ConnectionProgressReporter { #[new] #[pyo3(signature = (**options))] - fn new(py: Python<'_>, options: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "ConnectionProgressReporter", @@ -133,7 +133,11 @@ impl PlanningProgressReporter { impl PlanningProgressReporter { #[new] #[pyo3(signature = (*, stream, use_color = false))] - fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + pub(crate) fn new( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + use_color: bool, + ) -> PyResult { let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; Ok(Self { status: status.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 89409309b2..9951b30ba3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -5,7 +5,10 @@ use crate::bindings::_helpers::compile_driver::deferral::gather_deferred_relatio use crate::bindings::_helpers::compile_driver::deferral::resolve_deferred_target_config; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_plan_entries::python_read_source_names_for; +use crate::bindings::_helpers::compile_driver::python_plan_entries::{ + build_python_run_plan_outputs, python_read_source_names_for, +}; +use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,7 +20,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const SINGLE_CONNECTION: i64 = 1; @@ -122,32 +125,38 @@ pub(crate) fn pipeline_result<'py>( ], )?, )?; - let plan_output: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildExecutionPlan)?, - &[ - ("project", project.clone()), - ("adapter", inputs.adapter.clone()), - ("connection", inputs.connection.clone()), - ("selection", selection), - ("overrides", overrides), - ("deferral", deferral), - ("policies", policies), - ("on_progress", on_progress.clone()), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], + let plan_output: Bound<'py, PyAny> = build_execution_plan( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", inputs.adapter.clone()), + ("connection", inputs.connection.clone()), + ("selection", selection), + ("overrides", overrides), + ("deferral", deferral), + ("policies", policies), + ("on_progress", on_progress.clone()), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?), )?; let custom_materializations: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadCustomMaterializations)? .call1((discovered.getattr("materialization_files")?,))?; - let python_outputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonRunPlanOutputs)?, - &[ - ("discovered_inputs", discovered.clone()), - ("plan_output", plan_output), - ("run_selection", run_selection), - ("selected_python_node_names", selected_python_node_names), - ], + let python_outputs: Bound<'py, PyAny> = build_python_run_plan_outputs( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("plan_output", plan_output), + ("run_selection", run_selection), + ("selected_python_node_names", selected_python_node_names), + ], + )?), )?; call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index f4d373bc63..b983344d60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -209,33 +209,6 @@ const R_MANUAL_LOCATION: RecordSpec = RecordSpec { FieldSpec::required("reason"), ], }; -const R_NATIVE_SQL_TEST_PLANNING_REQUEST: RecordSpec = RecordSpec { - name: "NativeSqlTestPlanningRequest", - doc: "Compiled models and tests to plan, with the adapter facts native planning cannot derive.", - fields: &[ - FieldSpec::required("models"), - FieldSpec::required("rendered_model_sql"), - FieldSpec::required("functions"), - FieldSpec::required("tests"), - FieldSpec::required("rendered_test_overrides"), - FieldSpec::required("sql_analysis_enabled"), - FieldSpec::required("sql_analysis_dialect"), - FieldSpec::required("set_difference_operator"), - FieldSpec::required("requires_derived_table_aliases"), - FieldSpec::required("lexical_syntax"), - FieldSpec::required("render_sql"), - FieldSpec::required("include_plan"), - ], -}; -const R_NATIVE_SQL_TEST_CHAIN_REQUEST: RecordSpec = RecordSpec { - name: "NativeSqlTestChainRequest", - doc: "Compiled models and the tests whose unmocked model chains to resolve.", - fields: &[ - FieldSpec::required("models"), - FieldSpec::required("tests"), - FieldSpec::required("lexical_syntax"), - ], -}; const R_NATIVE_SQL_TEST_FAILURE: RecordSpec = RecordSpec { name: "NativeSqlTestFailure", doc: "The error one test's assembly raises: its kind and message.", @@ -936,8 +909,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_MODEL_COLUMN_LINEAGE, R_REFACTOR_PROJECT, R_MANUAL_LOCATION, - R_NATIVE_SQL_TEST_PLANNING_REQUEST, - R_NATIVE_SQL_TEST_CHAIN_REQUEST, R_NATIVE_SQL_TEST_FAILURE, R_NATIVE_SQL_TEST_ASSEMBLY, R_HEADER_SPAN, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs index a45f185962..42aede93ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs @@ -139,7 +139,10 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - fn resource_scope(py: Python<'_>, options: Option>) -> PyResult { + pub(crate) fn resource_scope( + py: Python<'_>, + options: Option>, + ) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "resource_scope", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 0d27260653..5eeffdde1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -10,6 +10,7 @@ use sqlbuild_state::freshness::main::configuration_error::configuration_error; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::freshness::exception_types::SourceFreshnessObservationError; use crate::bindings::_helpers::freshness::observation::{ Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, unknown, @@ -19,10 +20,10 @@ use crate::bindings::_helpers::freshness::support::text; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; +use crate::bindings::_helpers::records::fields::field as native_field; const ADAPTER_STRATEGY: &str = "adapter"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; @@ -34,10 +35,7 @@ fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult(py) { return Ok(Some(value.str()?.into_any())); } Ok(None) @@ -56,9 +54,9 @@ fn previous_records<'py>( let py: Python<'py> = adapter.py(); let read: StateRead<'py> = StateRead { connection: connection.clone(), - execute: native_field(adapter, "execute")?, + execute: native_field(&adapter, "execute")?, render_qualified_name: arguments.value("render_qualified_name")?, - render_read_latest_sql: native_field(adapter, "render_read_latest_source_freshness_sql")?, + render_read_latest_sql: native_field(&adapter, "render_read_latest_source_freshness_sql")?, }; let database: Bound<'py, PyAny> = arguments.value("state_database")?; let exists_by_schema: Bound<'py, PyAny> = arguments.value("state_table_exists_by_schema")?; @@ -70,10 +68,7 @@ fn previous_records<'py>( .is_truthy()?; let previous: Bound<'py, PyAny> = read_latest(&read, (&database, &schema), exists, source_names)?; - for (identity, candidate) in native_field(&previous, "records")? - .downcast::()? - .iter() - { + for (identity, candidate) in native_field(&previous, "records")?.downcast::()?.iter() { let replace: bool = match merged.get_item(&identity)? { None => true, Some(current) => candidate @@ -127,7 +122,7 @@ fn adapter_batch<'py>( }; let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); for source in sources { - let name: Bound<'py, PyAny> = native_field(source, "name")?; + let name: Bound<'py, PyAny> = native_field(&source, "name")?; unknown_sources.set_item(&name, unknown(&name, ("ERROR", message.clone()))?)?; } call_with( @@ -176,7 +171,7 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( if arguments.value("filter_previous_to_sources")?.is_truthy()? { let mut names: Vec = sources .iter() - .map(|source| native_field(source, "name")?.extract()) + .map(|source| native_field(&source, "name")?.extract()) .collect::>()?; names.sort(); Some(names) @@ -192,10 +187,10 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( let observed_sources: Bound<'py, PyDict> = PyDict::new(py); let mut adapter_sources: Vec> = Vec::new(); for source in &sources { - if native_field(source, "managed")?.is_truthy()? { + if native_field(&source, "managed")?.is_truthy()? { continue; } - let name: Bound<'py, PyAny> = native_field(source, "name")?; + let name: Bound<'py, PyAny> = native_field(&source, "name")?; let Some(observed) = observation_source(&observer.adapter, source)? else { unknown_sources.set_item( &name, @@ -327,13 +322,13 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( scope: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = scope.py(); - let downstream_deps: Bound<'py, PyAny> = native_field(scope, "downstream_deps")?; + let downstream_deps: Bound<'py, PyAny> = native_field(&scope, "downstream_deps")?; let changed: Bound<'py, PyDict> = PyDict::new(py); let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); let errors: Bound<'py, PyDict> = PyDict::new(py); let stale: Bound<'py, PySet> = PySet::empty(py)?; let blocked: Bound<'py, PySet> = PySet::empty(py)?; - for identity in native_field(source_freshness, "changed_identities")?.try_iter()? { + for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { let identity: Bound<'py, PyAny> = identity?; let models: Bound<'py, PyFrozenSet> = downstream_models(&native_field(&identity, "source_name")?, &downstream_deps)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index 1d7d91d50b..ab2baf3104 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -8,12 +8,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; /// Observed records grouped by `(database, schema)` in first-seen order. type RecordsByLocation<'py> = Vec<( @@ -27,7 +28,7 @@ fn successful_model_names<'py>(result: &Bound<'py, PyAny>) -> PyResult = host::entry(py, ExecutorHost::ExecutionStatus)?.getattr("SUCCESS")?; let mut names: Vec> = Vec::new(); - for model_result in native_field(result, "model_results")?.try_iter()? { + for model_result in native_field(&result, "model_results")?.try_iter()? { let model_result: Bound<'py, PyAny> = model_result?; let status: Bound<'py, PyAny> = native_field(&model_result, "status")?; let value: Bound<'py, PyAny> = status.getattr_opt("value")?.unwrap_or(status); @@ -105,12 +106,12 @@ fn eligible_records<'py>( run_id: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = plan.py(); - let freshness: Bound<'py, PyAny> = native_field(plan, "source_freshness")?; + let freshness: Bound<'py, PyAny> = native_field(&plan, "source_freshness")?; if freshness.is_none() || native_field(&freshness, "propagation")?.is_none() { return Ok(Vec::new()); } let entries_by_name: Bound<'py, PyDict> = PyDict::new(py); - for entry in native_field(plan, "model_entries")?.try_iter()? { + for entry in native_field(&plan, "model_entries")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; entries_by_name.set_item(native_field(&entry, "name")?, entry)?; } @@ -158,23 +159,23 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { &[ ( "render_qualified_name", - native_field(adapter, "render_qualified_name")?, + native_field(&adapter, "render_qualified_name")?, ), ( "render_framework_type", - native_field(adapter, "render_framework_type")?, + native_field(&adapter, "render_framework_type")?, ), ( "render_insert_records_sql", - native_field(adapter, "render_insert_source_freshness_records_sql")?, + native_field(&adapter, "render_insert_source_freshness_records_sql")?, ), ( "render_create_table_sql", - native_field(adapter, "render_create_source_freshness_table_sql")?, + native_field(&adapter, "render_create_source_freshness_table_sql")?, ), ( "render_create_index_sqls", - native_field(adapter, "render_create_source_freshness_index_sqls")?, + native_field(&adapter, "render_create_source_freshness_index_sqls")?, ), ], ) @@ -187,11 +188,10 @@ fn write_records<'py>( grouped: &RecordsByLocation<'py>, ) -> PyResult<()> { let py: Python<'py> = adapter.py(); - let cost_context: Bound<'py, PyAny> = registry::native(py, NativeAttr::CostContext)?; for ((database, schema), records) in grouped { let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); for record in records { - let source_name: Bound<'py, PyAny> = native_field(record, "source_name")?; + let source_name: Bound<'py, PyAny> = native_field(&record, "source_name")?; let mut position: Option = None; for (index, (known, _)) in by_source.iter().enumerate() { if known.eq(&source_name)? { @@ -205,17 +205,24 @@ fn write_records<'py>( } } for (source_name, source_records) in by_source { - let scope: Bound<'py, PyAny> = call_with( - &native_field(&cost_context, "resource_scope")?, - &[ - ("resource_type", PyString::new(py, "source").into_any()), - ("resource_name", source_name), - ( - "phase", - PyString::new(py, "freshness_finalization").into_any(), - ), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + CostContext::resource_scope( + py, + Some(keywords( + py, + &[ + ("resource_type", PyString::new(py, "source").into_any()), + ("resource_name", source_name), + ( + "phase", + PyString::new(py, "freshness_finalization").into_any(), + ), + ], + )?), + )?, + )? + .into_any(); within(&scope, |_| { write_source_freshness_records( py, @@ -223,7 +230,7 @@ fn write_records<'py>( py, &[ ("connection", connection.clone()), - ("execute", native_field(adapter, "execute")?), + ("execute", native_field(&adapter, "execute")?), ("database", database.clone()), ("schema", schema.clone()), ("records", PyTuple::new(py, &source_records)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 2f7e75a44e..2450e4e5c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -6,23 +6,21 @@ use pyo3::types::{PyDict, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::freshness::exception_types::{ + SourceFreshnessInputError, SourceFreshnessObservationError, +}; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; -pub(crate) const OBSERVATION_ERROR: NativeAttr = NativeAttr::SourceFreshnessObservationError; -pub(crate) const INPUT_ERROR: NativeAttr = NativeAttr::SourceFreshnessInputError; +pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; +pub(crate) const INPUT_ERROR: fn(String) -> PyErr = SourceFreshnessInputError::new_err; /// The native error `class(message)` as a raisable error, with an optional `__cause__`. pub(crate) fn freshness_error( py: Python<'_>, - (class, message): (NativeAttr, &str), + (class, message): (fn(String) -> PyErr, &str), cause: Option, ) -> PyErr { - let error: PyErr = match registry::native(py, class).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => return error, - }; + let error: PyErr = class(message.to_owned()); if let Some(cause) = cause { error.set_cause(py, Some(cause)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index f505a54ff6..fe6cf37fa3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -8,6 +8,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::cli::progress_reporters::{ + ConnectionProgressReporter, PlanningProgressReporter, +}; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; @@ -16,9 +19,9 @@ use crate::bindings::_helpers::janitor::janitor_planning::{ }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = @@ -43,7 +46,7 @@ fn stdout(py: Python<'_>) -> PyResult> { } fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { - native_field(invocation, "use_color")?.extract() + native_field(&invocation, "use_color")?.extract() } /// `OperationLifecycle(operation_kind=..., operation_name=...)`. @@ -60,30 +63,23 @@ pub(crate) fn compile_janitor_project<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); let context: Bound<'py, PyAny> = resolve_adapter_connection_context( - &native_field(invocation, "discovered_inputs")?, - &native_field(invocation, "effective_project_dir")?, + &native_field(&invocation, "discovered_inputs")?, + &native_field(&invocation, "effective_project_dir")?, ( - native_field(invocation, "selected_target")?, + native_field(&invocation, "selected_target")?, py.None().into_bound(py), ), true, )?; let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PlanningProgressReporter)?, - &[ - ("stream", stdout(py)?), - ( - "use_color", - PyBool::new(py, use_color(invocation)?) - .to_owned() - .into_any(), - ), - ], - )?; - let as_target: Bound<'py, PyAny> = native_field(invocation, "as_target")?; + let status: Bound<'py, PyAny> = Bound::new( + py, + PlanningProgressReporter::new(py, &stdout(py)?, use_color(invocation)?)?, + )? + .into_any(); + let as_target: Bound<'py, PyAny> = native_field(&invocation, "as_target")?; let target: Bound<'py, PyAny> = if as_target.is_none() { - native_field(invocation, "selected_target")? + native_field(&invocation, "selected_target")? } else { as_target }; @@ -97,14 +93,11 @@ pub(crate) fn compile_janitor_project<'py>( &[ ( "discovered_inputs", - native_field(invocation, "discovered_inputs")?, + native_field(&invocation, "discovered_inputs")?, ), ("adapter", native_field(&context, "adapter")?), ("selected_target", target), - ( - "resolved_connection", - native_field(&context, "connection_config")?, - ), + ("resolved_connection", native_field(&context, "connection_config")?), ], )?), )?; @@ -139,19 +132,26 @@ pub(crate) fn connect_janitor_warehouse<'py>( compiled: &JanitorCompiled<'py>, ) -> PyResult> { let py: Python<'py> = invocation.py(); - let progress: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ConnectionProgressReporter)?, - &[ - ("adapter_name", compiled.adapter_name.clone()), - ("stream", stdout(py)?), - ( - "use_color", - PyBool::new(py, use_color(invocation)?) - .to_owned() - .into_any(), - ), - ], - )?; + let progress: Bound<'py, PyAny> = Bound::new( + py, + ConnectionProgressReporter::new( + py, + Some(keywords( + py, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?), + )?, + )? + .into_any(); let started: Instant = Instant::now(); progress.call_method1("on_connection_start", (1,))?; let connected: PyResult> = within( @@ -184,9 +184,7 @@ pub(crate) fn close_janitor_warehouse( } fn candidate_count(plan: &Bound<'_, PyAny>) -> PyResult { - let mut count: usize = native_field(plan, "old_name_views")? - .getattr("drops")? - .len()?; + let mut count: usize = native_field(&plan, "old_name_views")?.getattr("drops")?.len()?; for field in CANDIDATE_FIELDS { count += plan.getattr(field)?.len()?; } @@ -234,9 +232,8 @@ pub(crate) fn build_janitor_execution_plan<'py>( }, ), &JanitorSettings { - retention_days: native_field(settings, "retention_days")?.extract()?, - delete_tracked_only: native_field(&janitor, "delete_tracked_only")? - .is_truthy()?, + retention_days: native_field(&settings, "retention_days")?.extract()?, + delete_tracked_only: native_field(&janitor, "delete_tracked_only")?.is_truthy()?, exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, direct_mode: true, state_history_versions: settings diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index e301b367cd..d1606ba79f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -236,7 +236,7 @@ pub(crate) fn detect_model_changes<'py>( ) } -fn scope_model_changes<'py>( +pub(crate) fn scope_model_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), query_change_tracking: Option, (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs index 9c13e79504..08cbc3a1b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs @@ -17,7 +17,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_entries::plan_output::plan_selected_sql_tests; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; @@ -158,19 +159,13 @@ pub(crate) struct PlanningRequest { pub(crate) sql_test_selection: Py, } -/// `plan_selected_sql_tests(**request)`, looked up on the module so tests can replace it. +/// Plan the request's selected SQL tests. fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { - call_with( - ®istry::native(py, NativeAttr::PlanSelectedSqlTests)?, - &[ - ("project", request.project.bind(py).clone()), - ("adapter", request.adapter.bind(py).clone()), - ("selected_keys", request.selected_keys.bind(py).clone()), - ( - "sql_test_selection", - request.sql_test_selection.bind(py).clone(), - ), - ], + plan_selected_sql_tests( + request.project.bind(py), + request.adapter.bind(py), + request.selected_keys.bind(py), + request.sql_test_selection.bind(py), ) .map(Bound::unbind) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 2dfaca4bd6..6ba727a16c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -78,16 +78,29 @@ pub(crate) fn build_direct_model_version_identities<'py>( &["seeds", "source_version_hashes", "hook_functions"], ), )?; - let functions: Bound<'py, PyAny> = arguments.value("functions")?; - let scope: Bound<'py, PyAny> = arguments.value("scope")?; - let dialect: Option = arguments.value("dialect")?.extract()?; - let function_hashes: Bound<'py, PyDict> = - function_local_hashes(&functions, dialect.as_deref())?; - let hooks: Bound<'py, PyAny> = arguments.value("hook_functions")?; + direct_model_version_identities( + (&arguments.value("functions")?, &arguments.value("scope")?), + arguments.value("dialect")?.extract()?, + ( + &arguments.value("seeds")?, + &arguments.value("source_version_hashes")?, + &arguments.value("hook_functions")?, + ), + ) +} + +/// Current direct model identities of `(functions, scope)` with optional seeds, source hashes and hooks. +pub(crate) fn direct_model_version_identities<'py>( + (functions, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: Option, + (seeds, source_hashes, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = functions.py(); + let function_hashes: Bound<'py, PyDict> = function_local_hashes(functions, dialect.as_deref())?; let hook_hashes: Bound<'py, PyDict> = if hooks.is_none() { PyDict::new(py) } else { - hook_version_hashes(&hooks)? + hook_version_hashes(hooks)? }; let mut graph = IdentityGraph::default(); for function in functions.try_iter()? { @@ -106,7 +119,6 @@ pub(crate) fn build_direct_model_version_identities<'py>( } let seed_hashes: Bound<'py, PyDict> = PyDict::new(py); let seed_metadata: Bound<'py, PyDict> = PyDict::new(py); - let seeds: Bound<'py, PyAny> = arguments.value("seeds")?; if !seeds.is_none() { for seed in seeds.try_iter()? { let seed: Bound<'py, PyAny> = seed?; @@ -125,9 +137,8 @@ pub(crate) fn build_direct_model_version_identities<'py>( }, ); } - let source_hashes: Bound<'py, PyAny> = arguments.value("source_version_hashes")?; if source_hashes.is_truthy()? { - for (name, source_hash) in string_pairs(&source_hashes)? { + for (name, source_hash) in string_pairs(source_hashes)? { graph.nodes.insert( key_of(SOURCE_RESOURCE, &name), IdentityNode { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 79ffcd78a0..b8e3e1006d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -35,15 +35,27 @@ pub(crate) fn build_migration_fingerprint( let (key, value): (String, String) = item?.extract()?; identities.insert(key, value); } - let sql: String = arguments.value("query_sql")?.extract()?; - let metadata: String = arguments.value("metadata_json")?.extract()?; let dialect: Option = arguments.value("dialect")?.extract()?; - migration_fingerprint::migration_fingerprint( - &sql, - &metadata, + fingerprint( + &arguments.value("query_sql")?.extract::()?, + &arguments.value("metadata_json")?.extract::()?, &identities, + dialect.as_deref(), + ) +} + +/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. +pub(crate) fn fingerprint( + sql: &str, + metadata: &str, + identities: &BTreeMap, + dialect: Option<&str>, +) -> PyResult> { + migration_fingerprint::migration_fingerprint( + sql, + metadata, + identities, dialect - .as_deref() .filter(|value| !value.is_empty()) .unwrap_or("generic"), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 4cc53740ba..b1d045fa3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,33 +1,60 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; +use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::IntoPyObject; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; +use pyo3::sync::PyOnceLock; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods}; -use std::collections::HashSet; +use pyo3::{Py, PyRefMut, Python}; +use std::collections::{BTreeMap, HashSet}; +use crate::bindings::_helpers::records::fields::field as native_field; + +static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); + +fn identity_work(py: Python<'_>) -> PyResult> { + Ok(IDENTITY_WORK + .get_or_try_init(py, || Py::new(py, IdentityWork::default()))? + .bind(py) + .borrow_mut()) +} + +/// Report identity builds and per-model stale-warning change detections, optionally resetting them. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn planner_identity_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, BTreeMap)> { + let mut work = identity_work(py)?; + let info = (work.identity_builds, work.stale_detections.clone()); + if reset { + *work = IdentityWork::default(); + } + Ok(info) +} fn version_identities<'py>( project: &Bound<'py, PyAny>, scope: Bound<'py, PyAny>, ) -> PyResult> { - native( - project.py(), - NativeAttr::BuildDirectModelVersionIdentities, - &[ - ("functions", native_field(project, "functions")?), - ("seeds", native_field(project, "seeds")?), - ("scope", scope), - ("hook_functions", native_field(project, "hook_functions")?), - ("dialect", native_field(project, "sql_analysis_dialect")?), - ], + identity_work(project.py())?.identity_builds += 1; + direct_model_version_identities( + (&native_field(&project, "functions")?, &scope), + native_field(&project, "sql_analysis_dialect")?.extract()?, + ( + &native_field(&project, "seeds")?, + &project.py().None().into_bound(project.py()), + &native_field(&project, "hook_functions")?, + ), ) } @@ -75,7 +102,7 @@ fn reusable_model_names( let stale = class_field(identities, |i: &PlannerIdentityContext| { &i.stale_warning_identities })?; - for name in native_field(execution_changes, "models")?.try_iter()? { + for name in native_field(&execution_changes, "models")?.try_iter()? { let name = name?; let same = |field: &str| -> PyResult { execution @@ -153,32 +180,29 @@ pub(crate) fn detect_stale_warning_changes<'py>( let stale_identities = class_field(identities, |i: &PlannerIdentityContext| { &i.stale_warning_identities })?; - let remaining_changes = native( - py, - NativeAttr::DetectModelChangesInScope, - &[ - ("project", project.clone()), - ( - "scope", - replace( - &stale_scope, - &[("selected_keys", frozenset(py, &remaining)?)], - )?, - ), - ("snapshot", snapshot.clone()), - ( - "query_change_tracking", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "expected_version_hashes", - native_field(&stale_identities, "model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - native_field(&stale_identities, "model_metadata_jsons")?, - ), - ], + { + let mut work = identity_work(py)?; + for key in &remaining { + *work + .stale_detections + .entry(native_field(&key, "name")?.extract()?) + .or_default() += 1; + } + } + let remaining_changes = scope_model_changes( + ( + project, + &replace( + &stale_scope, + &[("selected_keys", frozenset(py, &remaining)?)], + )?, + snapshot, + ), + Some(true), + ( + native_field(&stale_identities, "model_version_hashes")?, + native_field(&stale_identities, "model_metadata_jsons")?, + ), )?; let models = PyDict::new(py); for (name, change) in execution_changes @@ -190,7 +214,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( models.set_item(name, change)?; } } - for (name, change) in remaining_changes.downcast::()?.iter() { + for (name, change) in remaining_changes.iter() { models.set_item(name, change)?; } native( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index baecdfe807..dcb5eee1d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -1,5 +1,7 @@ //! Plan output fields gathered across the plan phases before the output is built once. +use std::collections::BTreeMap; + use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; @@ -68,3 +70,11 @@ pub(crate) struct PlannerIdentityContext { pub(crate) version_identities: Py, pub(crate) stale_warning_identities: Py, } + +/// Planner identity work since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct IdentityWork { + pub(crate) identity_builds: u64, + pub(crate) stale_detections: BTreeMap, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 7004855328..5335281b81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -17,16 +17,16 @@ use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ native, replace, sorted_names, strings, text, tuple, }; -use crate::bindings::_helpers::planner_phases::retention::plan_retention; +use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::records::fields::field as native_field; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -112,16 +112,13 @@ pub(crate) fn base_plan_output<'py>( changes: reconciliation.changes.clone(), model_entries: entries.entries, model_warnings: entries.warnings, - reload_sources: native_field(overrides, "reload_sources")?, + reload_sources: native_field(&overrides, "reload_sources")?, extras, })?, metadata: Vec::new(), }; output.set("source_freshness", source_freshness.clone()); - output.set( - "cursor_overrides", - native_field(overrides, "cursor_overrides")?, - ); + output.set("cursor_overrides", native_field(&overrides, "cursor_overrides")?); Ok(output) } @@ -228,15 +225,7 @@ pub(crate) fn add_storage_policies<'py>( ], )?; let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let table_types = native( - py, - NativeAttr::PlanTableTypes, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scope", selected.clone()), - ], - )?; + let table_types = plan_table_types(runtime, warehouse, &selected)?; let retention = plan_retention(runtime, warehouse, &selected)?; output.set("model_entries", entries); output.set("migration_entries", migration_entries); @@ -266,7 +255,7 @@ pub(crate) fn add_plan_warnings<'py>( let warnings = concat(py, &[output.field("warnings")?, freshness])?; output.set("warnings", warnings); } - if !native_field(policies, "selection_diagnostics")?.is_truthy()? { + if !native_field(&policies, "selection_diagnostics")?.is_truthy()? { return Ok(output); } let stale = object( @@ -336,7 +325,7 @@ fn age_names<'py>(statuses: &Bound<'py, PyAny>, wanted: &str) -> PyResult(source_freshness: &Bound<'py, PyAny>) -> PyResult> { let py = source_freshness.py(); - let propagation = native_field(source_freshness, "propagation")?; + let propagation = native_field(&source_freshness, "propagation")?; let propagated = |field: &str| -> PyResult> { if propagation.is_none() { return tuple(py, Vec::new()); @@ -352,32 +341,29 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( let py = source_freshness.py(); let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; - diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; + diagnostics.set_item("enabled", native_field(&policies, "selection_diagnostics")?)?; output.metadata.push(( "direct_source_freshness", freshness_metadata(source_freshness)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 5092e097f9..5c7f3e8641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -16,6 +16,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::identities::build_planner_identity_context, module )?)?; + module.add_function(wrap_pyfunction!( + super::identities::planner_identity_work_info, + module + )?)?; module.add_function(wrap_pyfunction!( super::identities::detect_stale_warning_changes_of, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 15d4c1c727..f828abad66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; @@ -24,8 +25,8 @@ use crate::bindings::_helpers::planner_strategy::models::{ }; use crate::bindings::_helpers::planner_strategy::types::{FixtureKey, FixtureRelationInput}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; + +static FIXTURE_READS: PyOnceLock> = PyOnceLock::new(); /// Count one batch resolution of SQL-test model chains. pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { @@ -33,8 +34,12 @@ pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { Ok(()) } -fn work_counts(py: Python<'_>) -> PyResult> { - Ok(registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?) +/// The extension-wide fixture read cache and planning work counters. +pub(crate) fn work_counts(py: Python<'_>) -> PyResult> { + Ok(FIXTURE_READS + .get_or_try_init(py, || Py::new(py, FixtureReadCacheHost::default()))? + .bind(py) + .clone()) } /// The planning context of a project, over its models or the given name-to-model map. @@ -48,7 +53,7 @@ pub(crate) fn planning_context( Some(models) => models, None => { let models = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in native_field(&project, "models")?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, &model)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 0124530157..8935deb2a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -18,14 +18,11 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; -use crate::bindings::_helpers::planner_strategy::models::{ - FixtureReadCacheHost, RelationFixturePlanningContext, -}; +use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use crate::bindings::types::CompilerDetach; +use crate::bindings::_helpers::records::fields::field as native_field; /// The columns each fixture must supply for the chain's models, and the fixtures read through a star. pub(crate) fn required_fixture_columns( @@ -84,8 +81,7 @@ pub(crate) fn required_fixture_columns( stars: model_stars(&model, &schema, adapter)?, }); } - let owner = - registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; + let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); let cache = std::mem::take(&mut owner.cache); let (cache, result) = catch_compiler_panic(|| { @@ -175,10 +171,6 @@ fn model_stars( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add( - "_fixture_read_cache", - pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, - )?; module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) } @@ -186,8 +178,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfunction] #[pyo3(signature = (*, reset = false))] fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { - let owner = - registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; + let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); if reset { owner.cache = Default::default(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index f7e93f1c97..9ea14abd7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -4,22 +4,24 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; +use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{optional_text, text}; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; +use pyo3::PyRefMut; use pyo3::exceptions::{PyOSError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyListMethods, PyTuple}; -use pyo3::{pyclass, pymethods}; +use pyo3::{pyclass, pyfunction, pymethods}; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::hash::{DefaultHasher, Hash, Hasher}; +use crate::bindings::_helpers::records::fields::field as native_field; const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; @@ -31,6 +33,39 @@ const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_ type MemoKey = (String, String, Option, Vec<(String, String)>); +static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); + +fn fingerprint_work(py: Python<'_>) -> PyResult> { + Ok(FINGERPRINT_WORK + .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? + .bind(py) + .borrow_mut()) +} + +fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let mut work = fingerprint_work(py)?; + work.computations += 1; + work.distinct.insert(hasher.finish()); + Ok(()) +} + +/// Report migration fingerprints computed and distinct requests among them, optionally resetting. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn migration_fingerprint_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, usize)> { + let mut work = fingerprint_work(py)?; + let info = (work.computations, work.distinct.len()); + if reset { + *work = FingerprintWork::default(); + } + Ok(info) +} + /// Reuse migration fingerprints across rename maps, planning phases and invocations. #[pyclass(module = "sqlbuild._native")] pub(crate) struct MigrationFingerprintCache { @@ -313,20 +348,10 @@ impl MigrationFingerprintCache { let stored = match self.stored(py, &cache_key)? { Some(stored) => stored, None => { - let identities = PyDict::new(py); - for (name, identity) in &key.3 { - pyo3::types::PyDictMethods::set_item(&identities, name, identity)?; - } - let computed: Option = call_with( - ®istry::native(py, NativeAttr::BuildMigrationFingerprint)?, - &[ - ("query_sql", text(py, &key.0)), - ("metadata_json", text(py, &key.1)), - ("ref_identities", identities.into_any()), - ("dialect", optional_text(py, key.2.as_deref())), - ], - )? - .extract()?; + count_computation(py, &key)?; + let identities: BTreeMap = key.3.iter().cloned().collect(); + let computed: Option = + fingerprint(&key.0, &key.1, &identities, key.2.as_deref())?; self.remember(&cache_key, computed.as_deref())?; computed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs index b48621b451..69d9ef9fe4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod constants; pub(crate) mod cursors; pub(crate) mod fingerprint_cache; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod semantic_sources; pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs new file mode 100644 index 0000000000..6c19c1fcea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs @@ -0,0 +1,13 @@ +//! Warehouse planning state shared across planner runs. + +use std::collections::BTreeSet; + +use pyo3::pyclass; + +/// Migration fingerprints computed since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FingerprintWork { + pub(crate) computations: u64, + pub(crate) distinct: BTreeSet, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index 5d6419d550..939d91e174 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -5,6 +5,10 @@ use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_function(wrap_pyfunction!( + super::fingerprint_cache::migration_fingerprint_work_info, + module + )?)?; module.add_function(wrap_pyfunction!( super::snapshot::gather_warehouse_snapshot, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index afdc0ce45c..2ae4bc14a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -350,10 +350,8 @@ native_ids! { NativeRulesEvaluation => "NativeRulesEvaluation", NativeSqlTestArtifact => "NativeSqlTestArtifact", NativeSqlTestAssembly => "NativeSqlTestAssembly", - NativeSqlTestChainRequest => "NativeSqlTestChainRequest", NativeSqlTestFailure => "NativeSqlTestFailure", NativeSqlTestPlan => "NativeSqlTestPlan", - NativeSqlTestPlanningRequest => "NativeSqlTestPlanningRequest", NestedProgressChildRow => "NestedProgressChildRow", NormalizedDbtProfileConnection => "NormalizedDbtProfileConnection", OldNameView => "OldNameView", @@ -688,7 +686,6 @@ native_ids! { BuildDbtInitProjectToml => "build_dbt_init_project_toml", BuildDeclarationScope => "build_declaration_scope", BuildDeferredLocations => "build_deferred_locations", - BuildDirectModelVersionIdentities => "build_direct_model_version_identities", BuildDirectSemanticUses => "build_direct_semantic_uses", BuildDisplayOnlySqlbuildPlan => "build_display_only_sqlbuild_plan", BuildDownstreamDeps => "build_downstream_deps", @@ -710,7 +707,6 @@ native_ids! { BuildIntermediateSourceMap => "build_intermediate_source_map", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", - BuildMigrationFingerprint => "build_migration_fingerprint", BuildModelInputs => "build_model_inputs", BuildModelLocations => "build_model_locations", BuildModelVersionIdentityMetadataJson => "build_model_version_identity_metadata_json", @@ -805,7 +801,6 @@ native_ids! { DescribeFullRebuildCause => "describe_full_rebuild_cause", DetectChanges => "detect_changes", DetectModelChanges => "detect_model_changes", - DetectModelChangesInScope => "detect_model_changes_in_scope", DetectStaleWarningChanges => "detect_stale_warning_changes", DirectorySnapshotScope => "directory_snapshot_scope", DiscoverEventExporterFunctions => "discover_event_exporter_functions", @@ -855,7 +850,6 @@ native_ids! { FindPythonStringLocations => "find_python_string_locations", FinishRuleRun => "finish_rule_run", FixtureColumnInferences => "FixtureColumnInferences", - FixtureReadCache => "_fixture_read_cache", FormatAuditExecutionJson => "format_audit_execution_json", FormatBuildExecutionJson => "format_build_execution_json", FormatBuildFooter => "format_build_footer", @@ -957,7 +951,6 @@ native_ids! { ParseQueryShape => "parse_query_shape", PlanColumnMigrations => "plan_column_migrations", PlanCompiledSqlTestArtifacts => "plan_compiled_sql_test_artifacts", - PlanCompiledSqlTests => "plan_compiled_sql_tests", PlanDbtInteropCommand => "plan_dbt_interop_command", PlanDbtInteropFromProject => "plan_dbt_interop_from_project", PlanHasExecutableWork => "plan_has_executable_work", @@ -966,11 +959,9 @@ native_ids! { PlanRetention => "plan_retention", PlanRuleFixes => "plan_rule_fixes", PlanRuleRun => "plan_rule_run", - PlanSelectedSqlTests => "plan_selected_sql_tests", PlanSqlTests => "plan_sql_tests", PlanSqlTestsNatively => "plan_sql_tests_natively", PlanStaticSqlTests => "plan_static_sql_tests", - PlanTableTypes => "plan_table_types", PlanningProgressReporter => "PlanningProgressReporter", PrepareModelHeaderTokens => "prepare_model_header_tokens", PreparedCompileArtifacts => "PreparedCompileArtifacts", @@ -1014,7 +1005,6 @@ native_ids! { ResolveBoundedCursorOverride => "resolve_bounded_cursor_override", ResolveCloneBoundary => "resolve_clone_boundary", ResolveCompileAnalysisSelection => "resolve_compile_analysis_selection", - ResolveCompiledSqlTestChains => "resolve_compiled_sql_test_chains", ResolveConfigFilePath => "resolve_config_file_path", ResolveConfiguredRuleSelection => "resolve_configured_rule_selection", ResolveCostRun => "resolve_cost_run", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs index abb88195cf..c0cabbca1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -115,7 +115,7 @@ fn write(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { impl ExecutionEventWriter { #[new] #[pyo3(signature = (*, path = None))] - fn new(py: Python<'_>, path: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, path: Option>) -> PyResult { let configured: Bound<'_, PyAny> = py .import("os")? .getattr("environ")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index a7b8394942..6e30a5a7cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -6,14 +6,16 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use crate::bindings::_helpers::sql_test_glue::planning::resolve_compiled_sql_test_chains; +use crate::bindings::_helpers::sql_test_glue::planning::{ + ChainRequest, PlanningRequest, plan_compiled_sql_tests, resolve_compiled_sql_test_chains, +}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; -use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::types::{PyDict, PyTuple}; +use crate::bindings::_helpers::records::fields::field as native_field; pub(super) fn chains<'py>( project: &Bound<'py, PyAny>, @@ -23,23 +25,16 @@ pub(super) fn chains<'py>( if !tests.is_truthy()? { return Ok(PyTuple::empty(py).into_any()); } - let request = call_with( - ®istry::class(py, RecordId::NativeSqlTestChainRequest)?, - &[ - ("models", native_field(project, "models")?), - ("tests", tests.clone()), - ( - "lexical_syntax", - project - .getattr("sql_lexical_syntax")? - .getattr("native_mapping")?, - ), - ], - )?; - let native = request - .extract() - .and_then(|request| resolve_compiled_sql_test_chains(py, request)) - .and_then(|chains| object(py, chains)); + let request = ChainRequest { + models: native_field(&project, "models")?.extract()?, + tests: tests.extract()?, + lexical_syntax: project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")? + .extract()?, + }; + let native = + resolve_compiled_sql_test_chains(py, request).and_then(|chains| object(py, chains)); let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { @@ -80,57 +75,34 @@ pub(crate) fn planned<'py>( let model_sql = super::plan_rendering::models(project, adapter)?; let functions = super::plan_rendering::functions(project, adapter)?; let overrides = super::plan_rendering::overrides(project, tests, adapter)?; - let request = call_with( - ®istry::class(py, RecordId::NativeSqlTestPlanningRequest)?, - &[ - ("models", native_field(project, "models")?), - ("rendered_model_sql", model_sql.into_any()), - ("functions", functions.into_any()), - ("tests", tests.clone()), - ("rendered_test_overrides", overrides.into_any()), - ("sql_analysis_enabled", options.sql_analysis_enabled.clone()), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "requires_derived_table_aliases", - adapter.call_method0("requires_derived_table_aliases")?, - ), - ( - "lexical_syntax", - project - .getattr("sql_lexical_syntax")? - .getattr("native_mapping")?, - ), - ( - "render_sql", - PyBool::new(py, artifact || options.render_sql) - .to_owned() - .into_any(), - ), - ( - "include_plan", - PyBool::new(py, !artifact && options.include_plan) - .to_owned() - .into_any(), - ), - ], - )?; - let native = registry::native(py, NativeAttr::PlanCompiledSqlTests)?.call1((request,)); - let native = match native { + let request = PlanningRequest { + models: native_field(&project, "models")?.extract()?, + rendered_model_sql: model_sql.extract()?, + functions: functions.extract()?, + tests: tests.extract()?, + rendered_test_overrides: overrides.extract()?, + sql_analysis_enabled: options.sql_analysis_enabled.extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + requires_derived_table_aliases: adapter + .call_method0("requires_derived_table_aliases")? + .extract()?, + lexical_syntax: project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")? + .extract()?, + render_sql: artifact || options.render_sql, + include_plan: !artifact && options.include_plan, + }; + let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { return Err(super::plan_errors::planning_error(py, error, tests)?); } Err(error) => return Err(error), }; - let planning_ns: u64 = native.get_item(1)?.extract()?; - let rendering_ns: u64 = native.get_item(2)?.extract()?; let end: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; @@ -148,8 +120,8 @@ pub(crate) fn planned<'py>( add_compile_timing(phase, elapsed); } let mut plans: Vec> = Vec::new(); - for row in native.get_item(0)?.try_iter()? { - let plan = super::plan_rows::plan(&row?)?; + for row in rows { + let plan = super::plan_rows::plan(py, row)?; if artifact { let sql = native_field(&plan, "sql")?; if sql.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 9976e742c1..a4791201f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -3,70 +3,68 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::sql_test_glue::planning::{AssertionRow, ChainRow, PlanRow}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; -fn step<'py>(row: &Bound<'py, PyAny>, chain: bool) -> PyResult> { - let py = row.py(); - let kwargs = PyDict::new(py); - let fields: &[&str] = if chain { +fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult> { + let (model, sql, expected_sql, lifted, body, expected_columns, expected_lifted) = row; + call_with( + ®istry::class(py, RecordId::ChainStep)?, &[ - "model_name", - "resolved_sql", - "expected_cte_sql", - "lifted_ctes", - "comparison_body_sql", - "expected_columns", - "expected_lifted_ctes", - ] - } else { - &["name", "resolved_sql", "lifted_ctes", "comparison_body_sql"] - }; - for (index, name) in fields.iter().enumerate() { - let value = row.get_item(index)?; - kwargs.set_item( - *name, - if ["lifted_ctes", "expected_columns", "expected_lifted_ctes"].contains(name) - && !value.is_none() - { - tuple(&value)? - } else { - value - }, - )?; - } - registry::class( - py, - if chain { - RecordId::ChainStep - } else { - RecordId::SqlTestAssertionStep - }, - )? - .call((), Some(&kwargs)) + ("model_name", model.into_bound_py_any(py)?), + ("resolved_sql", sql.into_bound_py_any(py)?), + ("expected_cte_sql", expected_sql.into_bound_py_any(py)?), + ("lifted_ctes", PyTuple::new(py, lifted)?.into_any()), + ("comparison_body_sql", body.into_bound_py_any(py)?), + ( + "expected_columns", + match expected_columns { + Some(columns) => PyTuple::new(py, columns)?.into_any(), + None => py.None().into_bound(py), + }, + ), + ( + "expected_lifted_ctes", + PyTuple::new(py, expected_lifted)?.into_any(), + ), + ], + ) +} + +fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult> { + let (name, sql, lifted, body) = row; + call_with( + ®istry::class(py, RecordId::SqlTestAssertionStep)?, + &[ + ("name", name.into_bound_py_any(py)?), + ("resolved_sql", sql.into_bound_py_any(py)?), + ("lifted_ctes", PyTuple::new(py, lifted)?.into_any()), + ("comparison_body_sql", body.into_bound_py_any(py)?), + ], + ) } -pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - let py = row.py(); - let mut chain: Vec> = Vec::new(); - for value in row.get_item(1)?.try_iter()? { - chain.push(step(&value?, true)?); +pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { + let (sql, chain_rows, assertion_rows, model_names, warning_rows, error_messages) = row; + let mut chain: Vec> = Vec::new(); + for value in chain_rows { + chain.push(chain_step(py, value)?); } - let mut assertions: Vec> = Vec::new(); - for value in row.get_item(2)?.try_iter()? { - assertions.push(step(&value?, false)?); + let mut assertions: Vec> = Vec::new(); + for value in assertion_rows { + assertions.push(assertion_step(py, value)?); } - let mut warnings: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); let severity = registry::class(py, RecordId::WarningSeverity)?; - for value in row.get_item(4)?.try_iter()? { - let value = value?; + for (model_name, level, message) in warning_rows { warnings.push(call_with( ®istry::class(py, RecordId::PlanWarning)?, &[ - ("model_name", value.get_item(0)?), - ("severity", severity.call1((value.get_item(1)?,))?), - ("message", value.get_item(2)?), + ("model_name", model_name.into_bound_py_any(py)?), + ("severity", severity.call1((level,))?), + ("message", message.into_bound_py_any(py)?), ], )?); } @@ -75,10 +73,13 @@ pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> &[ ("chain", PyTuple::new(py, chain)?.into_any()), ("assertions", PyTuple::new(py, assertions)?.into_any()), - ("model_names", tuple(&row.get_item(3)?)?), + ("model_names", PyTuple::new(py, model_names)?.into_any()), ("warnings", PyTuple::new(py, warnings)?.into_any()), - ("error_messages", tuple(&row.get_item(5)?)?), - ("sql", row.get_item(0)?), + ( + "error_messages", + PyTuple::new(py, error_messages)?.into_any(), + ), + ("sql", sql.into_bound_py_any(py)?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index 39a41e690e..7dd187e01d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -2,8 +2,7 @@ use std::collections::{BTreeMap, HashMap}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; @@ -22,9 +21,9 @@ use crate::bindings::types::CompilerDetach; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -type CtePair = (String, String); +pub(crate) type CtePair = (String, String); /// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. -type ChainRow = ( +pub(crate) type ChainRow = ( String, String, Option, @@ -34,11 +33,11 @@ type ChainRow = ( Vec, ); /// `(name, resolved SQL, lifted CTEs, comparison body)`. -type AssertionRow = (String, String, Vec, Option); +pub(crate) type AssertionRow = (String, String, Vec, Option); /// `(model, severity, message)`. -type WarningRow = (Option, &'static str, String); +pub(crate) type WarningRow = (Option, &'static str, String); /// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. -type PlanRow = ( +pub(crate) type PlanRow = ( Option, Vec, Vec, @@ -47,50 +46,32 @@ type PlanRow = ( Vec, ); -/// The Python planning request: compiled models and tests, with the adapter's facts. -#[derive(FromPyObject)] +/// The planning request: compiled models and tests, with the adapter's facts. pub(crate) struct PlanningRequest<'py> { - #[pyo3(attribute)] - models: Vec>, + pub(crate) models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. - #[pyo3(attribute)] - rendered_model_sql: HashMap, - #[pyo3(attribute)] - functions: Vec<(String, String, String, String, String)>, - #[pyo3(attribute)] - tests: Vec>, + pub(crate) rendered_model_sql: HashMap, + pub(crate) functions: Vec<(String, String, String, String, String)>, + pub(crate) tests: Vec>, /// Cursor-rendered model query overrides by test index, replacing the test's own. - #[pyo3(attribute)] - rendered_test_overrides: HashMap>, - #[pyo3(attribute)] - sql_analysis_enabled: bool, - #[pyo3(attribute)] - sql_analysis_dialect: Option, - #[pyo3(attribute)] - set_difference_operator: String, - #[pyo3(attribute)] - requires_derived_table_aliases: bool, - #[pyo3(attribute)] - lexical_syntax: LexicalSyntaxInput, - #[pyo3(attribute)] - render_sql: bool, - #[pyo3(attribute)] - include_plan: bool, + pub(crate) rendered_test_overrides: HashMap>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) sql_analysis_dialect: Option, + pub(crate) set_difference_operator: String, + pub(crate) requires_derived_table_aliases: bool, + pub(crate) lexical_syntax: LexicalSyntaxInput, + pub(crate) render_sql: bool, + pub(crate) include_plan: bool, } -/// The Python chain request: compiled models and the tests whose chains to resolve. -#[derive(FromPyObject)] +/// The chain request: compiled models and the tests whose chains to resolve. pub(crate) struct ChainRequest<'py> { - #[pyo3(attribute)] - models: Vec>, - #[pyo3(attribute)] - tests: Vec>, - #[pyo3(attribute)] - lexical_syntax: LexicalSyntaxInput, + pub(crate) models: Vec>, + pub(crate) tests: Vec>, + pub(crate) lexical_syntax: LexicalSyntaxInput, } /// Each test's ordered unmocked model chain; direct-logic tests have none. -#[pyfunction] pub(crate) fn resolve_compiled_sql_test_chains( py: Python<'_>, request: ChainRequest<'_>, @@ -118,7 +99,6 @@ pub(crate) fn resolve_compiled_sql_test_chains( } /// Plan SQL tests in one batch and return each plan with its distinct error messages. -#[pyfunction] pub(crate) fn plan_compiled_sql_tests( py: Python<'_>, request: PlanningRequest<'_>, @@ -298,9 +278,3 @@ fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { step.comparison_body_sql, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_compiled_sql_tests, module)?)?; - module.add_function(wrap_pyfunction!(resolve_compiled_sql_test_chains, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 2e414fa4ec..9b9f6f8a0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, planning}; +use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -23,6 +23,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; authored_ctes::register(module)?; - planning::register(module)?; compiled_tests::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a5be1ba378..ed0833d006 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -311,31 +311,6 @@ type ProofRow = tuple[ type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] -type CtePairRow = tuple[str, str] - -type NativeChainStepRow = tuple[ - str, - str, - str | None, - list[CtePairRow], - str | None, - list[str] | None, - list[CtePairRow], -] - -type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] - -type NativePlanWarningRow = tuple[str | None, str, str] - -type NativeSqlTestPlanRow = tuple[ - str | None, - list[NativeChainStepRow], - list[NativeAssertionStepRow], - list[str], - list[NativePlanWarningRow], - list[str], -] - type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] type ProjectProvider = Provider @@ -3931,6 +3906,8 @@ def simplify_fixture_null_projections(*, sql: str, relation: Any) -> str: ... def is_empty_fixture_query(sql: str) -> bool: ... def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... def fixture_planning_work_info(*, reset: bool = False) -> tuple[int, int]: ... +def planner_identity_work_info(*, reset: bool = False) -> tuple[int, dict[str, int]]: ... +def migration_fingerprint_work_info(*, reset: bool = False) -> tuple[int, int]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... @@ -6020,10 +5997,6 @@ def render_janitor_completion( ) -> str: ... # Native analysis: SQL test planning glue. -def plan_compiled_sql_tests( - request: NativeSqlTestPlanningRequest, / -) -> tuple[list[NativeSqlTestPlanRow], int, int]: ... -def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... def assemble_native_sql_tests( *, inputs: CompileInputsView ) -> tuple[NativeSqlTestAssembly, ...]: ... @@ -10242,50 +10215,6 @@ class ManualLocation(NativeRecord): reason: str, ) -> None: ... -class NativeSqlTestPlanningRequest(NativeRecord): - """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" - - models: tuple[CompiledModelView, ...] - rendered_model_sql: dict[int, str] - functions: tuple[tuple[str, str, str, str, str], ...] - tests: tuple[CompiledSqlTest, ...] - rendered_test_overrides: dict[int, dict[str, str]] - sql_analysis_enabled: bool - sql_analysis_dialect: str | None - set_difference_operator: str - requires_derived_table_aliases: bool - lexical_syntax: dict[str, object] - render_sql: bool - include_plan: bool - def __init__( - self, - models: tuple[CompiledModelView, ...], - rendered_model_sql: dict[int, str], - functions: tuple[tuple[str, str, str, str, str], ...], - tests: tuple[CompiledSqlTest, ...], - rendered_test_overrides: dict[int, dict[str, str]], - sql_analysis_enabled: bool, - sql_analysis_dialect: str | None, - set_difference_operator: str, - requires_derived_table_aliases: bool, - lexical_syntax: dict[str, object], - render_sql: bool, - include_plan: bool, - ) -> None: ... - -class NativeSqlTestChainRequest(NativeRecord): - """Compiled models and the tests whose unmocked model chains to resolve.""" - - models: tuple[CompiledModelView, ...] - tests: tuple[CompiledSqlTest, ...] - lexical_syntax: dict[str, object] - def __init__( - self, - models: tuple[CompiledModelView, ...], - tests: tuple[CompiledSqlTest, ...], - lexical_syntax: dict[str, object], - ) -> None: ... - class NativeSqlTestFailure(NativeRecord): """The error one test's assembly raises: its kind and message.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index f8dc62f435..4feed479e4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -3,7 +3,7 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import ANY, Mock +from unittest.mock import ANY import pytest @@ -49,8 +49,6 @@ def test_given_project_when_compiling_then_native_seams_answer( ) -> None: prepared_project: Path = tmp_path / "orders" prepare_compile_reuse_project(project_dir=prepared_project) - native_sql_test_planning: Mock = Mock(wraps=native_module.plan_compiled_sql_tests) - monkeypatch.setattr(native_module, "plan_compiled_sql_tests", native_sql_test_planning) native_run, native_seams = seam_recording_compile( project_dir=prepared_project, monkeypatch=monkeypatch, capsys=capsys ) @@ -58,7 +56,6 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_run.returncode == 0, native_run.stderr assert native_run.compiled assert native_seams == test_case.expected_native_returns - planned_counts: list[int] = [] for seams in (native_seams,): assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) @@ -70,14 +67,10 @@ def test_given_project_when_compiling_then_native_seams_answer( assert isinstance(planned_artifacts, tuple) assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) - planned_counts.append(len(planned_artifacts)) completed: object = seams["complete_semantic_project"][0] assert isinstance(completed, native_module.NativeProjectView) assert any(model.inferred_columns for model in completed.models) assert isinstance(seams["evaluate_model_contracts"][0], ContractValidationResult) - assert [ - len(call.args[0].tests) for call in native_sql_test_planning.call_args_list - ] == planned_counts if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 2090faf3ca..6828de306d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from collections.abc import Callable, Mapping +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from textwrap import dedent @@ -13,8 +13,7 @@ import pytest from _pytest.capture import CaptureResult -import sqlbuild._native as fingerprint_cache_module -from sqlbuild._native import FINGERPRINT_TABLE_NAME +from sqlbuild._native import FINGERPRINT_TABLE_NAME, migration_fingerprint_work_info from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer @@ -750,34 +749,16 @@ def state_tables(*, project_dir: Path, schema: str) -> tuple[str, ...]: ) -type FingerprintComputation = tuple[str, str, tuple[tuple[str, str], ...], str | None] +def fingerprint_work() -> tuple[int, int]: + """Return migration fingerprints computed and distinct requests since the last reset.""" + return migration_fingerprint_work_info() -def record_fingerprint_computations( - monkeypatch: pytest.MonkeyPatch, -) -> list[FingerprintComputation]: - """Record every migration fingerprint the planner actually computes.""" - computed: list[FingerprintComputation] = [] - original: Callable[..., str | None] = fingerprint_cache_module.build_migration_fingerprint +def reset_fingerprint_work() -> None: + """Forget migration fingerprint work recorded so far in this process.""" - def recording( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computed.append((query_sql, metadata_json, tuple(sorted(ref_identities.items())), dialect)) - return original( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(fingerprint_cache_module, "build_migration_fingerprint", recording) - return computed + migration_fingerprint_work_info(reset=True) def models_with_stored_migration_fingerprints(project_dir: Path) -> tuple[str, ...]: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py index 5053547d82..0e1f4b1ed9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py @@ -11,15 +11,15 @@ ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, - FingerprintComputation, build_each, + fingerprint_work, load_raw_orders, models_with_stored_migration_fingerprints, original_order_models, plan_json, planned_migrations, - record_fingerprint_computations, renamed_order_models, + reset_fingerprint_work, run_sqb, write_project, ) @@ -58,17 +58,17 @@ def test_given_project_state_when_planning_then_fingerprints_only_migration_cand test_case: MigrationFingerprintWorkTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) build_each(project_dir=tmp_path, model_sets=test_case.prior_models, capsys=capsys) write_project(project_dir=tmp_path, models=test_case.models()) - computed: list[FingerprintComputation] = record_fingerprint_computations(monkeypatch) + reset_fingerprint_work() migrations: int = len(planned_migrations(plan_json(project_dir=tmp_path, capsys=capsys))) assert migrations == test_case.expected_migrations - assert len(computed) == len(set(computed)) == test_case.expected_computations + computations, distinct = fingerprint_work() + assert computations == distinct == test_case.expected_computations @pytest.mark.parametrize( @@ -117,19 +117,19 @@ def test_given_project_state_when_building_then_stores_fingerprints_computed_onc test_case: MigrationFingerprintWorkTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) build_each(project_dir=tmp_path, model_sets=test_case.prior_models, capsys=capsys) write_project(project_dir=tmp_path, models=test_case.models()) migrations: int = len(planned_migrations(plan_json(project_dir=tmp_path, capsys=capsys))) - computed: list[FingerprintComputation] = record_fingerprint_computations(monkeypatch) + reset_fingerprint_work() result: CliRun = run_sqb(project_dir=tmp_path, args=test_case.command, capsys=capsys) assert result.exit_code == 0, result.output assert migrations == test_case.expected_migrations - assert len(computed) == len(set(computed)) == test_case.expected_computations + computations, distinct = fingerprint_work() + assert computations == distinct == test_case.expected_computations assert models_with_stored_migration_fingerprints(tmp_path) == (test_case.expected_stored_models) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index b005a94aba..addaa79155 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -1,13 +1,10 @@ from __future__ import annotations -from collections import Counter from dataclasses import replace from typing import Any -from unittest.mock import Mock import pytest -import sqlbuild._native as identity_planning from sqlbuild._native import ( BackfillAction, CursorBounds, @@ -29,6 +26,7 @@ WarningSeverity, compiled_model_record, format_plan, + planner_identity_work_info, project_with, ) from sqlbuild.adapter.contract.types import BuiltinAdapter @@ -84,7 +82,6 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d test_case: PlannerChangeDetectionWorkTestCase, adapter: DuckDbAdapter, connection: Any, - monkeypatch: pytest.MonkeyPatch, ) -> None: connection.execute("CREATE TABLE staging.stg_orders AS SELECT 1 AS order_id") project: CompiledProjectView = build_project_from_test_case( @@ -114,14 +111,7 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - stale_detector: Mock = Mock(wraps=identity_planning.detect_model_changes_in_scope) - monkeypatch.setattr(identity_planning, "detect_model_changes_in_scope", stale_detector) - identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) - monkeypatch.setattr( - identity_planning, - "build_direct_model_version_identities", - identity_builder, - ) + planner_identity_work_info(reset=True) plan: PlanOutput = build_execution_plan_from_kwargs( project=project, @@ -131,11 +121,9 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d selection_diagnostics=test_case.selection_diagnostics, ) - stale_calls: Counter[str] = Counter() - for call in stale_detector.call_args_list: - stale_calls.update(key.name for key in call.kwargs["scope"].selected_keys) - assert dict(stale_calls) == test_case.expected_stale_detection_calls - assert identity_builder.call_count == test_case.expected_identity_builds + identity_builds, stale_detections = planner_identity_work_info() + assert stale_detections == test_case.expected_stale_detection_calls + assert identity_builds == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 3f84e5983c..10c2bcbc94 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -4,11 +4,13 @@ import json from collections.abc import Callable +from dataclasses import replace from pathlib import Path from textwrap import dedent from typing import Any import duckdb +import pytest from sqlbuild._native import ( CompilePipelineOptions, @@ -23,6 +25,7 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildExecutionResult, @@ -474,20 +477,19 @@ def verify_warehouse_state( assert cursor.fetchone() is None, f"Relation {relation} should not exist but was found" -def drift_for_models( - *, model_names: tuple[str, ...] -) -> Callable[..., tuple[TableTypePlanEntry, ...]]: - """Return a planner stand-in reporting permanent-desired drift for the named models.""" +def drift_for_models(*, model_names: tuple[str, ...], monkeypatch: pytest.MonkeyPatch) -> None: + """Hand the build scheduler a plan reporting permanent-desired drift for the named models.""" - def plan_table_types( - *, runtime: Any, warehouse: Any, scope: Any - ) -> tuple[TableTypePlanEntry, ...]: - del warehouse, scope - models_by_name: dict[str, Any] = {model.name: model for model in runtime.project.models} - return tuple( + original: Callable[..., None] = BuildScheduler.__init__ + + def drifted_init(self: BuildScheduler, *, plan: PlanOutput, **kwargs: Any) -> None: + destinations: dict[str, Any] = { + entry.name: entry.destination for entry in plan.model_entries + } + entries: tuple[TableTypePlanEntry, ...] = tuple( TableTypePlanEntry( model_name=name, - destination=models_by_name[name].destination, + destination=destinations[name], copy_name=f"__sqb_type_swap__{name}", desired_type="permanent", actual_type="transient", @@ -497,8 +499,9 @@ def plan_table_types( ) for name in model_names ) + original(self, plan=replace(plan, table_type_entries=entries), **kwargs) - return plan_table_types + monkeypatch.setattr(BuildScheduler, "__init__", drifted_init) def recording_table_type_conversion(*, conversions: list[tuple[str, int]]) -> Callable[..., None]: diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index 08999a5dab..15a8a68b07 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -8,7 +8,6 @@ import duckdb import pytest -import sqlbuild._native as planner_assembly_module import sqlbuild.executor.build.classes.build_scheduler as build_scheduler_module from tests.integration.src.sqlbuild.executor.build._test_types import TableTypeRoutingTestCase from tests.integration.src.sqlbuild.executor.build.helpers import ( @@ -77,11 +76,7 @@ def test_given_table_type_drift_when_building_then_only_carried_over_tables_conv with duckdb.connect(str(tmp_path / "demo.duckdb")) as setup: setup.execute(_NEW_ORDER_SQL) conversions: list[tuple[str, int]] = [] - monkeypatch.setattr( - planner_assembly_module, - "plan_table_types", - drift_for_models(model_names=("customer_totals", "orders")), - ) + drift_for_models(model_names=("customer_totals", "orders"), monkeypatch=monkeypatch) monkeypatch.setattr( build_scheduler_module, "apply_table_type_conversion", @@ -125,11 +120,7 @@ def test_given_failing_table_type_conversion_when_building_then_model_fails_with with duckdb.connect(str(tmp_path / "demo.duckdb")) as setup: setup.execute(_NEW_ORDER_SQL) conversions: list[tuple[str, int]] = [] - monkeypatch.setattr( - planner_assembly_module, - "plan_table_types", - drift_for_models(model_names=("customer_totals", "orders")), - ) + drift_for_models(model_names=("customer_totals", "orders"), monkeypatch=monkeypatch) monkeypatch.setattr( build_scheduler_module, "apply_table_type_conversion", diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py index 1c39fe8ed9..db6c3178e5 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py @@ -35,7 +35,9 @@ class PlanWorkTestCase: @dataclass(frozen=True) class PipelinePhaseTimingTestCase: description: str - clock_values: tuple[float, ...] + compile_seconds: float + connection_seconds: float + rules_seconds: float expected_compile_seconds: float expected_planning_seconds: float expected_total_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 4437f1965b..cb01e9b97b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -17,6 +17,7 @@ SourceEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType @@ -87,3 +88,30 @@ def build_plan_output_with_model(name: str = "orders") -> PlanOutput: model_entries=(entry,), selected_keys=frozenset((entry.key,)), ) + + +class ManualClock: + """A monotonic clock that only moves when a test advances it.""" + + def __init__(self) -> None: + self.now: float = 0.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class SlowConnectionDuckDbAdapter(DuckDbAdapter): + """An in-memory DuckDB adapter whose connections take `connection_seconds` on a manual clock.""" + + def __init__(self, *, clock: ManualClock, connection_seconds: float) -> None: + super().__init__() + self._clock: ManualClock = clock + self._connection_seconds: float = connection_seconds + + def connect(self, config: dict[str, Any]) -> Any: + del config + self._clock.advance(self._connection_seconds) + return super().connect({}) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 0a41ca55d2..3bc86e35aa 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -15,13 +15,17 @@ CompilePipelineOptions, CompilePipelineResult, PartialBuildPhaseTimings, - PlanOutput, RulesRunResult, + discover_project_inputs, ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase +from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import ( + ManualClock, + SlowConnectionDuckDbAdapter, +) @pytest.mark.parametrize( @@ -29,14 +33,18 @@ [ PipelinePhaseTimingTestCase( description="slow planning connection is excluded from direct compilation", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), + compile_seconds=2.0, + connection_seconds=10.0, + rules_seconds=0.0, expected_compile_seconds=2.0, expected_planning_seconds=10.0, - expected_total_seconds=13.0, + expected_total_seconds=12.0, ), PipelinePhaseTimingTestCase( description="rules wall time is reported and excluded from compile and planning", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 3.0, 13.0, 13.0), + compile_seconds=2.0, + connection_seconds=10.0, + rules_seconds=1.0, expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, @@ -54,42 +62,33 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - monotonic: Mock = Mock(side_effect=test_case.clock_values) + clock: ManualClock = ManualClock() project: CompiledProjectView = CompiledProject( run_id="timing-run", effective_target_name="test", effective_connection={}, effective_vars={}, ) - pipeline_result: CompilePipelineResult = CompilePipelineResult( - project=project, plan_output=PlanOutput() - ) - adapter: Mock = Mock() - monkeypatch.setattr(time, "monotonic", monotonic) - monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr(_native, "build_execution_plan", Mock(return_value=PlanOutput())) + + def compiled(**kwargs: object) -> CompiledProjectView: + del kwargs + clock.advance(test_case.compile_seconds) + return project + + def evaluated(*args: object, **kwargs: object) -> RulesRunResult: + del args, kwargs + clock.advance(test_case.rules_seconds) + return RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) + + monkeypatch.setattr(time, "monotonic", clock) + monkeypatch.setattr(_native, "build_compiled_project", compiled) monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) - monkeypatch.setattr( - _native, - "build_python_run_plan_outputs", - Mock( - return_value=Mock( - plan_output=pipeline_result.plan_output, - selected_python_node_names=frozenset(), - python_plan_entries=(), - ) - ), + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "timing"\nadapter = "duckdb"\n', encoding="utf-8" ) - (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) - monkeypatch.setattr( - _native, - "evaluate_rules", - Mock( - return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) - ), - ) - tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=monotonic) + monkeypatch.setattr(_native, "evaluate_rules", evaluated) + tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=clock) progress: list[str] = [] progress_hooks: dict[bool, ConnectionHooks] = { True: ConnectionHooks(on_progress=progress.append), @@ -98,8 +97,10 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di with tracker.scope(): result: CompilePipelineResult = _native.run_compile_pipeline( - discovered_inputs=Mock(project_dir=tmp_path), - adapter=adapter, + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=SlowConnectionDuckDbAdapter( + clock=clock, connection_seconds=test_case.connection_seconds + ), options=CompilePipelineOptions(connection_config={}), hooks=progress_hooks[test_case.report_progress], ) @@ -108,7 +109,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di assert result.compile_seconds == test_case.expected_compile_seconds assert result.planning_seconds == test_case.expected_planning_seconds assert timings.total_seconds == test_case.expected_total_seconds - assert (result.compile_seconds or 0) + (result.planning_seconds or 0) < ( + assert (result.compile_seconds or 0) + (result.planning_seconds or 0) <= ( timings.total_seconds or 0 ) assert set(test_case.expected_progress_fragments) <= set(progress) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index ca26c95c10..a504096ade 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -5,7 +5,7 @@ import importlib.metadata import json import sqlite3 -from collections.abc import Mapping +from collections.abc import Callable from pathlib import Path import pytest @@ -13,7 +13,6 @@ import sqlbuild._native as native_module from sqlbuild._native import ( FixtureColumnInference, - build_migration_fingerprint, infer_fixture_column_facts, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -31,30 +30,6 @@ DATABASE_GLOB: str = "migration-fingerprints-v*/migration-fingerprints.sqlite3" -def count_fingerprint_computations(monkeypatch: pytest.MonkeyPatch) -> list[int]: - """Count every fingerprint computed instead of reused.""" - - computations: list[int] = [] - - def counting( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computations.append(1) - return build_migration_fingerprint( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) - return computations - - def keep_cache(root: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Leave the persisted cache untouched.""" @@ -129,3 +104,14 @@ def counting( monkeypatch.setattr(native_module, "infer_fixture_column_facts", counting) return inferences + + +class SqlTestsProject: + """A project whose SQL tests are read through a callback each time planning asks for them.""" + + def __init__(self, read_tests: Callable[[], tuple[object, ...]]) -> None: + self._read_tests: Callable[[], tuple[object, ...]] = read_tests + + @property + def sql_tests(self) -> tuple[object, ...]: + return self._read_tests() diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py index 021e1b6932..16eac03704 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py @@ -3,13 +3,15 @@ from __future__ import annotations import json -from collections.abc import Mapping from pathlib import Path import pytest -import sqlbuild._native as native_module -from sqlbuild._native import MigrationFingerprintCache, build_migration_fingerprint +from sqlbuild._native import ( + MigrationFingerprintCache, + build_migration_fingerprint, + migration_fingerprint_work_info, +) from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( MigrationFingerprintCacheTestCase, PersistedMigrationFingerprintTestCase, @@ -19,7 +21,6 @@ ORDERS_SQL, PLAIN_METADATA, corrupt_database, - count_fingerprint_computations, keep_cache, replace_digests_with_non_ascii, tamper_entries, @@ -84,26 +85,9 @@ ids=lambda case: case.description, ) def test_given_fingerprint_requests_when_cached_then_matches_direct_build_with_minimal_work( - test_case: MigrationFingerprintCacheTestCase, monkeypatch: pytest.MonkeyPatch + test_case: MigrationFingerprintCacheTestCase, ) -> None: - computations: list[int] = [] - - def counting( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computations.append(1) - return build_migration_fingerprint( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) + migration_fingerprint_work_info(reset=True) cache: MigrationFingerprintCache = MigrationFingerprintCache() results: list[str | None] = [ @@ -126,7 +110,7 @@ def counting( for metadata, identities, dialect in test_case.requests ] assert all(result is not None for result in results) - assert len(computations) == test_case.expected_computations + assert migration_fingerprint_work_info()[0] == test_case.expected_computations @pytest.mark.parametrize( @@ -175,7 +159,7 @@ def test_given_persisted_fingerprints_when_planning_again_then_reuses_only_valid ] first.persist() test_case.between_runs(tmp_path, monkeypatch) - computations: list[int] = count_fingerprint_computations(monkeypatch) + migration_fingerprint_work_info(reset=True) second: MigrationFingerprintCache = MigrationFingerprintCache(root=tmp_path) results: list[str | None] = [ @@ -188,7 +172,7 @@ def test_given_persisted_fingerprints_when_planning_again_then_reuses_only_valid assert results == expected assert all(result is not None for result in results) - assert len(computations) == test_case.expected_second_run_computations + assert migration_fingerprint_work_info()[0] == test_case.expected_second_run_computations if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 91f8d27399..093e7c4ddb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -4,78 +4,33 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( CompiledObjectKey, PlannedSqlTests, PlannerInputError, - PlanWarning, SqlTestPlanning, SqlTestSelection, - WarningSeverity, start_sql_test_planning, ) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.classes.helpers import SqlTestsProject _KEYS: frozenset[CompiledObjectKey] = frozenset( {CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders")} ) -_PLANNED: PlannedSqlTests = PlannedSqlTests( - selected_keys=_KEYS, - warnings=( - PlanWarning(model_name="orders", severity=WarningSeverity.WARNING, message="planned"), - ), -) @pytest.mark.parametrize( "test_case", [ BackgroundSqlTestPlanningTestCase( - description="enabled planning returns the background result", + description="enabled planning reads the project's SQL tests once", enabled=True, expected_planner_calls=1, ), - ], - ids=lambda case: case.description, -) -def test_given_enabled_planning_when_joining_then_returns_planned_tests( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - calls: list[frozenset[CompiledObjectKey]] = [] - - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection - calls.append(selected_keys) - return _PLANNED - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - - planning: SqlTestPlanning = start_sql_test_planning( - sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] - adapter=object(), # ty: ignore[invalid-argument-type] - selected_keys=_KEYS, - enabled=test_case.enabled, - ) - result: PlannedSqlTests = planning.result() - - assert result == _PLANNED - assert calls == [_KEYS] * test_case.expected_planner_calls - - -@pytest.mark.parametrize( - "test_case", - [ BackgroundSqlTestPlanningTestCase( description="disabled planning never plans and returns no tests", enabled=False, @@ -84,27 +39,18 @@ def plan( ], ids=lambda case: case.description, ) -def test_given_disabled_planning_when_joining_then_returns_empty_tests( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch +def test_given_planning_when_joining_then_returns_planned_tests( + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: - calls: list[frozenset[CompiledObjectKey]] = [] + reads: list[int] = [] - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection - calls.append(selected_keys) - return _PLANNED - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) + def read_tests() -> tuple[object, ...]: + reads.append(1) + return () planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -112,7 +58,7 @@ def plan( result: PlannedSqlTests = planning.result() assert result == PlannedSqlTests(selected_keys=_KEYS) - assert len(calls) == test_case.expected_planner_calls + assert len(reads) == test_case.expected_planner_calls @pytest.mark.parametrize( @@ -128,23 +74,14 @@ def plan( ids=lambda case: case.description, ) def test_given_failing_planning_when_joining_then_raises_original_error( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, selected_keys, sql_test_selection + def read_tests() -> tuple[object, ...]: raise PlannerInputError(str(test_case.expected_error)) - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -165,28 +102,21 @@ def plan( ids=lambda case: case.description, ) def test_given_running_planning_when_listing_threads_then_no_python_worker_blocks_exit( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: started: threading.Event = threading.Event() release: threading.Event = threading.Event() + reads: list[int] = [] - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection + def read_tests() -> tuple[object, ...]: + reads.append(1) started.set() release.wait(timeout=5) - return PlannedSqlTests(selected_keys=selected_keys) - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) + return () planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -199,4 +129,4 @@ def plan( planning.result() assert daemon_by_name.get("sqlbuild-test-planning", True) is True - assert test_case.expected_planner_calls == 1 + assert len(reads) == test_case.expected_planner_calls diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index 463fa7cd55..f1b6418b8c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -10,10 +10,12 @@ _SLOW_TEST_PLANNING: str = dedent( """ import sys + import threading import time import types import sqlbuild._native as native + from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter class SlowFinalizer: @@ -25,15 +27,16 @@ def __del__(self): holder = types.ModuleType("holder") holder.finalizer = SlowFinalizer() sys.modules["holder"] = holder - plan_tests = native.plan_selected_sql_tests + set_difference_operator = DuckDbAdapter.render_set_difference_operator - def slow_plan_selected_sql_tests(**arguments): - time.sleep(1.0) - return plan_tests(**arguments) + def slow_set_difference_operator(self): + if threading.current_thread() is not threading.main_thread(): + time.sleep(1.0) + return set_difference_operator(self) - native.plan_selected_sql_tests = slow_plan_selected_sql_tests + DuckDbAdapter.render_set_difference_operator = slow_set_difference_operator """ ) _FAILED_BUILD: str = dedent( @@ -47,8 +50,16 @@ def slow_plan_selected_sql_tests(**arguments): """ from sqlbuild._native import SqlTestSelection + + class SlowSqlTestsProject: + @property + def sql_tests(self): + time.sleep(1.0) + return () + + native.start_sql_test_planning( - project=None, + project=SlowSqlTestsProject(), adapter=None, selected_keys=frozenset(), sql_test_selection=SqlTestSelection(), @@ -99,6 +110,12 @@ def test_given_sql_test_planning_running_when_interpreter_finalizes_then_process 'SELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) + (project_dir / "tests" / "unit").mkdir(parents=True) + (project_dir / "tests" / "unit" / "test_orders.sql").write_text( + "TEST ();\n\nWITH\n__source__raw_orders AS (\n SELECT 21 AS order_id\n),\n" + "__expected__orders AS (\n SELECT 21 AS order_id\n)\nSELECT 1\n", + encoding="utf-8", + ) script: Path = tmp_path / "planning_exit.py" script.write_text(test_case.script, encoding="utf-8") From 66e1fc4d790a345fc04da24e406b202de53668e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:42:55 +0100 Subject: [PATCH 02/11] refactor(graph): hold the project graph natively and hash Python-node identities at discovery ProjectGraph is a native class over the compiled project and its native lineage graph; selection, DAG and Python run-selection readers use it without attribute lookups. Planner scope, loader and static command phases call their native helpers directly. Discovered Python nodes carry their identity, hashed once by the user-code host at discovery; graphs built from hand-made records still hash on demand. --- .../_helpers/compile_driver/pipeline.rs | 2 +- .../compile_driver/pipeline_entries.rs | 40 +-- .../compile_driver/selection_phases.rs | 270 +++++++++--------- .../_helpers/compile_records/constants.rs | 24 ++ .../_helpers/compiler_records/constants.rs | 14 - .../_helpers/contracts/spec_resolution.rs | 2 +- .../_helpers/dag_artifact/artifact.rs | 19 +- .../discovery/imported_python_nodes.rs | 15 + .../bindings/_helpers/graph/project_graph.rs | 94 +++++- .../_helpers/graph/python_node_graph.rs | 145 ++++++---- .../_helpers/graph/python_selection.rs | 31 +- .../src/bindings/_helpers/graph/views.rs | 2 +- .../_helpers/planner_entries/command_plans.rs | 4 +- .../_helpers/planner_scope/loaders.rs | 32 +-- .../bindings/_helpers/planner_scope/scope.rs | 50 ++-- .../_helpers/planner_selection/selectors.rs | 15 +- .../_helpers/planner_selection/unit_tests.rs | 2 +- .../_helpers/record_behaviours/constants.rs | 9 +- .../_helpers/record_behaviours/discovery.rs | 41 +-- .../src/bindings/_helpers/records/ids.rs | 1 - src/sqlbuild/_native.pyi | 17 +- .../main/compile/test_compile_output.py | 4 +- .../sqlbuild/compiler/dag/main/test_build.py | 4 +- 23 files changed, 471 insertions(+), 366 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index f79c418d9b..0be1890ab6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -41,7 +41,7 @@ fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { /// Compile a project without opening a warehouse planning connection. #[pyfunction] #[pyo3(signature = (**options))] -fn compile_project_phase<'py>( +pub(crate) fn compile_project_phase<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 64231949f7..30e0a2fbd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -4,8 +4,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyFloat}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Discovered Python node functions whose selectors need the Python graph. @@ -99,13 +103,12 @@ pub(crate) fn build_project_graph_from_compiled_project<'py>( /// The project's natively held graph; dict views are built only when a boundary reads them. pub(crate) fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - call_with( - ®istry::class(project.py(), RecordId::ProjectGraph)?, - &[ - ("project", project.clone()), - ("native", project.getattr("lineage_graph")?), - ], - ) + let native: Bound<'py, NativeProjectGraph> = project.getattr("lineage_graph")?.cast_into()?; + Ok(Bound::new( + project.py(), + CompiledProjectGraph::new(project.clone().unbind(), native.unbind()), + )? + .into_any()) } /// Return whether a planned build has runtime work to execute. @@ -174,14 +177,17 @@ fn run_compile_only_pipeline<'py>( &["on_progress"], ), )?; - let compiled: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileProjectPhase)?, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("options", arguments.value("options")?), - ("on_progress", arguments.value("on_progress")?), - ], + let compiled: Bound<'py, PyAny> = compile_project_phase( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("options", arguments.value("options")?), + ("on_progress", arguments.value("on_progress")?), + ], + )?), )?; call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 7776309998..622e50fa5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -5,8 +5,19 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::assembly::assemble_project; +use crate::bindings::_helpers::compile_driver::deferral::{ + build_deferred_locations, resolve_deferred_target_config, +}; +use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_entries::command_plans::{ + resolve_static_command_scope, resolve_static_relation_context, +}; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -30,17 +41,11 @@ fn unit_test_selectors_applied<'py>( if unselective(&resolved)? { return Ok(resolved); } - let split: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::SplitProjectUnitTestSelectors)?, - &[ - ("select", resolved.getattr("select")?), - ("exclude", resolved.getattr("exclude")?), - ("discovered_inputs", discovered.clone()), - ( - "accepts_unit_tests", - resolved.getattr("accepts_unit_test_selectors")?, - ), - ], + let split: Bound<'py, PyAny> = split_project_unit_test_selectors( + &resolved.getattr("select")?, + &resolved.getattr("exclude")?, + discovered, + resolved.getattr("accepts_unit_test_selectors")?.extract()?, )?; imported(py, DATACLASSES_MODULE, "replace")?.call( (resolved,), @@ -61,28 +66,19 @@ fn planner_scopes<'py>( selection: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); - call_with( - ®istry::native(py, NativeAttr::ResolvePlannerScopes)?, - &[ - ("project", project.clone()), - ( - "selection", - call_with( - ®istry::class(py, RecordId::PlannerSelection)?, - &[ - ("select", selection.getattr("select")?), - ("exclude", selection.getattr("exclude")?), - ], - )?, - ), - ( - "policies", - call_with( - ®istry::class(py, RecordId::PlannerPolicies)?, - &[("auto_load_sources", selection.getattr("auto_load_sources")?)], - )?, - ), - ], + resolve_planner_scopes( + project, + &call_with( + ®istry::class(py, RecordId::PlannerSelection)?, + &[ + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ], + )?, + &call_with( + ®istry::class(py, RecordId::PlannerPolicies)?, + &[("auto_load_sources", selection.getattr("auto_load_sources")?)], + )?, ) } @@ -119,13 +115,7 @@ fn resolve_configured_rule_selection<'py>( { return Ok(PyFrozenSet::empty(py)?.into_any()); } - let selection: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveCompileAnalysisSelection)?, - &[ - ("options", options.clone()), - ("discovered_inputs", discovered_inputs.clone()), - ], - )?; + let selection: Bound<'py, PyAny> = compile_analysis_selection(options, discovered_inputs)?; if selection.is_none() || unselective(&selection)? { return Ok(py.None().into_bound(py)); } @@ -150,17 +140,20 @@ fn resolve_analysis_model_names<'py>( if selection.is_none() || unselective(selection)? { return Ok(py.None().into_bound(py)); } - let structural_project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleProject)?, - &[ - ("inputs", compile_inputs.clone()), - ("inference_profile", inference_profile.clone()), - ( - "skip_column_inference", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), - ], + let structural_project: Bound<'py, PyAny> = assemble_project( + py, + Some(keywords( + py, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ( + "skip_column_inference", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?), )?; let models: Bound<'py, PyAny> = scoped_models(&planner_scopes(&structural_project, selection)?)?; @@ -178,34 +171,41 @@ fn deferred_locations<'py>( if defer_to.is_none() { return Ok(py.None().into_bound(py)); } - let deferred_target: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveDeferredTargetConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("defer_to", defer_to), - ( - "current_target_name", - project.getattr("effective_target_name")?, - ), - ], + let deferred_target: Bound<'py, PyAny> = resolve_deferred_target_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("defer_to", defer_to), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?), )?; - call_with( - ®istry::native(py, NativeAttr::BuildDeferredLocations)?, - &[ - ("project", project.clone()), - ("deferred_target_config", deferred_target), - ("effective_vars", project.getattr("effective_vars")?), - ("default_schema", adapter.call_method0("default_schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ], - ) + Ok(build_deferred_locations( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("deferred_target_config", deferred_target), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + )?), + )? + .into_any()) } /// Run canonical compile, selector, and relation phases without build planning. @@ -231,67 +231,67 @@ pub(crate) fn compile_static_command_context<'py>( { selected_keys = PyFrozenSet::empty(py)?.into_any(); } - let compiled: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileProjectPhase)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("options", options.clone()), - ( - "on_progress", - request.optional("on_progress")?.unwrap_or_else(none), - ), - ], + let compiled: Bound<'py, PyAny> = compile_project_phase( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("options", options.clone()), + ( + "on_progress", + request.optional("on_progress")?.unwrap_or_else(none), + ), + ], + )?), )?; let project: Bound<'py, PyAny> = compiled.getattr("project")?; let deferred: Bound<'py, PyAny> = deferred_locations((&discovered, &adapter), &options, &project)?; - let scope: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveStaticCommandScope)?, - &[ - ("project", project.clone()), - ( - "selection", - call_with( - ®istry::class(py, RecordId::PlannerSelection)?, - &[ - ("select", options.getattr("select")?), - ("exclude", options.getattr("exclude")?), - ("selected_keys", selected_keys), - ("sql_test_selection", options.getattr("sql_test_selection")?), - ], - )?, - ), - ("auto_load_sources", options.getattr("auto_load_sources")?), - ], + let scope: Bound<'py, PyAny> = resolve_static_command_scope( + &project, + &call_with( + ®istry::class(py, RecordId::PlannerSelection)?, + &[ + ("select", options.getattr("select")?), + ("exclude", options.getattr("exclude")?), + ("selected_keys", selected_keys), + ("sql_test_selection", options.getattr("sql_test_selection")?), + ], + )?, + options.getattr("auto_load_sources")?.extract()?, )?; - let relations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveStaticRelationContext)?, - &[ - ("project", project.clone()), - ("adapter", adapter), - ("scope", scope.clone()), - ( - "deferral", - call_with( - ®istry::class(py, RecordId::DeferralInputs)?, - &[ - ("deferred_locations", deferred), - ("defer_sources_to", options.getattr("defer_sources_to")?), - ( - "source_deferral_enabled", - options.getattr("source_deferral_enabled")?, - ), - ], - )?, - ), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ( - "relation_keys", - request.optional("relation_keys")?.unwrap_or_else(none), - ), - ], + let relations: Bound<'py, PyAny> = resolve_static_relation_context( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter), + ("scope", scope.clone()), + ( + "deferral", + call_with( + ®istry::class(py, RecordId::DeferralInputs)?, + &[ + ("deferred_locations", deferred), + ("defer_sources_to", options.getattr("defer_sources_to")?), + ( + "source_deferral_enabled", + options.getattr("source_deferral_enabled")?, + ), + ], + )?, + ), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ( + "relation_keys", + request.optional("relation_keys")?.unwrap_or_else(none), + ), + ], + )?), )?; call_with( ®istry::class(py, RecordId::StaticCommandContext)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index fc12fc6178..f6cf74c7a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -944,6 +944,12 @@ const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { FieldDefault::NativeMember(RecordId::LoaderConnectionMode, "SQLBUILD"), ), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { @@ -961,6 +967,12 @@ const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("meta", FieldDefault::None), FieldSpec::new("retry", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { @@ -980,6 +992,12 @@ const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("column_lineage", FieldDefault::None), FieldSpec::new("retry", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { @@ -1000,6 +1018,12 @@ const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("description", FieldDefault::None), FieldSpec::new("meta", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_HOOK_FUNCTION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index b983344d60..6cef6bd314 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -862,19 +862,6 @@ const R_SQL_LEXICAL_SYNTAX: RecordSpec = RecordSpec { ), ], }; -const R_PROJECT_GRAPH: RecordSpec = RecordSpec { - name: "ProjectGraph", - doc: "Static compiled project graph held natively; dict views are built only when read.", - fields: &[ - FieldSpec::required("project"), - FieldSpec { - name: "native", - default: FieldDefault::Required, - compared: false, - shown: false, - }, - ], -}; const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { name: "CompilePipelineResult", doc: "Complete output from the compile-and-plan pipeline.", @@ -964,6 +951,5 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_PYTHON_IDENTITY_DEPENDENCY, R_PYTHON_NODE_IDENTITY, R_SQL_LEXICAL_SYNTAX, - R_PROJECT_GRAPH, R_COMPILE_PIPELINE_RESULT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index 8be885312e..7ee266f7af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -246,7 +246,7 @@ fn loader_destination_parts<'py>( loader_parts(py, destination, (default_database, default_schema)) } -fn loader_parts<'py>( +pub(crate) fn loader_parts<'py>( py: Python<'py>, destination: &str, (default_database, default_schema): (Option>, Option>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs index a4a36ea427..84e847763f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; -use pyo3::{PyRef, pyfunction, wrap_pyfunction}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; @@ -10,8 +10,7 @@ use sqlbuild_core::json::errors::JsonEmitError; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; @@ -39,15 +38,19 @@ pub(crate) fn build_dag_json( ) -> PyResult { compiler_guard(|| { let py = graph.py(); - let project: DagProjectInput = project_input(&native_field(graph, "project")?)?; + let graph: &Bound<'_, CompiledProjectGraph> = graph.cast()?; + let project: DagProjectInput = project_input(graph.get().project(py))?; let python: Option = python_graph .filter(|python_graph| !python_graph.is_none()) .map(python_graph_input) .transpose()?; - let native: Bound<'_, PyAny> = native_field(graph, "native")?; - let native: PyRef<'_, NativeProjectGraph> = native.extract()?; - build(native.graph(), &project, python.as_ref(), project_name) - .map_err(|error| python_error(error, py)) + build( + graph.get().native(py).get().graph(), + &project, + python.as_ref(), + project_name, + ) + .map_err(|error| python_error(error, py)) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index 5a113f57be..22de979c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::discovery::imported_modules::{ use crate::bindings::_helpers::discovery::imported_providers; use crate::bindings::_helpers::discovery::python_hook_files::discover_python_hook_files; +use crate::bindings::_helpers::graph::python_node_graph::node_identity; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; @@ -33,6 +34,8 @@ struct NodeKind { reader: UserCodeHost, record: RecordId, bucket: usize, + /// The node kind and whether its identity hashes documentation fields. + identity: (&'static str, bool), fields: &'static [&'static str], } @@ -42,6 +45,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadLoaderDefinition, record: RecordId::DiscoveredLoaderFunction, bucket: 0, + identity: ("LOADER", false), fields: &[ "depends_on", "destination", @@ -57,6 +61,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadTaskDefinition, record: RecordId::DiscoveredTaskFunction, bucket: 1, + identity: ("TASK", true), fields: &[ "depends_on", "tags", @@ -70,6 +75,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadAssetDefinition, record: RecordId::DiscoveredAssetFunction, bucket: 2, + identity: ("ASSET", true), fields: &[ "depends_on", "tags", @@ -85,6 +91,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadCheckDefinition, record: RecordId::DiscoveredCheckFunction, bucket: 3, + identity: ("CHECK", true), fields: &[ "depends_on", "severity", @@ -131,6 +138,14 @@ impl<'py> Buckets<'py> { pairs.push((field, read.getattr(*field)?)); } pairs.push(("provider_usages", provider_usages(function, providers)?)); + let get = |name: &str| -> PyResult> { + match pairs.iter().find(|(field, _)| *field == name) { + Some((_, value)) => Ok(value.clone()), + None => Ok(py.None().into_bound(py)), + } + }; + let identity: Bound<'py, PyAny> = node_identity(py, kind.identity, &get)?; + pairs.push(("identity", identity)); self.nodes[kind.bucket].push(discovered(py, kind.record, &pairs)?); return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index c4415b453d..7ab69a9101 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -2,7 +2,8 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; @@ -20,6 +21,7 @@ use sqlbuild_analysis::graph::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::graph::views::lineage_graph_views; /// `(key, deps, tags, model folder)`. pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); @@ -45,6 +47,14 @@ pub(crate) struct NativeProjectGraph { graph: ProjectGraph, } +/// Static compiled project graph held natively; dict views are built only when read. +#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] +pub(crate) struct CompiledProjectGraph { + project: Py, + native: Py, + views: PyOnceLock>, +} + fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { deps.iter().map(|dep| (dep.clone(), key.clone())).collect() } @@ -72,6 +82,85 @@ impl NativeProjectGraph { } } +impl CompiledProjectGraph { + /// The graph of `project` over its native lineage graph. + pub(crate) fn new(project: Py, native: Py) -> Self { + Self { + project, + native, + views: PyOnceLock::new(), + } + } + + /// The compiled project this graph indexes. + pub(crate) fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.project.bind(py) + } + + /// The native lineage graph. + pub(crate) fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { + self.native.bind(py) + } + + fn view(&self, py: Python<'_>, name: &str) -> PyResult> { + Ok(self.lineage_views(py)?.bind(py).getattr(name)?.unbind()) + } +} + +#[pymethods] +impl CompiledProjectGraph { + #[new] + #[pyo3(signature = (*, project, native))] + fn py_new(project: Py, native: Py) -> Self { + Self::new(project, native) + } + + #[getter(project)] + fn project_attribute(&self, py: Python<'_>) -> Py { + self.project.clone_ref(py) + } + + #[getter(native)] + fn native_attribute(&self, py: Python<'_>) -> Py { + self.native.clone_ref(py) + } + + /// Lineage edges and selector indexes, converted once. + #[getter] + fn lineage_views(&self, py: Python<'_>) -> PyResult> { + self.views + .get_or_try_init(py, || { + Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) + }) + .map(|views| views.clone_ref(py)) + } + + #[getter] + fn upstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "upstream_deps") + } + + #[getter] + fn downstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "downstream_deps") + } + + #[getter] + fn tag_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "tag_index") + } + + #[getter] + fn path_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "path_index") + } + + #[getter] + fn all_keys(&self, py: Python<'_>) -> PyResult> { + self.view(py, "all_keys") + } +} + /// The project graph indexes of these resources, in project order. pub(crate) fn project_graph(resources: Vec) -> PyResult { compiler_guard(|| { @@ -220,5 +309,6 @@ impl NativeProjectGraph { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index a07b3878b6..91f1c34ff1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -13,9 +13,9 @@ use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_s use crate::bindings::_helpers::graph::views::compiled_graph_keys; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const NAME_KEY: &str = "name"; @@ -32,7 +32,6 @@ struct Vocabulary<'py> { node: Bound<'py, PyAny>, kinds: Bound<'py, PyAny>, sql_ref: Bound<'py, PyAny>, - identity: Bound<'py, PyAny>, providers: Bound<'py, PyDict>, } @@ -44,16 +43,72 @@ fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> if value.is_none() { return Ok(value.clone()); } - native_field(value, "value") + native_field(&value, "value") +} + +/// The decorator configuration the identity of one node of `kind` hashes, read through `get`. +fn decorator_config<'py>( + py: Python<'py>, + (kind, documented): (&str, bool), + get: &dyn Fn(&str) -> PyResult>, +) -> PyResult> { + let config: Bound<'py, PyDict> = PyDict::new(py); + match kind { + "LOADER" => { + config.set_item("columns", get("columns")?)?; + config.set_item("contract", get("contract")?)?; + config.set_item("cursor_column", get("cursor_column")?)?; + config.set_item("destination", get("destination")?)?; + config.set_item("unique_key", get("unique_key")?)?; + config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; + } + "ASSET" => { + config.set_item("columns", get("columns")?)?; + config.set_item("column_lineage", get("column_lineage")?)?; + } + "CHECK" => { + config.set_item("severity", get("severity")?.getattr("value")?)?; + } + _ => {} + } + if documented { + for field in ["description", "group", "meta", "tags"] { + config.set_item(field, get(field)?)?; + } + } + Ok(config) +} + +/// The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. +pub(crate) fn node_identity<'py>( + py: Python<'py>, + (kind, documented): (&str, bool), + get: &dyn Fn(&str) -> PyResult>, +) -> PyResult> { + let kind_value: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)? + .getattr(kind)? + .getattr("value")?; + call_with( + &host::entry(py, UserCodeHost::PythonNodeIdentity)?, + &[ + ("kind", kind_value), + ("name", get("name")?), + ("function", get("function")?), + ("file_path", get("file_path")?), + ( + "decorator_config", + decorator_config(py, (kind, documented), get)?.into_any(), + ), + ], + ) } -/// The kind-specific decorator configuration and metadata record of one discovered function. -fn kind_parts<'py>( +/// The kind's metadata field and record of one discovered function. +fn kind_metadata<'py>( py: Python<'py>, kind: &str, discovered: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, &'static str, Bound<'py, PyAny>)> { - let config: Bound<'py, PyDict> = PyDict::new(py); +) -> PyResult<(&'static str, Bound<'py, PyAny>)> { let get = |name: &str| attribute_value(discovered, name); let metadata = |class: RecordId, fields: &[&str]| -> PyResult> { let pairs: Vec<(&str, Bound<'py, PyAny>)> = fields @@ -63,14 +118,9 @@ fn kind_parts<'py>( call_with(®istry::class(py, class)?, &pairs) }; Ok(match kind { - "LOADER" => { - config.set_item("columns", get("columns")?)?; - config.set_item("contract", get("contract")?)?; - config.set_item("cursor_column", get("cursor_column")?)?; - config.set_item("destination", get("destination")?)?; - config.set_item("unique_key", get("unique_key")?)?; - config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; - let record = metadata( + "LOADER" => ( + "loader", + metadata( RecordId::DiscoveredPythonLoaderMetadata, &[ "destination", @@ -81,28 +131,23 @@ fn kind_parts<'py>( "contract", "connection_mode", ], - )?; - (config, "loader", record) - } + )?, + ), "TASK" => ( - config, "task", metadata(RecordId::DiscoveredPythonTaskMetadata, &["retry"])?, ), - "ASSET" => { - config.set_item("columns", get("columns")?)?; - config.set_item("column_lineage", get("column_lineage")?)?; - let record = metadata( + "ASSET" => ( + "asset", + metadata( RecordId::DiscoveredPythonAssetMetadata, &["columns", "column_lineage", "retry"], - )?; - (config, "asset", record) - } - _ => { - config.set_item("severity", get("severity")?.getattr("value")?)?; - let record = metadata(RecordId::DiscoveredPythonCheckMetadata, &["severity"])?; - (config, "check", record) - } + )?, + ), + _ => ( + "check", + metadata(RecordId::DiscoveredPythonCheckMetadata, &["severity"])?, + ), }) } @@ -114,24 +159,13 @@ fn python_node<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let get = |name: &str| attribute_value(discovered, name); - let (config, metadata_field, metadata) = kind_parts(py, kind, discovered)?; - if documented { - for field in ["description", "group", "meta", "tags"] { - config.set_item(field, get(field)?)?; - } - } + let (metadata_field, metadata) = kind_metadata(py, kind, discovered)?; let kind_member: Bound<'py, PyAny> = vocabulary.kinds.getattr(kind)?; let usages: Bound<'py, PyAny> = provider_usages(&get("function")?, &vocabulary.providers)?; - let identity: Bound<'py, PyAny> = call_with( - &vocabulary.identity, - &[ - ("kind", native_field(&kind_member, "value")?), - ("name", get("name")?), - ("function", get("function")?), - ("file_path", get("file_path")?), - ("decorator_config", config.into_any()), - ], - )?; + let mut identity: Bound<'py, PyAny> = get("identity")?; + if identity.is_none() { + identity = node_identity(py, (kind, documented), &get)?; + } let depends_on: Bound<'py, PyAny> = get("depends_on")?; let mut sql_deps: Vec> = Vec::new(); for dependency in depends_on.try_iter()? { @@ -169,10 +203,10 @@ fn upstream_names( ) -> PyResult>> { let by_key: Bound<'_, PyDict> = PyDict::new(py); for node in nodes { - by_key.set_item(native_field(node, "function")?, native_field(node, "name")?)?; + by_key.set_item(native_field(&node, "function")?, native_field(&node, "name")?)?; } for node in nodes { - let name: Bound<'_, PyAny> = native_field(node, "name")?; + let name: Bound<'_, PyAny> = native_field(&node, "name")?; by_key.set_item( PyTuple::new(py, [PyString::new(py, NAME_KEY).into_any(), name.clone()])?, name, @@ -183,7 +217,7 @@ fn upstream_names( .iter() .map(|node| { let mut names: Vec = Vec::new(); - for dependency in native_field(node, "depends_on")?.try_iter()? { + for dependency in native_field(&node, "depends_on")?.try_iter()? { let dependency: Bound<'_, PyAny> = dependency?; if dependency.is_instance(sql_ref)? { continue; @@ -200,8 +234,8 @@ fn upstream_names( fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { Ok(InventoryNode { - name: native_field(node, "name")?.extract()?, - tags: native_field(node, "tags")?.extract()?, + name: native_field(&node, "name")?.extract()?, + tags: native_field(&node, "tags")?.extract()?, relative_path: node .getattr("relative_path")? .call_method0("as_posix")? @@ -216,7 +250,7 @@ pub(crate) fn python_node_graph<'py>( ) -> PyResult> { let py: Python<'py> = discovered_inputs.py(); let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(discovered_inputs, "providers")?.try_iter()? { + for provider in native_field(&discovered_inputs, "providers")?.try_iter()? { let provider: Bound<'py, PyAny> = provider?; providers.set_item(native_field(&provider, "name")?, provider)?; } @@ -224,7 +258,6 @@ pub(crate) fn python_node_graph<'py>( node: registry::class(py, RecordId::DiscoveredPythonNode)?, kinds: host::entry(py, UserCodeHost::PythonNodeKind)?, sql_ref: host::entry(py, UserCodeHost::SqlResourceRef)?, - identity: host::entry(py, UserCodeHost::PythonNodeIdentity)?, providers, }; let mut nodes: Vec> = Vec::new(); @@ -292,8 +325,8 @@ fn graph_record<'py>( let by_name: Bound<'py, PyDict> = PyDict::new(py); let by_selector: Bound<'py, PyDict> = PyDict::new(py); for node in nodes { - let name: Bound<'py, PyAny> = native_field(node, "name")?; - let kind: String = native_field(node, "kind")?.getattr("value")?.extract()?; + let name: Bound<'py, PyAny> = native_field(&node, "name")?; + let kind: String = native_field(&node, "kind")?.getattr("value")?.extract()?; by_name.set_item(&name, node)?; by_selector.set_item(format!("{kind}:{}", name.str()?), node)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index a0761289a9..84d62c541c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -17,10 +17,10 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; type NameRows = Vec<(String, Vec)>; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. @@ -64,18 +64,16 @@ fn name_rows(mapping: &Bound<'_, PyAny>) -> PyResult { fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { let mut sql_deps: Vec = Vec::new(); - for reference in native_field(node, "sql_deps")?.try_iter()? { + for reference in native_field(&node, "sql_deps")?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; sql_deps.push(( - native_field(&reference, "kind")? - .getattr("value")? - .extract()?, + native_field(&reference, "kind")?.getattr("value")?.extract()?, native_field(&reference, "name")?.extract()?, )); } Ok(PythonSelectionNode { - name: native_field(node, "name")?.extract()?, - kind: native_field(node, "kind")?.getattr("value")?.extract()?, + name: native_field(&node, "name")?.extract()?, + kind: native_field(&node, "kind")?.getattr("value")?.extract()?, sql_deps, }) } @@ -88,9 +86,9 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { .try_iter()? .map(|node| selection_node(&node?)) .collect::>()?, - upstream: name_rows(&native_field(graph, "upstream_deps")?)?, - downstream: name_rows(&native_field(graph, "downstream_deps")?)?, - tags: name_rows(&native_field(graph, "tag_index")?)?, + upstream: name_rows(&native_field(&graph, "upstream_deps")?)?, + downstream: name_rows(&native_field(&graph, "downstream_deps")?)?, + tags: name_rows(&native_field(&graph, "tag_index")?)?, paths: graph .getattr("path_index")? .call_method0("items")? @@ -101,9 +99,8 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { } /// `(name, loader)` of every source in the compiled project. -fn project_sources(project_graph: &Bound<'_, PyAny>) -> PyResult)>> { - project_graph - .getattr("project")? +fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>> { + project .getattr("sources")? .try_iter()? .map(|source| { @@ -168,9 +165,9 @@ pub(crate) fn resolve_python_sql_run_selection( exclude: Vec, ) -> PyResult<(Vec, Vec)> { let py: Python<'_> = project_graph.py(); - let native: Bound<'_, NativeProjectGraph> = - native_field(project_graph, "native")?.downcast_into()?; - let sources: Vec<(String, Option)> = project_sources(project_graph)?; + let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; + let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); + let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index af037455e5..74cbc632aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -40,7 +40,7 @@ pub(crate) fn compiled_graph_keys<'py>( /// Selector name to key, in project order, without the other graph views. #[pyfunction] -fn lineage_graph_names<'py>( +pub(crate) fn lineage_graph_names<'py>( py: Python<'py>, graph: PyRef<'py, NativeProjectGraph>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index a383c0794a..52f30a7ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -92,7 +92,7 @@ fn base_plan_output<'py>( /// Resolve command selection through the canonical build selector phase. #[pyfunction] #[pyo3(signature = (*, project, selection, auto_load_sources = false))] -fn resolve_static_command_scope<'py>( +pub(crate) fn resolve_static_command_scope<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, auto_load_sources: bool, @@ -113,7 +113,7 @@ fn resolve_static_command_scope<'py>( /// Resolve canonical locations and source routing without warehouse inventory. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_static_relation_context<'py>( +pub(crate) fn resolve_static_relation_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs index 429e968709..54ff7f9661 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -1,13 +1,12 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::loader_parts; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_scope::values::{ dependency_names, edges, keys, loader_graph, loader_names, project_graph, }; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -15,6 +14,7 @@ use sqlbuild_analysis::planner_scope::main::{ expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, }; use std::collections::HashSet; +use crate::bindings::_helpers::records::fields::field as native_field; #[pyfunction] #[pyo3(signature = (*, loader_function, loader_functions))] @@ -35,26 +35,20 @@ pub(crate) fn loader_to_source_entry<'py>( loader: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let name = native_field(loader, "name")?; - let mut destination = native_field(loader, "destination")?; + let name = native_field(&loader, "name")?; + let mut destination = native_field(&loader, "destination")?; if destination.is_none() { destination = format!("__loader__{}", name.extract::()?) .into_pyobject(py)? .into_any(); } - let parts = call_with( - ®istry::native(py, NativeAttr::LoaderDestinationPartsFunction)?, - &[ - ("destination", destination), - ( - "default_database", - native_field(project, "effective_target_database")?, - ), - ( - "default_schema", - native_field(project, "effective_target_schema")?, - ), - ], + let parts = loader_parts( + py, + &destination.extract::()?, + ( + Some(native_field(&project, "effective_target_database")?), + Some(native_field(&project, "effective_target_schema")?), + ), )?; let kwargs = PyDict::new(py); kwargs.set_item("name", &name)?; @@ -85,7 +79,7 @@ pub(crate) fn intermediate_map<'py>( let graph = project_graph(project)?; let names = intermediate_sources(&graph, &keys(selected)?, upstream); let loaders = PyDict::new(project.py()); - for loader in native_field(project, "loader_functions")?.try_iter()? { + for loader in native_field(&project, "loader_functions")?.try_iter()? { let loader = loader?; loaders.set_item(native_field(&loader, "name")?, &loader)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index a6f82e5a16..0ca96ebe10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -2,11 +2,17 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::lineage_graph_names; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, + topologically_order_keys, +}; use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; use crate::bindings::_helpers::planner_scope::values::{edges, keys}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::PyDictMethods; @@ -16,6 +22,7 @@ use sqlbuild_analysis::planner_scope::main::{ managed_sources::managed_sources, source_selectors::source_selectors, }; use std::collections::HashSet; +use crate::bindings::_helpers::records::fields::field as native_field; #[pyfunction] #[pyo3(signature = (*, selected_keys, upstream_deps, project))] @@ -25,12 +32,9 @@ pub(crate) fn managed_source_upstream_keys<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let mut managed: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in native_field(&project, "sources")?.try_iter()? { let source = source?; - if !native_field(&source, "source_entry")? - .getattr("loader")? - .is_none() - { + if !native_field(&source, "source_entry")?.getattr("loader")?.is_none() { managed.insert(native_field(&source, "name")?.extract()?); } } @@ -67,22 +71,13 @@ pub(crate) fn build_planner_scope<'py>( let select = default_tuple(&args.value("select")?); let exclude = default_tuple(&args.value("exclude")?); let auto_load = args.value("auto_load_sources")?.is_truthy()?; - let mut upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((&project,))?; - let mut downstream = - registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; - let lineage = native_field(&project, "lineage_graph")?; - let all_keys = registry::native(py, NativeAttr::LineageGraphNames)?.call1((&lineage,))?; + let mut upstream = build_execution_upstream_deps(&project)?.into_any(); + let mut downstream = build_downstream_deps(&upstream)?.into_any(); + let lineage: Bound<'py, NativeProjectGraph> = native_field(&project, "lineage_graph")?.cast_into()?; + let all_keys = lineage_graph_names(py, lineage.borrow())?.into_any(); let mut selected = args.value("selected_keys")?; if selected.is_none() { - selected = call_with( - ®istry::native(py, NativeAttr::ResolveGraphSelection)?, - &[ - ("graph", lineage), - ("select", select.clone()), - ("exclude", exclude), - ], - )?; + selected = resolve_graph_selection(lineage.borrow(), &select, &exclude)?.into_any(); } let names: Vec<(String, (String, String))> = all_keys .call_method0("items")? @@ -118,17 +113,10 @@ pub(crate) fn build_planner_scope<'py>( expand_selected_loader_dependencies(&project, &selected, &upstream, Some(&executable))?; selected = expanded.into_any(); upstream = deps.into_any(); - downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; + downstream = build_downstream_deps(&upstream)?.into_any(); } - let origins = - registry::native(py, NativeAttr::BuildExecutionEdgeOrigins)?.call1((&project,))?; - let order = call_with( - ®istry::native(py, NativeAttr::TopologicallyOrderKeys)?, - &[ - ("upstream", upstream.clone()), - ("injected_edge_origins", origins), - ], - )?; + let origins = build_execution_edge_origins(&project)?.into_any(); + let order = topologically_order_keys(&upstream, Some(origins))?.into_any(); let models = PyDict::new(py); for model in native_field(&project, "models")?.try_iter()? { let model = model?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 863f746c4d..55c6d00b5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -17,12 +17,12 @@ use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::un use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::graph::algorithms::closure; use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const VIEW_OPTIONS: &[&str] = &[ "all_keys", @@ -148,8 +148,11 @@ pub(crate) fn select_project_graph<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, ) -> PyResult> { - let native: PyRef<'py, NativeProjectGraph> = native_field(graph, "native")?.extract()?; - resolved(native.graph(), (select, exclude)) + let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + resolved( + graph.get().native(graph.py()).get().graph(), + (select, exclude), + ) } /// Model, source, seed and function selectors resolved against compiled graph mappings. @@ -222,11 +225,11 @@ fn match_project_selector<'py>( tag_index, path_index, })?; - let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; + let value: Bound<'py, PyString> = native_field(&parsed, "value")?.downcast_into()?; let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { Ok(graph_names(graph.graph())) })?; - let kind: String = native_field(parsed, "kind")?.str()?.extract()?; + let kind: String = native_field(&parsed, "kind")?.str()?.extract()?; selected( &carrier, match_selector(graph.graph(), &kind, &carrier.send(&value)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index e8e322e596..b7cb0c1ccf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -142,7 +142,7 @@ fn split_python<'py>( /// Move unit-test selectors out of resource selectors, or reject them for this command. #[pyfunction] #[pyo3(signature = (*, select, exclude, discovered_inputs, accepts_unit_tests))] -fn split_project_unit_test_selectors<'py>( +pub(crate) fn split_project_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 8025590a3b..cfc3cf2d89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -17,8 +17,7 @@ use crate::bindings::_helpers::record_behaviours::cursors::{ snapshot_watermarks_available, }; use crate::bindings::_helpers::record_behaviours::discovery::{ - constant_logical_type, constant_scalar_type, graph_all_keys, graph_downstream_deps, - graph_lineage_views, graph_path_index, graph_tag_index, graph_upstream_deps, + constant_logical_type, constant_scalar_type, }; use crate::bindings::_helpers::record_behaviours::durations::{ duration_add_to, duration_fixed_seconds, duration_has_calendar_component, @@ -212,11 +211,6 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ), ("ConstantDeclaration", "logical_type", constant_logical_type), ("ConstantDeclaration", "scalar_type", constant_scalar_type), - ("ProjectGraph", "upstream_deps", graph_upstream_deps), - ("ProjectGraph", "downstream_deps", graph_downstream_deps), - ("ProjectGraph", "tag_index", graph_tag_index), - ("ProjectGraph", "path_index", graph_path_index), - ("ProjectGraph", "all_keys", graph_all_keys), ("CompilerDiagnostic", "is_error", diagnostic_is_error), ( "ModelInputBuildContext", @@ -267,7 +261,6 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ]; /// Computed attributes of compiler shapes that a record computes once and keeps. pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ - ("ProjectGraph", "lineage_views", graph_lineage_views), ("LoadedMacro", "injects_context", macro_injects_context), ("CompactLineageFacts", "facts", facts_projected), ("CompiledProject", "lineage_graph", project_lineage_graph), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs index 78d83afa8e..9066e08360 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs @@ -2,8 +2,6 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; - -use crate::bindings::_helpers::graph::views::lineage_graph_views; use crate::bindings::_helpers::records::fields::field as native_field; /// SQL scalar type labels by logical value kind. @@ -26,7 +24,7 @@ pub(crate) fn constant_logical_type(declaration: &Bound<'_, PyAny>) -> PyResult< /// `ConstantDeclaration.scalar_type`: the SQL scalar type label, or the upper-cased logical type. pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult> { - let value: Bound<'_, PyAny> = native_field(declaration, "value")?; + let value: Bound<'_, PyAny> = native_field(&declaration, "value")?; let kind: String = native_field(&value, "kind")?.str()?.extract()?; if let Some((_, label)) = SCALAR_TYPE_LABELS.iter().find(|(name, _)| *name == kind) { return label.into_py_any(declaration.py()); @@ -37,40 +35,3 @@ pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult

) -> PyResult> { - let native = native_field(graph, "native")?; - Ok(lineage_graph_views(graph.py(), native.extract()?)?.unbind()) -} - -fn view(graph: &Bound<'_, PyAny>, name: &str) -> PyResult> { - Ok(native_field(graph, "lineage_views")? - .getattr(name)? - .unbind()) -} - -/// `ProjectGraph.upstream_deps`. -pub(crate) fn graph_upstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "upstream_deps") -} - -/// `ProjectGraph.downstream_deps`. -pub(crate) fn graph_downstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "downstream_deps") -} - -/// `ProjectGraph.tag_index`. -pub(crate) fn graph_tag_index(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "tag_index") -} - -/// `ProjectGraph.path_index`. -pub(crate) fn graph_path_index(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "path_index") -} - -/// `ProjectGraph.all_keys`. -pub(crate) fn graph_all_keys(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "all_keys") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 2ae4bc14a9..30f735897e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -391,7 +391,6 @@ native_ids! { PolyglotAnalysisResult => "PolyglotAnalysisResult", Project => "Project", ProjectConfig => "ProjectConfig", - ProjectGraph => "ProjectGraph", ProjectPath => "ProjectPath", PythonHookEntry => "PythonHookEntry", PythonIdentityDependency => "PythonIdentityDependency", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ed0833d006..e2dc427940 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -16461,6 +16461,7 @@ class DiscoveredLoaderFunction(NativeRecord): description: str | None connection_mode: LoaderConnectionMode provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16477,6 +16478,7 @@ class DiscoveredLoaderFunction(NativeRecord): description: str | None = ..., connection_mode: LoaderConnectionMode = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredTaskFunction(NativeRecord): @@ -16493,6 +16495,7 @@ class DiscoveredTaskFunction(NativeRecord): meta: dict[str, object] | None retry: RetryPolicy | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16506,6 +16509,7 @@ class DiscoveredTaskFunction(NativeRecord): meta: dict[str, object] | None = ..., retry: RetryPolicy | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredAssetFunction(NativeRecord): @@ -16524,6 +16528,7 @@ class DiscoveredAssetFunction(NativeRecord): column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None retry: RetryPolicy | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16539,6 +16544,7 @@ class DiscoveredAssetFunction(NativeRecord): column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = ..., retry: RetryPolicy | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredCheckFunction(NativeRecord): @@ -16555,6 +16561,7 @@ class DiscoveredCheckFunction(NativeRecord): description: str | None meta: dict[str, object] | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16568,6 +16575,7 @@ class DiscoveredCheckFunction(NativeRecord): description: str | None = ..., meta: dict[str, object] | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredHookFunction(NativeRecord): @@ -16808,17 +16816,20 @@ class DiscoveredRuntimeExtensions(NativeRecord): lifecycle_shutdown_timeout_seconds: int | None = ..., ) -> None: ... -class ProjectGraph(NativeRecord): +class ProjectGraph: """Static compiled project graph held natively; dict views are built only when read.""" - project: CompiledProjectView - native: NativeProjectGraph def __init__( self, + *, project: CompiledProjectView, native: NativeProjectGraph, ) -> None: ... @property + def project(self) -> CompiledProjectView: ... + @property + def native(self) -> NativeProjectGraph: ... + @property def lineage_views(self) -> LineageGraphViews: ... @property def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index c25a082aee..3535e143f2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -64,7 +64,9 @@ def test_given_large_hook_integer_when_formatting_json_then_preserves_value( } ), ) - graph = replace(graph, project=project_with(project=graph.project, models=(model,))) + graph = ProjectGraph( + project=project_with(project=graph.project, models=(model,)), native=graph.native + ) payload: dict[str, object] = json.loads( format_compile_json( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 14303e8a88..606fd31189 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -319,8 +319,8 @@ def test_given_qualified_loader_destination_when_building_dag_then_parses_loader test_case: DagLoaderDestinationTestCase, ) -> None: graph: ProjectGraph = build_dag_artifact_test_graph() - graph = replace( - graph, + graph = ProjectGraph( + native=graph.native, project=project_with( project=graph.project, effective_target_database="default_db", From 7789ca8919f10dfb6b5fadd1a1ac19daff78211a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:34:52 +0100 Subject: [PATCH 03/11] refactor(compile): apply target defaults and build lineage rows from native views Native models, seeds and functions take adapter target defaults through direct Rust calls instead of Python method dispatch. The project lineage graph reads source, seed and function rows from their native records. Removes NativeAttr ids that no Rust code or pure-Rust crate names. --- .../compile_driver/target_defaults.rs | 71 +++++++----- .../compiled_project/compiled_model_view.rs | 2 +- .../compiled_project/function_views.rs | 22 +++- .../_helpers/compiled_project/project_view.rs | 31 +++-- .../_helpers/compiled_project/seed_views.rs | 25 +++- .../_helpers/compiled_project/source_views.rs | 15 ++- .../src/bindings/_helpers/records/ids.rs | 107 ------------------ 7 files changed, 129 insertions(+), 144 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index e0cc7f661d..18cd253746 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -4,8 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_project::records::project_with; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// The adapter defaults every compiled location falls back to. @@ -18,11 +21,11 @@ pub(crate) struct TargetDefaults<'py> { impl<'py> TargetDefaults<'py> { /// `(default_schema, default_database)`. - fn pair(&self) -> PyResult> { - PyTuple::new( - self.default_schema.py(), - [self.default_schema.clone(), self.default_database.clone()], - ) + fn pair(&self) -> PyResult<(Option, Option)> { + Ok(( + self.default_schema.extract()?, + self.default_database.extract()?, + )) } } @@ -41,13 +44,21 @@ pub(crate) fn apply_target_defaults<'py>( defaults: &TargetDefaults<'py>, ) -> PyResult> { let py: Python<'py> = project.py(); - let compiled_model: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledModel)?; - let native_seed: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledSeed)?; let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; let mut models: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - models.push(if model.is_instance(&compiled_model)? { + models.push(if let Ok(native) = model.cast::() { + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + )?, + )? + .into_any() + } else { replace.call( (&model,), Some(&keywords( @@ -58,21 +69,21 @@ pub(crate) fn apply_target_defaults<'py>( )], )?), )? - } else { - model.call_method1( - "with_target_defaults", - (defaults.pair()?, &defaults.render_qualified_name), - )? }); } let mut seeds: Vec> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { let seed: Bound<'py, PyAny> = seed?; - seeds.push(if seed.is_instance(&native_seed)? { - seed.call_method1( - "with_target_defaults", - (defaults.pair()?, &defaults.render_qualified_name), + seeds.push(if let Ok(native) = seed.cast::() { + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + )?, )? + .into_any() } else { replace.call( (&seed,), @@ -88,14 +99,22 @@ pub(crate) fn apply_target_defaults<'py>( } let mut functions: Vec> = Vec::new(); for function in project.getattr("functions")?.try_iter()? { - functions.push(function?.call_method1( - "with_target_defaults", - ( - defaults.pair()?, - &defaults.render_qualified_name, - &defaults.python_functions_inherit_default_namespace, - ), - )?); + let function: Bound<'py, PyAny> = function?; + let native: &Bound<'py, NativeCompiledFunction> = function.cast()?; + functions.push( + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + defaults + .python_functions_inherit_default_namespace + .extract()?, + )?, + )? + .into_any(), + ); } project_with( &project.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index f5960d374e..a06b43dfc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -806,7 +806,7 @@ impl NativeCompiledModel { } /// This model with the adapter's default schema and database filled into its destination. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index d4544b9531..7c265a45e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; const TABLE_FUNCTION_TYPE: &str = "table_fn"; const SCALAR_FUNCTION_TYPE: &str = "udf"; @@ -424,6 +425,25 @@ impl NativeCompiledFunction { Self::defaulted(owner, index, None) } + /// The function's project graph row: its key, the resources it reads and its tags. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + Ok(( + ( + function.attached.node_type().to_owned(), + function.attached.name.clone(), + ), + deps.to_vec(), + function.attached.tags.clone(), + None, + )) + }) + } + fn defaulted( owner: Py, index: usize, @@ -535,7 +555,7 @@ function_getters!(NativeCompiledFunction, { } /// This function with target defaults; a non-inheriting Python function is name-qualified. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 88ff02b850..b97f64945b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -9,10 +9,13 @@ use pyo3::{pyclass, pymethods}; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ NativeCompiledModel, model_graph_row, }; +use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, authored_expansion, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::compiled_project::source_views::NativeCompiledSource; use crate::bindings::_helpers::graph::project_graph::{ NativeProjectGraph, ResourceRow, project_graph, }; @@ -521,19 +524,33 @@ impl NativeProjectView { } let compiler: &CompilerHandles = &self.fields.compiler; for source in compiler.sources.bind(py).try_iter()? { - rows.push(resource_row(&source?, None)?); + let source: Bound<'_, PyAny> = source?; + rows.push(match source.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => resource_row(&source, None)?, + }); } for seed in compiler.seeds.bind(py).try_iter()? { let seed: Bound<'_, PyAny> = seed?; - let tags: Bound<'_, PyAny> = seed - .getattr(SCHEMA_ENTRY_ATTRIBUTE)? - .getattr(TAGS_ATTRIBUTE)?; - rows.push(resource_row(&seed, Some(&tags))?); + rows.push(match seed.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = seed + .getattr(SCHEMA_ENTRY_ATTRIBUTE)? + .getattr(TAGS_ATTRIBUTE)?; + resource_row(&seed, Some(&tags))? + } + }); } for function in compiler.functions.bind(py).try_iter()? { let function: Bound<'_, PyAny> = function?; - let tags: Bound<'_, PyAny> = function.getattr(TAGS_ATTRIBUTE)?; - rows.push(resource_row(&function, Some(&tags))?); + rows.push(match function.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = function.getattr(TAGS_ATTRIBUTE)?; + resource_row(&function, Some(&tags))? + } + }); } self.owner.bind(py).borrow_mut().record_graph_build(); Py::new(py, project_graph(rows)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 327b430fe6..e270cd4b0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -14,8 +14,10 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; const SEED_RESOURCE: &str = "seed"; +const TAGS_ATTRIBUTE: &str = "tags"; const NAME_ATTRIBUTE: &str = "name"; const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; @@ -192,6 +194,27 @@ impl NativeCompiledSeed { } } + /// The seed's project graph row: its key and its declared tags. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, py| { + let mut tags: Vec = Vec::new(); + for tag in seed + .schema_entry + .bind(py) + .getattr(TAGS_ATTRIBUTE)? + .try_iter()? + { + tags.push(tag?.str()?.extract()?); + } + Ok(( + (SEED_RESOURCE.to_owned(), seed.name.clone()), + Vec::new(), + tags, + None, + )) + }) + } + fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); @@ -262,7 +285,7 @@ impl NativeCompiledSeed { } /// This seed with the adapter's default schema and database filled into its destination. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index f21ce5a178..6915e80186 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; const SOURCE_RESOURCE: &str = "source"; @@ -184,7 +185,7 @@ impl NativeSourceInput { let attached: &AttachedSource = &source.attached; Ok(( attached.name.clone(), - !native_field(attached.entry.bind(py), "loader")?.is_none(), + !native_field(&attached.entry.bind(py), "loader")?.is_none(), attached.database.clone(), attached.schema.clone(), )) @@ -203,6 +204,18 @@ impl NativeCompiledSource { pub(crate) fn new(owner: Py, index: usize) -> Self { Self { owner, index } } + + /// The source's project graph row: its key, with no dependencies, tags or folder. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(( + (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + Vec::new(), + Vec::new(), + None, + )) + }) + } } #[pymethods] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 30f735897e..d674450405 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -658,25 +658,19 @@ native_ids! { pub(crate) enum NativeAttr { AnalyzeCompileProject => "analyze_compile_project", ApplyCommandWarehouse => "apply_command_warehouse", - ApplyConfiguredMicrobatchLimits => "apply_configured_microbatch_limits", - ApplyDeferredLocations => "apply_deferred_locations", ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleNativeProjectResources => "assemble_native_project_resources", AssembleNativeSqlTests => "assemble_native_sql_tests", AssembleProject => "assemble_project", - AssertNoUnresolvedSqlMarkers => "assert_no_unresolved_sql_markers", AttachChecks => "attach_checks", AttachResources => "attach_resources", AttachSqlbuildPlanOutput => "attach_sqlbuild_plan_output", - AuditIsSelected => "audit_is_selected", AuditProgressReporter => "AuditProgressReporter", BindingPositions => "BindingPositions", BuildAuditCommandPlan => "build_audit_command_plan", BuildCompileDbtManifestIndex => "build_compile_dbt_manifest_index", BuildCompileInputs => "build_compile_inputs", - BuildCompiledFunctionFingerprintSql => "build_compiled_function_fingerprint_sql", BuildCompiledProject => "build_compiled_project", BuildCursorInputRelations => "build_cursor_input_relations", BuildDagJson => "build_dag_json", @@ -687,31 +681,20 @@ native_ids! { BuildDeferredLocations => "build_deferred_locations", BuildDirectSemanticUses => "build_direct_semantic_uses", BuildDisplayOnlySqlbuildPlan => "build_display_only_sqlbuild_plan", - BuildDownstreamDeps => "build_downstream_deps", - BuildEffectiveConnection => "build_effective_connection", BuildEffectiveConnectionConfig => "build_effective_connection_config", BuildEffectiveRuntimeConfig => "build_effective_runtime_config", - BuildEffectiveSettings => "build_effective_settings", BuildEffectiveSettingsConfig => "build_effective_settings_config", BuildEffectiveTargetConfig => "build_effective_target_config", BuildEffectiveTargetNamespace => "build_effective_target_namespace", - BuildEffectiveVars => "build_effective_vars", - BuildExecutionEdgeOrigins => "build_execution_edge_origins", BuildExecutionPlan => "build_execution_plan", - BuildExecutionUpstreamDeps => "build_execution_upstream_deps", BuildFailedSqlbuildModelNames => "build_failed_sqlbuild_model_names", - BuildFunctionLocalHashes => "build_function_local_hashes", - BuildFunctionLocations => "build_function_locations", BuildIdentity => "BUILD_IDENTITY", BuildIntermediateSourceMap => "build_intermediate_source_map", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", BuildModelInputs => "build_model_inputs", - BuildModelLocations => "build_model_locations", - BuildModelVersionIdentityMetadataJson => "build_model_version_identity_metadata_json", BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", - BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", BuildPlannerScope => "build_planner_scope", BuildPlannerSourceFreshnessResult => "build_planner_source_freshness_result", @@ -724,50 +707,36 @@ native_ids! { BuildPythonRelationTargets => "build_python_relation_targets", BuildPythonRunPlanOutputs => "build_python_run_plan_outputs", BuildPythonSqlRunLifecycle => "build_python_sql_run_lifecycle", - BuildRelationCommandPlan => "build_relation_command_plan", - BuildRelationFixtureCompletion => "build_relation_fixture_completion", - BuildRelationFixtureContext => "build_relation_fixture_context", BuildRuleCatalogue => "build_rule_catalogue", BuildRulesRequest => "build_rules_request", BuildScenarioExecutionPlan => "build_scenario_execution_plan", BuildScenarioHashIndex => "build_scenario_hash_index", BuildScenarioRelationMap => "build_scenario_relation_map", BuildSeedCommandPlan => "build_seed_command_plan", - BuildSeedIdentity => "build_seed_identity", - BuildSeedLocations => "build_seed_locations", BuildSelectedAuditEntries => "build_selected_audit_entries", - BuildSourceLoadEntries => "build_source_load_entries", - BuildSourceLoadMap => "build_source_load_map", BuildSourceReadMap => "build_source_read_map", BuildSqlbuildPlanOutput => "build_sqlbuild_plan_output", BuildStaleOutOfSelectionWarnings => "build_stale_out_of_selection_warnings", BuildStaticPipelineResult => "build_static_pipeline_result", BuildTestCommandPlan => "build_test_command_plan", - BuildUpstreamIntermediateSourceMap => "build_upstream_intermediate_source_map", CheckBuildability => "check_buildability", CheckCompileReuse => "check_compile_reuse", CheckCursorTypeConsistency => "check_cursor_type_consistency", - CheckFullRebuildAllowed => "check_full_rebuild_allowed", - CheckSelectedScopeBuildability => "check_selected_scope_buildability", - ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", CollectCompileDiagnostics => "collect_compile_diagnostics", CommandWarehouseScopeContext => "_command_warehouse_scope_context", CompareContracts => "compare_contracts", CompileDbtInteropProject => "compile_dbt_interop_project", CompileFrontier => "compile_frontier", - CompileInputsView => "compile_inputs_view", CompileProjectPhase => "compile_project_phase", CompileReuseExcludedRootDirectories => "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES", CompileReuseRacyWindowNs => "COMPILE_REUSE_RACY_WINDOW_NS", CompileStaticCommandContext => "compile_static_command_context", CompiledCodeIdentity => "compiled_code_identity", - CompiledModelRecord => "compiled_model_record", CompiledProjectFacts => "compiled_project_facts", CompiledProjectRecord => "compiled_project_record", CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", - ComputeQueryHash => "compute_query_hash", ConnectionProgressReporter => "ConnectionProgressReporter", ContentDigest => "content_digest", ContextStoreToken => "context_store_token", @@ -775,10 +744,8 @@ native_ids! { CostContext => "CostContext", CostOutputPayload => "cost_output_payload", CurrentNativeProgressProjector => "current_native_progress_projector", - CurrentTerminalEventIndex => "current_terminal_event_index", CursorCompare => "cursor_compare", CursorExclusiveToInclusive => "cursor_exclusive_to_inclusive", - CursorInputsWithoutRows => "cursor_inputs_without_rows", CursorObservedPartition => "cursor_observed_partition", CursorParse => "cursor_parse", CursorRender => "cursor_render", @@ -794,10 +761,8 @@ native_ids! { DbtRunner => "DbtRunner", DebugDbtFromProject => "debug_dbt_from_project", DeclarationRows => "declaration_rows", - DeclaredColumnAliases => "declared_column_aliases", DeclaredRelations => "declared_relations", DescribeConnectionWarehouse => "describe_connection_warehouse", - DescribeFullRebuildCause => "describe_full_rebuild_cause", DetectChanges => "detect_changes", DetectModelChanges => "detect_model_changes", DetectStaleWarningChanges => "detect_stale_warning_changes", @@ -808,8 +773,6 @@ native_ids! { DiscoverModelFiles => "discover_model_files", DiscoverModelSchemaFiles => "discover_model_schema_files", DiscoverProjectInputs => "discover_project_inputs", - DiscoverProviderClasses => "discover_provider_classes", - DiscoverPythonHookFiles => "discover_python_hook_files", DiscoverPythonModuleFiles => "discover_python_module_files", DiscoverPythonNodeFunctions => "discover_python_node_functions", DiscoverSchemaFiles => "discover_schema_files", @@ -821,10 +784,6 @@ native_ids! { EffectiveConfigConnection => "effective_config_connection", EffectiveConfigSettings => "effective_config_settings", EffectiveConfigVars => "effective_config_vars", - EffectiveTargetConfig => "effective_target_config", - EffectivelyFullRefreshedModelNames => "effectively_full_refreshed_model_names", - EmptyCursorWindow => "empty_cursor_window", - EmptyInputRebuildRefusal => "empty_input_rebuild_refusal", EnforceModelExecutionLimit => "enforce_model_execution_limit", EnforcePlanSafetyPolicies => "enforce_plan_safety_policies", EvaluateModelContracts => "evaluate_model_contracts", @@ -839,16 +798,13 @@ native_ids! { ExecuteDbtSqlbuildWork => "execute_dbt_sqlbuild_work", ExecuteDbtWithoutStateTracking => "execute_dbt_without_state_tracking", ExecutionEventWriter => "ExecutionEventWriter", - ExpandProjectUpstreamKeys => "expand_project_upstream_keys", ExpandSqlMacros => "expand_sql_macros", ExplicitReferenceReport => "explicit_reference_report", ExplicitReferencesHelp => "explicit_references_help", ExternallyDescribedModels => "externally_described_models", - ExtractSeedColumns => "extract_seed_columns", FinalizeRuleFindingsRows => "finalize_rule_findings_rows", FindPythonStringLocations => "find_python_string_locations", FinishRuleRun => "finish_rule_run", - FixtureColumnInferences => "FixtureColumnInferences", FormatAuditExecutionJson => "format_audit_execution_json", FormatBuildExecutionJson => "format_build_execution_json", FormatBuildFooter => "format_build_footer", @@ -876,27 +832,21 @@ native_ids! { FormatSkippedTypeProofRules => "format_skipped_type_proof_rules", FormatTestExecutionJson => "format_test_execution_json", GatherDeferredRelations => "gather_deferred_relations", - GatherPlannerWarehouseState => "gather_planner_warehouse_state", GatherRedirectedCursorSnapshots => "gather_redirected_cursor_snapshots", - GatherSourceColumns => "gather_source_columns", - GatherSourceInspection => "gather_source_inspection", GetConfigCursorBound => "get_config_cursor_bound", GetConfigStr => "get_config_str", GetMaterializationType => "get_materialization_type", GetSemanticSourceColumns => "get_semantic_source_columns", - HardCodedRelationRemedy => "hard_coded_relation_remedy", IdenticalRenames => "identical_renames", InferFixtureColumnFacts => "infer_fixture_column_facts", InferNativeExpressionSourceShapes => "infer_native_expression_source_shapes", InferNativeQueryColumnRows => "infer_native_query_column_rows", InferNativeQueryColumns => "infer_native_query_columns", - InputsWith => "inputs_with", InstallRulesSkills => "install_rules_skills", KnownDeclaredTypes => "known_declared_types", KnownFunctionNames => "known_function_names", LaunchRecordOffered => "_launch_record_offered", LaunchRecordWithheld => "_launch_record_withheld", - LineageGraphNames => "lineage_graph_names", ListCostRuns => "list_cost_runs", LoadCompiledDbtManifest => "load_compiled_dbt_manifest", LoadDbtManifestIndex => "load_dbt_manifest_index", @@ -910,24 +860,15 @@ native_ids! { MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", - MatchProjectRelation => "match_project_relation", MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", - MergeRecomputedModelChanges => "merge_recomputed_model_changes", - MicrobatchGuidance => "microbatch_guidance", - MigrationFingerprintCache => "MigrationFingerprintCache", - MigrationFingerprintRefNames => "migration_fingerprint_ref_names", - MissingUpstreamMessage => "missing_upstream_message", - ModelDefinitionHash => "model_definition_hash", ModelHeaderFailure => "model_header_failure", ModelOutputColumns => "model_output_columns", ModelPlaceholders => "model_placeholders", ModelPythonHookNames => "model_python_hook_names", - ModelSyntaxChecks => "model_syntax_checks", NativeCompiledModel => "NativeCompiledModel", NativeCompiledProject => "NativeCompiledProject", - NativeCompiledSeed => "NativeCompiledSeed", NativeDeclarationContexts => "NativeDeclarationContexts", NativeModelConfigBuilder => "NativeModelConfigBuilder", NativeModelInput => "NativeModelInput", @@ -937,17 +878,13 @@ native_ids! { NativeProgressProjectorContext => "_native_progress_projector_context", NestedCommandProgressCallbacks => "NestedCommandProgressCallbacks", NormalizeAnalysisSql => "normalize_analysis_sql", - NormalizeCursorSnapshotGrain => "normalize_cursor_snapshot_grain", NormalizeDbtProfileConnection => "normalize_dbt_profile_connection", OldNameClaimErrors => "old_name_claim_errors", - OldNameReferenceMessage => "old_name_reference_message", - OldNameSchemasToRead => "old_name_schemas_to_read", OldNameViewSkipReason => "old_name_view_skip_reason", ParseDbtLsJsonLines => "parse_dbt_ls_json_lines", ParseHeaderValues => "parse_header_values", ParseModelHeaders => "parse_model_headers", ParseProjectSelector => "parse_project_selector", - ParseQueryShape => "parse_query_shape", PlanColumnMigrations => "plan_column_migrations", PlanCompiledSqlTestArtifacts => "plan_compiled_sql_test_artifacts", PlanDbtInteropCommand => "plan_dbt_interop_command", @@ -955,17 +892,13 @@ native_ids! { PlanHasExecutableWork => "plan_has_executable_work", PlanModel => "plan_model", PlanReferenceRenames => "plan_reference_renames", - PlanRetention => "plan_retention", PlanRuleFixes => "plan_rule_fixes", PlanRuleRun => "plan_rule_run", PlanSqlTests => "plan_sql_tests", PlanSqlTestsNatively => "plan_sql_tests_natively", PlanStaticSqlTests => "plan_static_sql_tests", PlanningProgressReporter => "PlanningProgressReporter", - PrepareModelHeaderTokens => "prepare_model_header_tokens", PreparedCompileArtifacts => "PreparedCompileArtifacts", - ProjectLineageViews => "project_lineage_views", - ProjectRelativePath => "project_relative_path", ProjectWith => "project_with", PromotionConflictDiagnostics => "promotion_conflict_diagnostics", ProveNativeDynamicContracts => "prove_native_dynamic_contracts", @@ -977,9 +910,7 @@ native_ids! { ReadOldNameViewHistories => "read_old_name_view_histories", ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", - RebindDeclarationScope => "rebind_declaration_scope", RecordCompileInputReads => "record_compile_input_reads", - RecordModelAnalyses => "record_model_analyses", RecordSettingsClassRead => "record_settings_class_read", RecordTemplateReads => "record_template_reads", RejectUnitTestSelectors => "reject_unit_test_selectors", @@ -989,72 +920,47 @@ native_ids! { RenameHints => "rename_hints", RenamesExplainChange => "renames_explain_change", RenderAuditSql => "render_audit_sql", - RenderConstantDeclaration => "render_constant_declaration", RenderDbtExecutionSummaryFooter => "render_dbt_execution_summary_footer", RenderDbtInteropPlan => "render_dbt_interop_plan", RenderDbtNodeResult => "render_dbt_node_result", RenderDiffErrorJson => "render_diff_error_json", RenderProjectVarText => "render_project_var_text", - RenderSourceReadRelation => "render_source_read_relation", RenderSourceRelation => "render_source_relation", - ReplacesIncrementalRelation => "replaces_incremental_relation", ReportCompileDiagnostic => "report_compile_diagnostic", ResolveAnalysisModelNames => "resolve_analysis_model_names", - ResolveAttachmentKind => "resolve_attachment_kind", - ResolveBoundedCursorOverride => "resolve_bounded_cursor_override", ResolveCloneBoundary => "resolve_clone_boundary", ResolveCompileAnalysisSelection => "resolve_compile_analysis_selection", ResolveConfigFilePath => "resolve_config_file_path", ResolveConfiguredRuleSelection => "resolve_configured_rule_selection", ResolveCostRun => "resolve_cost_run", - ResolveCursorInputRoles => "resolve_cursor_input_roles", - ResolveCursorIntrinsics => "resolve_cursor_intrinsics", ResolveDbtConfig => "resolve_dbt_config", ResolveDbtConnectionConfig => "resolve_dbt_connection_config", ResolveDbtExecutable => "resolve_dbt_executable", ResolveDbtExecutionInvocation => "resolve_dbt_execution_invocation", ResolveDbtInteropPlan => "resolve_dbt_interop_plan", - ResolveDbtRefReferences => "resolve_dbt_ref_references", ResolveDbtReferenceRelation => "resolve_dbt_reference_relation", ResolveDeferredTargetConfig => "resolve_deferred_target_config", ResolveEffectiveAdapterName => "resolve_effective_adapter_name", - ResolveEffectiveCollectionRendering => "resolve_effective_collection_rendering", - ResolveEffectiveRunScope => "resolve_effective_run_scope", ResolveEffectiveScenarioConfig => "resolve_effective_scenario_config", - ResolveEffectiveTimestampGrain => "resolve_effective_timestamp_grain", ResolveExternalSqlReferenceResolver => "resolve_external_sql_reference_resolver", ResolveFunctionSql => "resolve_function_sql", ResolveFutureCursorSafety => "resolve_future_cursor_safety", - ResolveGraphSelection => "resolve_graph_selection", - ResolveLineageReference => "resolve_lineage_reference", - ResolveMicrobatchLookback => "resolve_microbatch_lookback", - ResolveModelActions => "resolve_model_actions", ResolveModelDeferralInputs => "resolve_model_deferral_inputs", - ResolveModelFullRefresh => "resolve_model_full_refresh", - ResolveModelHookEntries => "resolve_model_hook_entries", - ResolveModelPlanAction => "resolve_model_plan_action", ResolveModelSql => "resolve_model_sql", - ResolvePlannerScopes => "resolve_planner_scopes", ResolveProjectConnectionConfig => "resolve_project_connection_config", ResolveProjectSelectorTokens => "resolve_project_selector_tokens", ResolveProjectSelectors => "resolve_project_selectors", - ResolvePythonCheckExcludes => "resolve_python_check_excludes", ResolveRawDbtProfileConnection => "resolve_raw_dbt_profile_connection", ResolveRefReferences => "resolve_ref_references", ResolveRuleIgnoreSelectors => "resolve_rule_ignore_selectors", - ResolveSchemaActions => "resolve_schema_actions", ResolveSkippedViewChain => "resolve_skipped_view_chain", ResolveSourceReferences => "resolve_source_references", ResolveSqlbuildExecutionPlanOutput => "resolve_sqlbuild_execution_plan_output", ResolveSqlbuildModelSelectorNames => "resolve_sqlbuild_model_selector_names", - ResolveStartCursorConfig => "resolve_start_cursor_config", - ResolveStaticCommandScope => "resolve_static_command_scope", ResolveStaticRelationContext => "resolve_static_relation_context", - ResolveTableFunctionReferences => "resolve_table_function_references", ResolveTargetConfig => "resolve_target_config", ResolveTargetConnectionConfig => "resolve_target_connection_config", ResolveTargetName => "resolve_target_name", - ResolveUdfReferences => "resolve_udf_references", RouteDbtInteropArgs => "route_dbt_interop_args", RuleSelected => "rule_selected", RulesBulkCachePath => "rules_bulk_cache_path", @@ -1101,7 +1007,6 @@ native_ids! { SchemaValidations => "schema_validations", ScopeMetadataProjection => "scope_metadata_projection", SelectCatalogueRules => "select_catalogue_rules", - SelectLatestFingerprints => "select_latest_fingerprints", SelectProjectGraph => "select_project_graph", SelectedRuleCodes => "selected_rule_codes", SessionOutcomeAnalysis => "session_outcome_analysis", @@ -1110,27 +1015,19 @@ native_ids! { SourceBindingDiagnostics => "source_binding_diagnostics", SourceFreshnessInputError => "SourceFreshnessInputError", SourceFreshnessObservationError => "SourceFreshnessObservationError", - SplitProjectUnitTestSelectors => "split_project_unit_test_selectors", SplitSelectorExpansion => "split_selector_expansion", SplitUnitTestSelectors => "split_unit_test_selectors", - SqlCodeScanner => "SqlCodeScanner", - SqlRefKey => "sql_ref_key", SqlTestModelChainNamesByKey => "sql_test_model_chain_names_by_key", SqlTestScanCache => "SqlTestScanCache", SqlValueValidationError => "SqlValueValidationError", - StandalonePivotModels => "standalone_pivot_models", StaticSqlTestPlanningDiagnostics => "static_sql_test_planning_diagnostics", StoredOldNameViewColumns => "stored_old_name_view_columns", TerminalEventIndex => "TerminalEventIndex", TerminalEventIndexContext => "_terminal_event_index_context", TerminalEventIndexScopeFunction => "terminal_event_index_scope", - TopologicallyOrderKeys => "topologically_order_keys", TransientStatusReporter => "TransientStatusReporter", - TransitiveClosureMany => "transitive_closure_many", TypeCursorOverrides => "type_cursor_overrides", - UnreadableCursorInputsError => "unreadable_cursor_inputs_error", UpdateSqlbuildSkills => "update_sqlbuild_skills", - UpstreamLoaderDependencyNames => "upstream_loader_dependency_names", ValidateAuthoredSqlTestCtes => "validate_authored_sql_test_ctes", ValidateCappedWatermarkInputs => "validate_capped_watermark_inputs", ValidateCliWarehouse => "validate_cli_warehouse", @@ -1144,12 +1041,9 @@ native_ids! { ValidateSourceCursorInputColumns => "validate_source_cursor_input_columns", ValidateSqlSyntax => "validate_sql_syntax", WithBindingDiagnostics => "with_binding_diagnostics", - WithDeclaredSourceReads => "with_declared_source_reads", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", WithTypeRecovery => "with_type_recovery", - WithoutDeclaredColumnsHash => "without_declared_columns_hash", - WithoutDestinationCursor => "without_destination_cursor", WriteAtomically => "write_atomically", WriteCheckResults => "write_check_results", WriteCompileTarget => "write_compile_target", @@ -1162,7 +1056,6 @@ native_ids! { WriteRuntimeTarget => "write_runtime_target", WriteSourceFreshnessRecords => "write_source_freshness_records", WriteSqlbuildSkipNotice => "write_sqlbuild_skip_notice", - YamlEntryLine => "yaml_entry_line", } } From 52b4e616b0d29775eb05d51a8e0d1ba4ec07f8f8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:16:37 +0100 Subject: [PATCH 04/11] refactor(compile): call discovery, configuration and assembly stages directly and count stage work natively Discovery, model configuration and the lane A compile driver files (assembly*, project, discovery, inputs, config_validation) now call the native stage functions directly instead of looking them up on the sqlbuild._native module. Tests that spied on those module attributes read compile_stage_work_info counters (SQL test assembly, pivot proofs, declaration scopes, promotion conflicts, lineage graph builds) or the stage's own answers instead. Removed: the externally_described_models pyfunction (no Python caller), 30 unused NativeAttr ids, the header worker-pool error test and the unretained-model-facts test (both needed a replaced module function to reach an internal invariant). --- .../_helpers/analysis/explicit_references.rs | 2 +- .../analysis_session/stage_functions.rs | 18 ++- .../compile_attachment/declaration_scope.rs | 19 +-- .../compile_attachment/model_config.rs | 44 +++---- .../_helpers/compile_driver/assembly.rs | 29 ++--- .../compile_driver/assembly_checks.rs | 28 +++-- .../_helpers/compile_driver/assembly_facts.rs | 28 +++-- .../compile_driver/assembly_models.rs | 111 ++++++++++-------- .../compile_driver/assembly_project.rs | 17 ++- .../compile_driver/assembly_sql_tests.rs | 21 ++-- .../compile_driver/config_validation.rs | 29 +++-- .../_helpers/compile_driver/discovery.rs | 58 +++++---- .../_helpers/compile_driver/input_reads.rs | 2 +- .../_helpers/compile_driver/inputs.rs | 68 ++++++----- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/project.rs | 34 +++--- .../_helpers/compile_driver/registration.rs | 9 +- .../compile_driver/selection_phases.rs | 2 +- .../_helpers/compile_driver/stage_work.rs | 55 +++++++++ .../_helpers/compiled_project/project_view.rs | 2 + .../contracts/contract_diagnostics.rs | 2 +- .../_helpers/contracts/project_config.rs | 4 +- .../_helpers/contracts/resource_identities.rs | 2 +- .../_helpers/contracts/spec_resolution.rs | 2 +- .../_helpers/discovery/configuration.rs | 12 +- .../_helpers/discovery/description_inputs.rs | 54 ++++----- .../_helpers/discovery/header_values.rs | 50 +++----- .../_helpers/discovery/imported_modules.rs | 26 ++-- .../_helpers/discovery/imported_providers.rs | 31 ++--- .../_helpers/discovery/python_module_files.rs | 2 +- .../_helpers/model_config/config_templates.rs | 14 +-- .../_helpers/model_config/described_models.rs | 10 +- .../model_config/model_config_builder.rs | 4 +- .../_helpers/model_config/registration.rs | 5 +- .../project_assembly/syntax_validation.rs | 6 +- .../src/bindings/_helpers/records/ids.rs | 30 ----- .../_helpers/render/macro_expansion_result.rs | 2 +- .../_helpers/sql_test_glue/compiled_tests.rs | 2 +- .../bindings/_helpers/sql_test_glue/mod.rs | 2 +- .../_helpers/sqltext/model_headers.rs | 12 +- src/sqlbuild/_native.pyi | 4 +- .../cli/commands/main/compile/_test_types.py | 1 + .../cli/commands/main/compile/helpers.py | 3 +- .../compile/test_native_analysis_seams.py | 15 +-- .../sqlbuild/cli/commands/main/test_rules.py | 47 +------- .../compiler/analysis_session/helpers.py | 30 ++--- .../test_native_model_analysis.py | 9 +- .../compiler/compiled_project/_test_types.py | 8 -- .../compiler/compiled_project/helpers.py | 29 +---- .../test_native_compiled_project.py | 28 ----- .../src/sqlbuild/compiler/graph/helpers.py | 21 ---- .../graph/test_native_project_graph.py | 11 +- .../compiler/sql_test_glue/helpers.py | 48 ++++---- .../test_native_sql_test_assembly.py | 11 +- .../discovery/_helpers/_test_types.py | 6 - .../discovery/_helpers/test_sql_models.py | 25 ---- 56 files changed, 501 insertions(+), 644 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 1ddd6ac196..9b40f131af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -77,7 +77,7 @@ impl<'py> PythonSqlOwner<'py> { /// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. #[pyfunction] #[pyo3(signature = (*, project, discovered_inputs, enforce_explicit))] -fn explicit_reference_report<'py>( +pub(crate) fn explicit_reference_report<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 14310b945c..ae3dfe9365 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -169,10 +169,20 @@ fn prove_native_dynamic_contracts<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("prove_native_dynamic_contracts", request)?; - let py: Python<'py> = request.py(); - let session: Bound<'py, PyAny> = request.get("session")?; - let tables: Bound<'py, PyAny> = request.get("tables")?; - let models: Bound<'py, PyAny> = request.get("models")?; + dynamic_contract_proofs( + &request.get("session")?, + &request.get("tables")?, + &request.get("models")?, + ) +} + +/// The proof row per `(sql, families)` model from `session`, or from `tables` without one. +pub(crate) fn dynamic_contract_proofs<'py>( + session: &Bound<'py, PyAny>, + tables: &Bound<'py, PyAny>, + models: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = models.py(); if !models.is_truthy()? { return Ok(PyTuple::empty(py)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index ac40f36b3c..b2bdc37e0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -5,7 +5,7 @@ use pyo3::pyfunction; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compile_driver::stage_work::{DECLARATION_SCOPES, count_stage}; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; @@ -13,6 +13,7 @@ use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; +use crate::bindings::_helpers::records::fields::field as native_field; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( @@ -50,6 +51,7 @@ pub(crate) fn build_declaration_scope<'py>( compile_cache_dir: Option>, ) -> PyResult> { let py: Python<'py> = discovered_inputs.py(); + count_stage(py, DECLARATION_SCOPES, 1)?; let scan_cache: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::SqlTestScanCache)?, &[( @@ -57,12 +59,12 @@ pub(crate) fn build_declaration_scope<'py>( compile_cache_dir.unwrap_or_else(|| py.None().into_bound(py)), )], )?; - let test_files: Bound<'py, PyAny> = native_field(discovered_inputs, "test_files")?; - let scenario_files: Bound<'py, PyAny> = native_field(discovered_inputs, "scenario_files")?; + let test_files: Bound<'py, PyAny> = native_field(&discovered_inputs, "test_files")?; + let scenario_files: Bound<'py, PyAny> = native_field(&discovered_inputs, "scenario_files")?; registry::native(py, NativeAttr::ValidateAuthoredSqlTestCtes)?.call1(( &test_files, &scenario_files, - native_field(sql_lexical_syntax, "native_mapping")?, + native_field(&sql_lexical_syntax, "native_mapping")?, ))?; let native: Bound<'py, PyAny> = Bound::new( py, @@ -165,8 +167,7 @@ pub(crate) fn rebind_declaration_scope<'py>( ) -> PyResult> { let py: Python<'py> = scope.py(); let current: Bound<'py, PyTuple> = indexed_macro_metadata(loaded_macros)?; - let indexed: Bound<'py, PyTuple> = - indexed_macro_metadata(&native_field(scope, "loaded_macros")?)?; + let indexed: Bound<'py, PyTuple> = indexed_macro_metadata(&native_field(&scope, "loaded_macros")?)?; if current.as_any().ne(indexed.as_any())? { return Ok(py.None().into_bound(py)); } @@ -174,9 +175,9 @@ pub(crate) fn rebind_declaration_scope<'py>( discovered_inputs, loaded_macros, ( - native_field(scope, "index")?, - native_field(scope, "resolver")?.getattr("lookup")?, - native_field(scope, "sql_test_scans")?, + native_field(&scope, "index")?, + native_field(&scope, "resolver")?.getattr("lookup")?, + native_field(&scope, "sql_test_scans")?, ), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 1b98edf359..d2e2f9c348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -7,10 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; +use crate::bindings::_helpers::model_config::model_config_builder::NativeModelConfigBuilder; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; /// Capture the project inputs every model's native config build and validation read. #[pyfunction] @@ -20,8 +21,8 @@ pub(crate) fn native_model_config_session<'py>( validator: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let project_config: Bound<'py, PyAny> = native_field(inputs, "project_config")?; - let target: Bound<'py, PyAny> = native_field(inputs, "target_config")?; + let project_config: Bound<'py, PyAny> = native_field(&inputs, "project_config")?; + let target: Bound<'py, PyAny> = native_field(&inputs, "target_config")?; let declared: Bound<'py, PyAny> = native_field(&project_config, "materialization_defaults")?; let materialization_defaults: Bound<'py, PyAny> = if declared.is_truthy()? { declared @@ -39,37 +40,33 @@ pub(crate) fn native_model_config_session<'py>( .map(|class| registry::class(py, *class)) .collect::>>()?, )?; - let target_namespace: Bound<'py, PyAny> = if target.is_none() { - target.clone() + let target_namespace: Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = if target.is_none() { + None } else { - PyTuple::new( - py, - [ - native_field(&target, "database")?, - native_field(&target, "schema")?, - ], - )? - .into_any() + Some((native_field(&target, "database")?, native_field(&target, "schema")?)) }; - let builder: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeModelConfigBuilder)? - .call1(( + let builder: Bound<'py, PyAny> = Bound::new( + py, + NativeModelConfigBuilder::new( ( native_field(&project_config, "defaults")?, - native_field(&project_config, "path_defaults")?, + native_field(&project_config, "path_defaults")?.downcast_into()?, hook_entries, ), ( - native_field(inputs, "effective_vars")?, + native_field(&inputs, "effective_vars")?.downcast_into()?, unicode_environment(py)?, ), ( ( - native_field(inputs, "effective_target_name")?, - native_field(inputs, "run_id")?, + native_field(&inputs, "effective_target_name")?, + native_field(&inputs, "run_id")?, ), target_namespace, ), - ))?; + )?, + )? + .into_any(); let inherited: Bound<'py, PyDict> = PyDict::new(py); let mut materializations: Vec> = vec![py.None().into_bound(py)]; materializations.extend(table_backed(py)?); @@ -81,10 +78,7 @@ pub(crate) fn native_model_config_session<'py>( call_with( ®istry::class(py, RecordId::NativeModelConfigSession)?, &[ - ( - "path_defaults", - native_field(&project_config, "path_defaults")?, - ), + ("path_defaults", native_field(&project_config, "path_defaults")?), ("builder", builder), ("validator", validator.clone()), ("inherited_storage", inherited.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 2bd86cf8e0..f304cbf65d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -4,7 +4,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::analysis::explicit_references::explicit_reference_report; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::project::lineage_mode; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ @@ -14,8 +16,6 @@ use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const OPTIONAL: [&str; 5] = [ "inference_profile", @@ -52,8 +52,7 @@ pub(crate) fn assemble_project<'py>( analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), analysis_model_names: arguments.value("analysis_model_names")?.unbind(), }; - let collector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; + let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { Ok((referenced_project(py, request)?, violations.clone())) @@ -107,19 +106,15 @@ fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult = with_native_project(&project, &inputs.getattr("native_project")?)?; - let references: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ExplicitReferenceReport)?, - &[ - ("project", project.clone()), - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ( - "enforce_explicit", - inputs - .getattr("project_config")? - .getattr("references")? - .getattr("enforce_explicit")?, - ), - ], + let references: Bound<'_, PyAny> = explicit_reference_report( + py, + &project, + &inputs.getattr("discovered_inputs")?, + inputs + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")? + .extract()?, )?; let reference_diagnostics: Bound<'_, PyAny> = references.getattr("diagnostics")?; let unmatched: Bound<'_, PyAny> = references.getattr("unmatched")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 8e9a68e439..8e635a0f6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -8,9 +8,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use crate::bindings::_helpers::render::macro_expansion_result::{ + ExpansionRequest, expansion_result, +}; use pyo3::IntoPyObjectExt; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. @@ -69,7 +72,6 @@ pub(crate) fn test_model_query_overrides<'py>( &inputs.getattr("scope_index")?, (&loaded_macros, &lookup), )?; - let expand: Bound<'py, PyAny> = registry::native(py, NativeAttr::ExpandSqlMacros)?; for model_input in inputs.getattr("model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let file_path: Bound<'py, PyAny> = @@ -81,17 +83,17 @@ pub(crate) fn test_model_query_overrides<'py>( } else { model_input.getattr("query_sql")? }; - let expanded: Bound<'py, PyAny> = call_with( - &expand, - &[ - ("sql", macro_source_sql), - ("file_path", file_path), - ("loaded_macros", loaded_macros.clone()), - ("macro_overrides", macro_mocks.clone()), - ("macro_context", macro_context.clone()), - ("declaration_resolver", declaration_resolver.clone()), - ], - )? + let expanded: Bound<'py, PyAny> = expansion_result(ExpansionRequest { + sql: macro_source_sql.downcast_into()?, + file_path, + loaded_macros: loaded_macros.downcast()?.clone(), + macro_overrides: Some(macro_mocks.downcast()?.clone()), + macro_context: macro_context.clone(), + declaration_resolver: (!declaration_resolver.is_none()) + .then(|| declaration_resolver.clone()), + declarations: None, + consumer: None, + })? .getattr("sql")?; let query: String = validated_model_cursor_sql( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 7f00d931ba..c56cb968ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -2,15 +2,19 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple, PyTupleMethods}; +use crate::bindings::_helpers::analysis_session::stage_functions::dynamic_contract_proofs; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::stage_work::{ + PIVOT_PROOFS, SESSION_PIVOT_PROOFS, count_stage, +}; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -103,16 +107,18 @@ fn pivot_proofs<'py>( ), ], )?; - let proven: Vec> = call_with( - ®istry::native(py, NativeAttr::ProveNativeDynamicContracts)?, - &[ - ("session", session.clone()), - ("tables", tables), - ("models", PyTuple::new(py, pivots.values())?.into_any()), - ], + let proven: Vec> = dynamic_contract_proofs( + session, + &tables, + &PyTuple::new(py, pivots.values())?.into_any(), )? - .try_iter()? - .collect::>()?; + .iter() + .collect(); + count_stage(py, PIVOT_PROOFS, proven.len() as u64)?; + if !session.is_none() { + let session_proofs: usize = proven.iter().filter(|proof| !proof.is_none()).count(); + count_stage(py, SESSION_PIVOT_PROOFS, session_proofs as u64)?; + } if proven.len() != pivots.len() { return Err(PyValueError::new_err( "native pivot proofs do not match the standalone pivot models", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index c89ed3ab49..3152aa8747 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -1,32 +1,32 @@ //! Each compiled model of an assembled project, with its syntax checks and analysis fallback. use pyo3::IntoPyObject; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; +use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; +use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; +use crate::bindings::_helpers::project_assembly::syntax_validation::{ + ParseOptions, validate_hook_sql_syntax, validate_sql_syntax, +}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; -/// The Python stages each model's assembly calls, looked up once per project. -struct ModelStages<'py> { - placeholders: Bound<'py, PyAny>, - validate_hooks: Bound<'py, PyAny>, - validate_sql: Bound<'py, PyAny>, - native_input: Bound<'py, PyAny>, -} - /// One model's assembly inputs: the attached model, its native facts and its two gates. struct ModelRequest<'a, 'py> { model_input: &'a Bound<'py, PyAny>, @@ -42,12 +42,6 @@ pub(crate) fn compiled_models<'py>( (native_models, names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = inputs.py(); - let stages: ModelStages<'py> = ModelStages { - placeholders: registry::native(py, NativeAttr::ModelPlaceholders)?, - validate_hooks: registry::native(py, NativeAttr::ValidateHookSqlSyntax)?, - validate_sql: registry::native(py, NativeAttr::ValidateSqlSyntax)?, - native_input: registry::native(py, NativeAttr::NativeModelInput)?, - }; let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; let mut models: Vec> = Vec::new(); for (index, model_input) in inputs.getattr("model_inputs")?.try_iter()?.enumerate() { @@ -61,14 +55,13 @@ pub(crate) fn compiled_models<'py>( sql_validation_enabled: selected_for_analysis(&model_input, names)?, native_model: native_models.get_item(index)?, }; - models.push(compiled_model(&stages, context, &request)?); + models.push(compiled_model(context, &request)?); } PyTuple::new(py, models) } /// `_assemble_compiled_model`. fn compiled_model<'py>( - stages: &ModelStages<'py>, context: &AnalysisContext<'py>, request: &ModelRequest<'_, 'py>, ) -> PyResult> { @@ -86,31 +79,44 @@ fn compiled_model<'py>( )?, ) .into_any(); - let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; + let placeholder_values: Option> = model_placeholders(model_input)?; + let placeholders: Bound<'py, PyAny> = match &placeholder_values { + Some(values) => values.clone().into_any(), + None => py.None().into_bound(py), + }; let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; - let location: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - model_name.clone(), + let location = || -> PyResult<(String, Bound<'py, PyAny>)> { + Ok(( + model_name.extract()?, model_input.getattr("model_file")?.getattr("file_path")?, - ], - )?; - let syntax: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - placeholders.clone(), - context.profile.getattr("sql_analysis_dialect")?, - ], - )?; + )) + }; + let syntax = || -> PyResult { + Ok(( + placeholder_values + .as_ref() + .map(|values| values.extract()) + .transpose()?, + context.profile.getattr("sql_analysis_dialect")?.extract()?, + )) + }; + let validate_sql = || -> PyResult<()> { + validate_sql_syntax( + py, + &analysis_query_sql.extract::()?, + location()?, + syntax()?, + ) + }; let validates: bool = input_validated && !syntax_validated; if request.sql_validation_enabled && validates { for hook_name in HOOK_NAMES { - stages.validate_hooks.call1(( + validate_hook_sql_syntax( config_values.call_method1("get", (hook_name,))?, hook_name, - &location, - &syntax, - ))?; + location()?, + syntax()?, + )?; } } let mut session_analysis: Bound<'py, PyAny> = py.None().into_bound(py); @@ -137,16 +143,12 @@ fn compiled_model<'py>( session_analysis = PyTuple::new(py, [session, position])?.into_any(); if !analyses.succeeded.contains(&model_name)? { if validates { - stages - .validate_sql - .call1((&analysis_query_sql, &location, &syntax))?; + validate_sql()?; } fallback = fallback_columns(context, &analysis_query_sql, &placeholders)?; } } else if request.sql_validation_enabled && validates { - stages - .validate_sql - .call1((&analysis_query_sql, &location, &syntax))?; + validate_sql()?; } let declared_macro_deps: Bound<'py, PyAny> = model_input.getattr("macro_deps")?; let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { @@ -161,7 +163,7 @@ fn compiled_model<'py>( .into_any() }; let native_model: &Bound<'py, PyAny> = &request.native_model; - if model_input.is_instance(&stages.native_input)? { + if model_input.is_instance_of::() { let macro_list: Bound<'py, PyList> = PyList::new(py, macro_deps.try_iter()?.collect::>>()?)?; return model_input.call_method1( @@ -209,16 +211,21 @@ fn fallback_columns<'py>( ("placeholders", placeholders.clone()), ], )?; - call_with( - ®istry::native(py, NativeAttr::InferNativeQueryColumnRows)?, - &[ - ("queries", PyTuple::new(py, [query])?.into_any()), - ("profile", context.profile.clone()), - ("column_nullability_by_table", context.nullability.clone()), - ], - )? - .get_item(0)? - .get_item(1) + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let nullability: Bound<'py, PyAny> = if context.nullability.is_truthy()? { + context.nullability.clone() + } else { + empty.clone() + }; + let rows: Vec = query_column_rows( + &PyTuple::new(py, [query])?.into_any(), + &context.profile, + (&empty, &nullability), + )?; + rows.into_iter() + .next() + .map(|(_, columns, _)| columns) + .into_bound_py_any(py) } /// `_project_binding_diagnostics`: every analysed model's located binding diagnostics. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index a86956db51..1922d48a6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,7 +16,10 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_effective_adapter_name, resolve_effective_scenario_config, +}; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -228,12 +231,9 @@ fn project_fields<'py>( ("settings", inputs.getattr("effective_settings")?), ( "scenario", - call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveScenarioConfig)?, - &[ - ("project_config", project_config.clone()), - ("local_config", inputs.getattr("local_config")?), - ], + resolve_effective_scenario_config( + &project_config, + &inputs.getattr("local_config")?, )?, ), ("models", models.clone().into_any()), @@ -335,10 +335,9 @@ fn project_view<'py>( ) -> PyResult> { let py: Python<'py> = inputs.py(); let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; - let native_model: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledModel)?; let mut all_native: bool = true; for model in models.as_any().try_iter()? { - if !model?.is_instance(&native_model)? { + if !model?.is_instance_of::() { all_native = false; break; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 9e768fec7e..744d2bc143 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -1,13 +1,16 @@ //! Every SQL test assembled natively; macro-mocked tests' model queries run the user's macros. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyList, PyTuple}; +use pyo3::types::{PyList, PyTuple, PyTupleMethods}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::report_compile_diagnostic; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::stage_work::{SQL_TEST_ASSEMBLY, count_stage}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::compiled_tests::assemble_native_sql_tests; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -19,12 +22,9 @@ pub(crate) fn assembled_sql_tests<'py>( inputs: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let assemblies: Vec> = call_with( - ®istry::native(py, NativeAttr::AssembleNativeSqlTests)?, - &[("inputs", inputs.clone())], - )? - .try_iter()? - .collect::>()?; + count_stage(py, SQL_TEST_ASSEMBLY, 1)?; + let assemblies: Vec> = + assemble_native_sql_tests(py, inputs)?.iter().collect(); let test_inputs: Vec> = inputs .getattr("test_inputs")? .try_iter()? @@ -34,7 +34,6 @@ pub(crate) fn assembled_sql_tests<'py>( "native SQL test assembly returned a different number of tests", )); } - let report: Bound<'py, PyAny> = registry::native(py, NativeAttr::ReportCompileDiagnostic)?; let mocked_input: Bound<'py, PyAny> = registry::class(py, RecordId::CompileModelSqlTestInputPayload)?; let mut tests: Vec> = Vec::with_capacity(assemblies.len()); @@ -51,7 +50,7 @@ pub(crate) fn assembled_sql_tests<'py>( } for item in assembly.getattr("diagnostics")?.try_iter()? { let (key, diagnostic): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - call_with(&report, &[("key", key), ("diagnostic", diagnostic)])?; + report_compile_diagnostic(&key, &diagnostic)?; } let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; if payload.is_instance(&mocked_input)? && payload.getattr("macro_mocks")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 1dd915b13f..4b1fabf64e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -6,10 +6,15 @@ use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_ta use crate::bindings::_helpers::compile_driver::python_calls::call_with; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::project_config::{ + resolve_config_file_path, validate_local_sql_analysis, +}; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_effective_adapter_name, resolve_target_config, +}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; @@ -65,9 +70,7 @@ fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> P local_settings.getattr(SQL_ANALYSIS_KEY)?, ], )?; - registry::native(py, NativeAttr::ValidateLocalSqlAnalysis)? - .call1((config.project_dir, flags))?; - Ok(()) + validate_local_sql_analysis(py, config.project_dir, flags.extract()?) } /// The first target, by name, that selects a command-group warehouse. @@ -109,8 +112,7 @@ fn reject_group( (target, group, value): (String, String, String), ) -> PyErr { let raised = (|| -> PyResult { - let file_path: String = registry::native(py, NativeAttr::ResolveConfigFilePath)? - .call1((project_dir, local))? + let file_path: String = resolve_config_file_path(py, project_dir, local)? .str()? .extract()?; let file_name: &str = if local { @@ -168,16 +170,13 @@ fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> P names.extend(target_names(config.local_config)?); names.sort(); names.dedup(); - let resolve: Bound<'_, PyAny> = registry::native(py, NativeAttr::ResolveTargetConfig)?; let spec_error: Bound<'_, PyAny> = registry::error(py, ErrorId::SpecConfig)?; for name in names { - let resolved: PyResult> = call_with( - &resolve, - &[ - ("project_config", config.project_config.clone()), - ("local_config", config.local_config.clone()), - ("target_name", PyString::new(py, &name).into_any()), - ], + let resolved: PyResult> = resolve_target_config( + py, + config.project_config, + config.local_config, + PyString::new(py, &name).as_any(), ); let Err(error) = resolved else { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 256315d1ea..f48b6de73b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -11,12 +11,20 @@ use crate::bindings::_helpers::compile_driver::integration_loaders::integration_ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, within, }; +use crate::bindings::_helpers::contracts::project_config::{ + load_local_config_file, load_project_config_file, +}; use crate::bindings::_helpers::discovery::declaration_records::{ discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, discover_model_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, }; -use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::imported_providers::{ + discover_event_exporter_functions, discover_provider_classes, +}; +use crate::bindings::_helpers::discovery::imported_python_nodes::{ + discover_hook_functions, discover_materialization_files, discover_python_node_functions, +}; use crate::bindings::_helpers::discovery::model_records::{ ModelFileSelection, discovered_model_files, }; @@ -34,7 +42,7 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -105,19 +113,19 @@ const FILE_DISCOVERERS: [(&str, FileDiscoverer); 14] = [ const PYTHON_DISCOVERERS: [(&str, Discoverer); 5] = [ ( "python_nodes", - Discoverer::Native(NativeAttr::DiscoverPythonNodeFunctions), + Discoverer::Native(discover_python_node_functions), ), ( "materialization_files", - Discoverer::Native(NativeAttr::DiscoverMaterializationFiles), + Discoverer::Native(discover_materialization_files), ), ( "hook_functions", - Discoverer::Native(NativeAttr::DiscoverHookFunctions), + Discoverer::Native(discover_hook_functions), ), ( "event_exporters", - Discoverer::Native(NativeAttr::DiscoverEventExporterFunctions), + Discoverer::Native(discover_event_exporter_functions), ), ( "command_output_sinks", @@ -125,18 +133,32 @@ const PYTHON_DISCOVERERS: [(&str, Discoverer); 5] = [ ), ]; +/// A native discoverer of project Python that takes `project_dir` and `providers`. +type PythonDiscoverer = + for<'py> fn(Bound<'py, PyAny>, Option>) -> PyResult>; + /// Where a project-Python discoverer lives. #[derive(Clone, Copy)] enum Discoverer { - Native(NativeAttr), + Native(PythonDiscoverer), Host(UserCodeHost), } impl Discoverer { - fn function<'py>(self, py: Python<'py>) -> PyResult> { + fn discovered<'py>( + self, + project_dir: &Bound<'py, PyAny>, + providers: &Bound<'py, PyAny>, + ) -> PyResult> { match self { - Self::Native(id) => registry::native(py, id), - Self::Host(entry) => host::entry(py, entry), + Self::Native(discover) => discover(project_dir.clone(), Some(providers.clone())), + Self::Host(entry) => call_with( + &host::entry(project_dir.py(), entry)?, + &[ + ("project_dir", project_dir.clone()), + ("providers", providers.clone()), + ], + ), } } } @@ -203,9 +225,8 @@ fn assembled_inputs<'py>( ) -> PyResult> { let project_dir: &Bound<'py, PyAny> = request.project_dir.bind(py); let project_config: Bound<'py, PyAny> = - registry::native(py, NativeAttr::LoadProjectConfigFile)?.call1((project_dir,))?; - let local_config: Bound<'py, PyAny> = - registry::native(py, NativeAttr::LoadLocalConfigFile)?.call1((project_dir,))?; + load_project_config_file(py, project_dir)?.into_bound(py); + let local_config: Bound<'py, PyAny> = load_local_config_file(py, project_dir)?.into_bound(py); reject_invalid_project_configuration(&ProjectConfiguration { project_dir, project_config: &project_config, @@ -361,16 +382,7 @@ fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { - python.set_item( - field, - call_with( - &discoverer.function(py)?, - &[ - ("project_dir", project_dir.clone()), - ("providers", providers.clone()), - ], - )?, - )?; + python.set_item(field, discoverer.discovered(project_dir, &providers)?)?; } Ok(python) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs index bccd5ac5c6..ca98271b0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs @@ -69,7 +69,7 @@ pub(crate) fn context_read(py: Python<'_>, name: &str) -> PyResult<()> { /// Note a provider settings class instantiated from the environment and files. #[pyfunction] -fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { for recorder in recorders(settings_class.py())? { recorder.get().add_settings_class(settings_class); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index a5d9d34675..2ae7d99e06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -4,19 +4,23 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; use crate::bindings::_helpers::compile_attachment::effective_config::{ build_effective_connection, build_effective_settings, build_effective_vars, }; +use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; use crate::bindings::_helpers::compiled_project::attachment::{attach_checks, attach_resources}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::records::{compile_inputs_view, inputs_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::run_id; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_indexes::{ public_declaration_indexes, public_model_schema_index, @@ -114,8 +118,7 @@ pub(crate) fn attached_inputs<'py>( py: Python<'py>, request: &InputsRequest, ) -> PyResult> { - let collector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; + let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { Ok((compile_inputs(py, request)?, violations.clone())) @@ -147,14 +150,11 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult = host::entry(py, UserCodeHost::LoadProjectMacros)? .call1((discovered.getattr("macro_files")?,))?; let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; - let declaration_scope: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildDeclarationScope)?, - &[ - ("discovered_inputs", discovered.clone()), - ("loaded_macros", loaded_macros), - ("sql_lexical_syntax", syntax.clone()), - ("compile_cache_dir", target.compile_cache_dir.clone()), - ], + let declaration_scope: Bound<'py, PyAny> = build_declaration_scope( + &discovered, + &loaded_macros, + &syntax, + Some(target.compile_cache_dir.clone()), )?; let value_renderer: Bound<'py, PyAny> = adapter_context.getattr("value_renderer")?; let model_context: Bound<'py, PyAny> = call_with( @@ -310,7 +310,7 @@ fn model_build<'py>( (enums.into_any(), constants.into_any()); let schemas: Bound<'py, PyAny> = public_model_schema_index(discovered.clone())?.into_any(); let native_project: Bound<'py, PyAny> = - registry::native(py, NativeAttr::NativeCompiledProject)?.call0()?; + Bound::new(py, NativeCompiledProject::default())?.into_any(); let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; let model_context: Bound<'py, PyAny> = replace.call( (context,), @@ -323,26 +323,30 @@ fn model_build<'py>( ], )?), )?; - let inputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildModelInputs)?, - &[ - ("discovered_inputs", discovered.clone()), - ("context", model_context), - ( - "no_sql_validation", - request.no_sql_validation.bind(py).clone(), - ), - ( - "defer_model_sql_validation", - request.defer_model_sql_validation.bind(py).clone(), - ), - ( - "external_sql_reference_resolver", - request.external_sql_reference_resolver.bind(py).clone(), - ), - ("native_project", native_project.clone()), - ], - )?; + let inputs: Bound<'py, PyAny> = build_model_inputs( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("context", model_context), + ( + "no_sql_validation", + request.no_sql_validation.bind(py).clone(), + ), + ( + "defer_model_sql_validation", + request.defer_model_sql_validation.bind(py).clone(), + ), + ( + "external_sql_reference_resolver", + request.external_sql_reference_resolver.bind(py).clone(), + ), + ("native_project", native_project.clone()), + ], + )?), + )? + .into_any(); let context: Bound<'py, PyAny> = replace.call( (context,), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index ef8fdc85aa..b450d2fac9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -27,5 +27,6 @@ pub(crate) mod python_plan_entries; pub(crate) mod registration; pub(crate) mod relation_targets; pub(crate) mod selection_phases; +pub(crate) mod stage_work; pub(crate) mod target_defaults; pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index ca9031ae7c..c68c057499 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -20,10 +20,13 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridg use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; +use crate::bindings::_helpers::compile_driver::stage_work::{PROMOTION_CONFLICTS, count_stage}; use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; @@ -146,14 +149,8 @@ fn python_compiled_project<'py>( } let inference_profile: Bound<'py, PyAny> = adapter.call_method0("expression_inference_profile")?; - let analysis_model_names: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveAnalysisModelNames)?, - &[ - ("compile_inputs", compile_inputs.clone()), - ("inference_profile", inference_profile.clone()), - ("selection", selection.clone()), - ], - )?; + let analysis_model_names: Bound<'py, PyAny> = + resolve_analysis_model_names(&compile_inputs, &inference_profile, selection)?; let project: Bound<'py, PyAny> = assemble_project( py, Some(keywords( @@ -220,17 +217,14 @@ fn targeted_project<'py>( } else { pyo3::types::PyString::new(py, PROJECT_CONFIG_FILENAME).into_any() }; - let conflicts: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PromotionConflictDiagnostics)?, - &[ - ("project", project.clone()), - ( - "adapter_default", - adapter.call_method0("default_table_promotion_mode")?, - ), - ("settings_file", settings_file), - ], - )?; + count_stage(py, PROMOTION_CONFLICTS, 1)?; + let conflicts: Bound<'py, PyAny> = promotion_conflict_diagnostics( + py, + &project, + &adapter.call_method0("default_table_promotion_mode")?, + settings_file.extract()?, + )? + .into_any(); let diagnostics: Bound<'py, PyTuple> = PyTuple::new( py, project diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 917edd104b..620f94f94a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -1,11 +1,12 @@ //! Register the native compile command driver. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_driver::{ analysis, analysis_cache, assembly, command, deferral, diagnostic_collector, diff, discovery, input_reads, inputs, pipeline, pipeline_entries, plan_result, project, python_plan_entries, - relation_targets, selection_phases, targets, + relation_targets, selection_phases, stage_work, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -26,5 +27,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { python_plan_entries::register(module)?; relation_targets::register(module)?; selection_phases::register(module)?; + module.add_function(wrap_pyfunction!( + stage_work::compile_stage_work_info, + module + )?)?; targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 622e50fa5a..c750e7c002 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -131,7 +131,7 @@ fn resolve_configured_rule_selection<'py>( /// Return the models selected for deep analysis, or None to analyse every model. #[pyfunction] #[pyo3(signature = (*, compile_inputs, inference_profile, selection))] -fn resolve_analysis_model_names<'py>( +pub(crate) fn resolve_analysis_model_names<'py>( compile_inputs: &Bound<'py, PyAny>, inference_profile: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs new file mode 100644 index 0000000000..502b80cded --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -0,0 +1,55 @@ +//! How often each compile stage ran in this process, for tests that assert stage work. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Py, PyRefMut, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyclass, pyfunction}; + +pub(crate) const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; +pub(crate) const PROMOTION_CONFLICTS: &str = "promotion_conflict_diagnostics"; +pub(crate) const PIVOT_PROOFS: &str = "pivot_proofs"; +pub(crate) const SESSION_PIVOT_PROOFS: &str = "session_pivot_proofs"; +pub(crate) const DECLARATION_SCOPES: &str = "build_declaration_scope"; +pub(crate) const LINEAGE_GRAPH_BUILDS: &str = "lineage_graph_builds"; + +/// Compile stage runs since the last reset, by stage. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct StageWork { + runs: BTreeMap<&'static str, u64>, +} + +static STAGE_WORK: PyOnceLock> = PyOnceLock::new(); + +fn stage_work(py: Python<'_>) -> PyResult> { + Ok(STAGE_WORK + .get_or_try_init(py, || Py::new(py, StageWork::default()))? + .bind(py) + .borrow_mut()) +} + +/// Add `count` runs of `stage`. +pub(crate) fn count_stage(py: Python<'_>, stage: &'static str, count: u64) -> PyResult<()> { + *stage_work(py)?.runs.entry(stage).or_default() += count; + Ok(()) +} + +/// Report the runs of each compile stage so far, optionally resetting them. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn compile_stage_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult> { + let mut work = stage_work(py)?; + let info: BTreeMap = work + .runs + .iter() + .map(|(stage, count)| ((*stage).to_owned(), *count)) + .collect(); + if reset { + *work = StageWork::default(); + } + Ok(info) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index b97f64945b..59376c3924 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -6,6 +6,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{pyclass, pymethods}; +use crate::bindings::_helpers::compile_driver::stage_work::{LINEAGE_GRAPH_BUILDS, count_stage}; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ NativeCompiledModel, model_graph_row, }; @@ -553,6 +554,7 @@ impl NativeProjectView { }); } self.owner.bind(py).borrow_mut().record_graph_build(); + count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; Py::new(py, project_graph(rows)?) }) .map(|graph| graph.clone_ref(py)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 66a5399376..64b2d21b60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -206,7 +206,7 @@ fn evaluate_model_contracts<'py>( /// One K011 error per enforced-contract table model under immediate promotion. #[pyfunction] #[pyo3(signature = (*, project, adapter_default, settings_file))] -fn promotion_conflict_diagnostics<'py>( +pub(crate) fn promotion_conflict_diagnostics<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, adapter_default: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index 7370c4c364..9d5236873c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -186,7 +186,7 @@ pub(crate) fn load_local_config_file( /// The project or local configuration file path, rejecting an unconverted legacy file. #[pyfunction] -fn resolve_config_file_path<'py>( +pub(crate) fn resolve_config_file_path<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, local: bool, @@ -200,7 +200,7 @@ fn resolve_config_file_path<'py>( /// `validate_local_sql_analysis_policy` from the project requirement and the local setting. #[pyfunction] -fn validate_local_sql_analysis( +pub(crate) fn validate_local_sql_analysis( py: Python<'_>, project_dir: &Bound<'_, PyAny>, flags: (bool, bool, bool), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs index f19c8b5671..212835ff19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -15,7 +15,7 @@ const UDF_NODE_TYPE: &str = "udf"; /// Reject one identity unless it uses canonical lowercase snake_case. #[pyfunction] #[pyo3(signature = (*, name, kind, path, private_identity = false))] -fn validate_resource_identity( +pub(crate) fn validate_resource_identity( name: &Bound<'_, PyString>, kind: &str, path: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index 7ee266f7af..a262bdf308 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -143,7 +143,7 @@ fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bou /// Resolve scenario config, allowing local overrides to replace project rules. #[pyfunction] #[pyo3(signature = (*, project_config, local_config))] -fn resolve_effective_scenario_config<'py>( +pub(crate) fn resolve_effective_scenario_config<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index a3364bcffc..3b6b868ec7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -1,12 +1,15 @@ //! Project inputs holding only the project and local configuration. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; +use crate::bindings::_helpers::contracts::project_config::{ + load_local_config_file, load_project_config_file, +}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. @@ -31,12 +34,11 @@ pub(crate) fn discover_project_configuration<'py>( ("project_dir", project_dir.clone()), ( "project_config", - registry::native(py, NativeAttr::LoadProjectConfigFile)? - .call1((project_dir,))?, + load_project_config_file(py, project_dir)?.into_bound(py), ), ( "local_config", - registry::native(py, NativeAttr::LoadLocalConfigFile)?.call1((project_dir,))?, + load_local_config_file(py, project_dir)?.into_bound(py), ), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs index 2fb0491bd0..dc659ce42d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -1,12 +1,20 @@ //! Model description resolution for header-only tooling, without importing project Python. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyCFunction, PyFrozenSet}; +use pyo3::types::{PyCFunction, PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::project_config::load_project_config_file; +use crate::bindings::_helpers::discovery::declaration_records::discover_model_schema_files; +use crate::bindings::_helpers::discovery::model_records::{ + ModelFileSelection, discovered_model_files, +}; +use crate::bindings::_helpers::discovery::snapshot::in_snapshot; +use crate::bindings::_helpers::model_config::described_models::externally_described_models as described_models; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::declaration_indexes::public_model_schema_index; /// Models and model schemas, skipping files that cannot be read. #[pyfunction] @@ -15,20 +23,14 @@ pub(crate) fn discover_model_description_inputs<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let native = |name: NativeAttr| registry::native(py, name); let project_config: Bound<'py, PyAny> = - native(NativeAttr::LoadProjectConfigFile)?.call1((project_dir,))?; + load_project_config_file(py, project_dir)?.into_bound(py); let skip: Bound<'py, PyAny> = PyCFunction::new_closure(py, None, None, |_arguments, _keywords| -> PyResult<()> { Ok(()) })? .into_any(); - let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - let scope: Bound<'py, PyAny> = call_with( - &native(NativeAttr::DirectorySnapshotScope)?, - &[("project_dir", project_dir.clone())], - )?; - within(&scope, |_| { + in_snapshot(project_dir, || { call_with( ®istry::class(py, RecordId::DiscoveredProjectInputs)?, &[ @@ -39,22 +41,19 @@ pub(crate) fn discover_model_description_inputs<'py>( ), ( "model_files", - call_with( - &native(NativeAttr::DiscoverModelFiles)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_implicit_alias_columns", disabled.clone()), - ("extract_output_column_locations", disabled), - ("on_fault", skip.clone()), - ], - )?, + discovered_model_files( + project_dir, + (false, false), + ModelFileSelection { + selected_model_names: None, + on_fault: Some(skip.clone()), + }, + )? + .into_any(), ), ( "model_schema_files", - call_with( - &native(NativeAttr::DiscoverModelSchemaFiles)?, - &[("project_dir", project_dir.clone()), ("on_fault", skip)], - )?, + discover_model_schema_files(project_dir, Some(skip))?.into_any(), ), ], ) @@ -67,11 +66,8 @@ pub(crate) fn externally_described_models<'py>( ) -> PyResult> { let py: Python<'py> = project_dir.py(); let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; - let schemas: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicModelSchemaIndex)?.call1((&inputs,))?; - let described: Bound<'py, PyAny> = - registry::native(py, NativeAttr::ExternallyDescribedModels)?.call1((&inputs, schemas))?; - PyFrozenSet::new(py, described.try_iter()?.collect::>>()?) + let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; + PyFrozenSet::new(py, described_models(inputs, schemas)?) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index c10008829f..ff016dd6db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -6,20 +6,14 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::model_headers::{ + ColumnLocation, ParsedModelHeader, header_failure, model_output_columns, parse_model_headers, +}; const HEADER_CACHE_SIZE: usize = 4096; -/// `(name, line, column, end line, end column)`. -type LocationRow<'py> = ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, -); - /// Parsed headers by text: `(values or None, error or None)`. static HEADER_CACHE: PyOnceLock> = PyOnceLock::new(); @@ -49,14 +43,9 @@ fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { cache.clear(); } let names: Bound<'_, PyList> = missing.keys(); - let parsed: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ParseModelHeaders)?.call1((&names,))?; - for (header, row) in names.iter().zip(parsed.try_iter()?) { - let row: Bound<'_, PyAny> = row?; - cache.set_item( - header, - PyTuple::new(py, [row.get_item(0)?, row.get_item(2)?])?, - )?; + let parsed: Vec = parse_model_headers(py, names.extract()?)?; + for (header, (values, _offsets, error)) in names.iter().zip(parsed) { + cache.set_item(header, (values, error))?; } Ok(()) } @@ -79,11 +68,12 @@ fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult = request.get("header_line")?; - let (detail, help, line): (String, Option, Option) = - registry::native(py, NativeAttr::ModelHeaderFailure)? - .call1((error.str()?, &header, &header_line))? - .extract()?; + let header_line: Option = request.get("header_line")?.extract()?; + let (detail, help, line): (String, Option, Option) = header_failure( + &error.str()?.extract::()?, + &header.extract::()?, + header_line, + ); let file_path: String = request.get("file_path")?.str()?.extract()?; let location: String = match line { Some(line) => format!("{file_path}:{line}"), @@ -107,14 +97,13 @@ fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - rows: &Bound<'py, PyAny>, + rows: Vec, relative_path: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = rows.py(); + let py: Python<'py> = relative_path.py(); let location_class: Bound<'py, PyAny> = registry::class(py, RecordId::SourceLocation)?; let locations: Bound<'py, PyDict> = PyDict::new(py); - for row in rows.try_iter()? { - let (name, line, column, end_line, end_column): LocationRow<'py> = row?.extract()?; + for (name, line, column, end_line, end_column) in rows { let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("path", relative_path)?; arguments.set_item("line", line)?; @@ -133,14 +122,13 @@ fn model_output_column_locations<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("model_output_column_locations", request)?; - let py: Python<'py> = request.py(); let implicit: bool = match request.optional("extract_implicit_alias_columns")? { Some(value) => value.is_truthy()?, None => true, }; - let rows: Bound<'py, PyAny> = registry::native(py, NativeAttr::ModelOutputColumns)? - .call1((request.get("contents")?, implicit))?; - source_locations(&rows, &request.get("relative_path")?) + let contents: String = request.get("contents")?.extract()?; + let rows: Vec = model_output_columns(&contents, implicit)?; + source_locations(rows, &request.get("relative_path")?) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 8134adb1b4..94a67268c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -2,17 +2,18 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::discovery::locations::project_relative_path; +use crate::bindings::_helpers::discovery::python_module_files::discover_python_module_files; use crate::bindings::_helpers::event_exporting::scope::current_scope; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; +use crate::bindings::_helpers::records::fields::field as native_field; /// One project directory under discovery. pub(crate) struct ProjectDir<'py> { @@ -49,22 +50,9 @@ impl<'py> ProjectDir<'py> { root: &str, public_only: bool, ) -> PyResult>> { - let py: Python<'py> = self.py(); - call_with( - ®istry::native(py, NativeAttr::DiscoverPythonModuleFiles)?, - &[ - ("project_dir", self.path.clone()), - ("root", PyString::new(py, root).into_any()), - ( - "public_only", - pyo3::types::PyBool::new(py, public_only) - .to_owned() - .into_any(), - ), - ], - )? - .try_iter()? - .collect() + Ok(discover_python_module_files(&self.path, root, public_only)? + .iter() + .collect()) } /// `shim.(file_path=file_path, project_dir=project_dir)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index 16b9f904bb..aebb1f2581 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -4,18 +4,19 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods}; +use crate::bindings::_helpers::compile_driver::input_reads::record_settings_class_read; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::resource_identities::validate_resource_identity; use crate::bindings::_helpers::discovery::imported_modules::{ ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, provider_by_name, provider_usages, shim, tuple_of, }; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; const PROVIDER_ERROR: ErrorId = ErrorId::ProviderDiscovery; const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; @@ -31,16 +32,7 @@ fn reject_invalid_identity( name: &Bound<'_, PyAny>, relative_path: &Bound<'_, PyAny>, ) -> PyResult<()> { - let py: Python<'_> = name.py(); - call_with( - ®istry::native(py, NativeAttr::ValidateResourceIdentity)?, - &[ - ("name", name.clone()), - ("kind", PyString::new(py, "provider").into_any()), - ("path", relative_path.clone()), - ], - )?; - Ok(()) + validate_resource_identity(name.downcast()?, "provider", relative_path, false) } /// The provider's validated name, from its explicit `provider_name` or `name()`. @@ -49,7 +41,7 @@ fn provider_name<'py>( relative_path: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = provider_class.py(); - let explicit: Bound<'py, PyAny> = native_field(provider_class, "provider_name")?; + let explicit: Bound<'py, PyAny> = native_field(&provider_class, "provider_name")?; if !explicit.is_none() { reject_invalid_identity(&explicit, relative_path)?; } @@ -139,11 +131,10 @@ pub(crate) fn discover_provider_classes<'py>( ) -> PyResult> { let py: Python<'py> = project_dir.py(); let project = ProjectDir { path: project_dir }; - let record_read: Bound<'py, PyAny> = registry::native(py, NativeAttr::RecordSettingsClassRead)?; if let Some(cached) = project.cached_extensions()? { let providers: Bound<'py, PyAny> = cached.get_item(0)?; for provider in providers.try_iter()? { - record_read.call1((native_field(&provider?, "provider_class")?,))?; + record_settings_class_read(&native_field(&provider?, "provider_class")?)?; } return Ok(providers); } @@ -178,7 +169,7 @@ pub(crate) fn discover_provider_classes<'py>( )); } seen.push((name_text, file_path.clone())); - record_read.call1((&provider_class,))?; + record_settings_class_read(&provider_class)?; let settings: Bound<'py, PyAny> = provider_instance(&provider_class, (&name, &relative_path))?; providers.push(discovered( @@ -282,10 +273,10 @@ fn exporter_provider_usages<'py>( (project, providers): (&ProjectDir<'py>, &Bound<'py, PyDict>), ) -> PyResult> { let py: Python<'py> = project.py(); - let function: Bound<'py, PyAny> = native_field(declaration, "function")?; - let name: String = native_field(declaration, "name")?.extract()?; + let function: Bound<'py, PyAny> = native_field(&declaration, "function")?; + let name: String = native_field(&declaration, "name")?.extract()?; let relative_path: String = project - .relative(&native_field(declaration, "file_path")?)? + .relative(&native_field(&declaration, "file_path")?)? .str()? .extract()?; let empty: Bound<'py, PyAny> = imported(py, "inspect", "Parameter")?.getattr("empty")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs index 2836e3582e..be655ab2fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs @@ -14,7 +14,7 @@ use crate::bindings::models::NativeProjectTree; /// `sorted((project_dir / root).rglob("*.py"))` without `__init__` modules, or `_` parts too. #[pyfunction] #[pyo3(signature = (*, project_dir, root, public_only))] -fn discover_python_module_files<'py>( +pub(crate) fn discover_python_module_files<'py>( project_dir: &Bound<'py, PyAny>, root: &str, public_only: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 4fba09abbf..462efe8530 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -19,10 +19,9 @@ use sqlbuild_model_config::templates::models::{ }; use sqlbuild_model_config::templates::types::TemplateHost; +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::errors::NativeConfigError; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; @@ -241,15 +240,8 @@ impl<'py> TemplateHost for PythonHost<'py> { if value.is_exact_instance_of::() { return self.text_of(value); } - let rendered = - registry::native(self.py, NativeAttr::RenderProjectVarText).and_then(|render| { - let arguments = PyDict::new(self.py); - arguments.set_item("value", value)?; - arguments.set_item("label", label)?; - render.call((), Some(&arguments)) - }); - match rendered { - Ok(text) => self.text_of(&text), + match project_var_text(value, label) { + Ok(text) => Ok(text), Err(error) if error.is_instance_of::(self.py) => Err( TemplateFailure::Invalid(TemplateError::Message(error.value(self.py).to_string())), ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs index 83850b5aa7..3dcabf33db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -1,8 +1,7 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; @@ -24,8 +23,7 @@ fn non_empty_text(value: Option>) -> PyResult { } /// The file path of each model whose layered config or model schema gives it a description. -#[pyfunction] -fn externally_described_models<'py>( +pub(crate) fn externally_described_models<'py>( inputs: Bound<'py, PyAny>, model_schemas: Bound<'py, PyDict>, ) -> PyResult>> { @@ -74,7 +72,3 @@ fn externally_described_models<'py>( } Ok(described) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(externally_described_models, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 5ee6943fc6..ab7dc904ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -118,7 +118,7 @@ pub(crate) enum ConfigBuild<'py> { } /// `((target name, run id), (database, schema) or None)` of the active target. -type TargetInputs<'py> = ( +pub(crate) type TargetInputs<'py> = ( (Bound<'py, PyAny>, Bound<'py, PyAny>), Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, ); @@ -126,7 +126,7 @@ type TargetInputs<'py> = ( #[pymethods] impl NativeModelConfigBuilder { #[new] - fn new( + pub(crate) fn new( layers: (Bound<'_, PyAny>, Bound<'_, PyDict>, Bound<'_, PyTuple>), sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), target: TargetInputs<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index fc8b3615f2..f0cc73ea11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,15 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_values, described_models, header_columns, - model_config_builder, model_validator, template_data, unicode_environment, + column_migrations, config_errors, config_values, header_columns, model_config_builder, + model_validator, template_data, unicode_environment, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; config_values::register(module)?; - described_models::register(module)?; header_columns::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index 7503096c23..b8ded37b92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::records::registry; const GENERIC_DIALECT: &str = "generic"; /// `(placeholder defaults, dialect)` a statement parses with. -type ParseOptions = (Option>, Option); +pub(crate) type ParseOptions = (Option>, Option); /// What every statement of one model is parsed with. struct SyntaxContext<'a, 'py> { @@ -145,7 +145,7 @@ fn check_hook_value( /// Validate the SQL of one model's hook value, raising the first parse failure. #[pyfunction] #[pyo3(signature = (value, hook_name, model, parsing=(None, None)))] -fn validate_hook_sql_syntax( +pub(crate) fn validate_hook_sql_syntax( value: Bound<'_, PyAny>, hook_name: &str, model: (String, Bound<'_, PyAny>), @@ -179,7 +179,7 @@ fn validate_hook_sql_syntax( /// Validate that a model's query parses, raising its syntax error. #[pyfunction] #[pyo3(signature = (query_sql, model, parsing=(None, None)))] -fn validate_sql_syntax( +pub(crate) fn validate_sql_syntax( py: Python<'_>, query_sql: &str, model: (String, Bound<'_, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index d674450405..4c0390d152 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -661,7 +661,6 @@ native_ids! { ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleNativeSqlTests => "assemble_native_sql_tests", AssembleProject => "assemble_project", AttachChecks => "attach_checks", AttachResources => "attach_resources", @@ -692,7 +691,6 @@ native_ids! { BuildIntermediateSourceMap => "build_intermediate_source_map", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", - BuildModelInputs => "build_model_inputs", BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", BuildPlannerRelationsContext => "build_planner_relations_context", @@ -723,7 +721,6 @@ native_ids! { CheckCompileReuse => "check_compile_reuse", CheckCursorTypeConsistency => "check_cursor_type_consistency", CliStyle => "CliStyle", - CollectCompileDiagnostics => "collect_compile_diagnostics", CommandWarehouseScopeContext => "_command_warehouse_scope_context", CompareContracts => "compare_contracts", CompileDbtInteropProject => "compile_dbt_interop_project", @@ -767,14 +764,9 @@ native_ids! { DetectModelChanges => "detect_model_changes", DetectStaleWarningChanges => "detect_stale_warning_changes", DirectorySnapshotScope => "directory_snapshot_scope", - DiscoverEventExporterFunctions => "discover_event_exporter_functions", - DiscoverHookFunctions => "discover_hook_functions", - DiscoverMaterializationFiles => "discover_materialization_files", DiscoverModelFiles => "discover_model_files", DiscoverModelSchemaFiles => "discover_model_schema_files", DiscoverProjectInputs => "discover_project_inputs", - DiscoverPythonModuleFiles => "discover_python_module_files", - DiscoverPythonNodeFunctions => "discover_python_node_functions", DiscoverSchemaFiles => "discover_schema_files", DiscoverSourceFiles => "discover_source_files", DiscoverTestFiles => "discover_test_files", @@ -798,10 +790,7 @@ native_ids! { ExecuteDbtSqlbuildWork => "execute_dbt_sqlbuild_work", ExecuteDbtWithoutStateTracking => "execute_dbt_without_state_tracking", ExecutionEventWriter => "ExecutionEventWriter", - ExpandSqlMacros => "expand_sql_macros", - ExplicitReferenceReport => "explicit_reference_report", ExplicitReferencesHelp => "explicit_references_help", - ExternallyDescribedModels => "externally_described_models", FinalizeRuleFindingsRows => "finalize_rule_findings_rows", FindPythonStringLocations => "find_python_string_locations", FinishRuleRun => "finish_rule_run", @@ -840,7 +829,6 @@ native_ids! { IdenticalRenames => "identical_renames", InferFixtureColumnFacts => "infer_fixture_column_facts", InferNativeExpressionSourceShapes => "infer_native_expression_source_shapes", - InferNativeQueryColumnRows => "infer_native_query_column_rows", InferNativeQueryColumns => "infer_native_query_columns", InstallRulesSkills => "install_rules_skills", KnownDeclaredTypes => "known_declared_types", @@ -863,15 +851,8 @@ native_ids! { MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", - ModelHeaderFailure => "model_header_failure", - ModelOutputColumns => "model_output_columns", - ModelPlaceholders => "model_placeholders", ModelPythonHookNames => "model_python_hook_names", - NativeCompiledModel => "NativeCompiledModel", - NativeCompiledProject => "NativeCompiledProject", NativeDeclarationContexts => "NativeDeclarationContexts", - NativeModelConfigBuilder => "NativeModelConfigBuilder", - NativeModelInput => "NativeModelInput", NativeModelLoop => "NativeModelLoop", NativeModelValidator => "NativeModelValidator", NativeProgressProjector => "NativeProgressProjector", @@ -883,7 +864,6 @@ native_ids! { OldNameViewSkipReason => "old_name_view_skip_reason", ParseDbtLsJsonLines => "parse_dbt_ls_json_lines", ParseHeaderValues => "parse_header_values", - ParseModelHeaders => "parse_model_headers", ParseProjectSelector => "parse_project_selector", PlanColumnMigrations => "plan_column_migrations", PlanCompiledSqlTestArtifacts => "plan_compiled_sql_test_artifacts", @@ -900,10 +880,7 @@ native_ids! { PlanningProgressReporter => "PlanningProgressReporter", PreparedCompileArtifacts => "PreparedCompileArtifacts", ProjectWith => "project_with", - PromotionConflictDiagnostics => "promotion_conflict_diagnostics", - ProveNativeDynamicContracts => "prove_native_dynamic_contracts", PublicDeclarationIndexes => "public_declaration_indexes", - PublicModelSchemaIndex => "public_model_schema_index", PythonReadSourceNamesFor => "python_read_source_names_for", ReadCostRun => "read_cost_run", ReadLatestFingerprints => "read_latest_fingerprints", @@ -911,7 +888,6 @@ native_ids! { ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", RecordCompileInputReads => "record_compile_input_reads", - RecordSettingsClassRead => "record_settings_class_read", RecordTemplateReads => "record_template_reads", RejectUnitTestSelectors => "reject_unit_test_selectors", RejectUnparseableHeaderRewrites => "reject_unparseable_header_rewrites", @@ -927,10 +903,8 @@ native_ids! { RenderProjectVarText => "render_project_var_text", RenderSourceRelation => "render_source_relation", ReportCompileDiagnostic => "report_compile_diagnostic", - ResolveAnalysisModelNames => "resolve_analysis_model_names", ResolveCloneBoundary => "resolve_clone_boundary", ResolveCompileAnalysisSelection => "resolve_compile_analysis_selection", - ResolveConfigFilePath => "resolve_config_file_path", ResolveConfiguredRuleSelection => "resolve_configured_rule_selection", ResolveCostRun => "resolve_cost_run", ResolveDbtConfig => "resolve_dbt_config", @@ -1034,12 +1008,8 @@ native_ids! { ValidateCompileDbtModelNames => "validate_compile_dbt_model_names", ValidateCompileDbtModelReference => "validate_compile_dbt_model_reference", ValidateDbtInitProject => "validate_dbt_init_project", - ValidateHookSqlSyntax => "validate_hook_sql_syntax", - ValidateLocalSqlAnalysis => "validate_local_sql_analysis", ValidatePreviewTarget => "validate_preview_target", - ValidateResourceIdentity => "validate_resource_identity", ValidateSourceCursorInputColumns => "validate_source_cursor_input_columns", - ValidateSqlSyntax => "validate_sql_syntax", WithBindingDiagnostics => "with_binding_diagnostics", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 8e58c7ded7..5111817fb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -228,7 +228,7 @@ pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult(request: ExpansionRequest<'py>) -> PyResult> { +pub(crate) fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { let py: Python<'py> = request.sql.py(); let sql: Bound<'py, PyString> = request.sql.clone(); let Some(output) = macro_output(request)? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index bac9ffcbf3..eba68527af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -293,7 +293,7 @@ fn assembly<'py>( /// One assembly per test input, with the error its assembly raises, if any. #[pyfunction] #[pyo3(signature = (*, inputs))] -fn assemble_native_sql_tests<'py>( +pub(crate) fn assemble_native_sql_tests<'py>( py: Python<'py>, inputs: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index f36ea6e975..2ff1cb2cb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,6 +1,6 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; -mod compiled_tests; +pub(crate) mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 2f7cec3bcf..c66b211658 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -7,14 +7,17 @@ use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::sqltext::authored_values::optional_authored_value_to_python; use crate::bindings::types::CompilerDetach; -type ParsedModelHeader = ( +pub(crate) type ParsedModelHeader = ( Option>, Option>, Option, ); #[pyfunction] -fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { +pub(crate) fn parse_model_headers( + py: Python<'_>, + headers: Vec, +) -> PyResult> { let parsed = py .compiler_detach(|| { sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) @@ -72,7 +75,10 @@ pub(crate) fn location_rows( /// Simple top-level SELECT output names and their authored locations. #[pyfunction] -fn model_output_columns(contents: &str, implicit_aliases: bool) -> PyResult> { +pub(crate) fn model_output_columns( + contents: &str, + implicit_aliases: bool, +) -> PyResult> { compiler_guard(|| { Ok(location_rows( sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e2dc427940..4232b772cd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3907,6 +3907,7 @@ def is_empty_fixture_query(sql: str) -> bool: ... def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... def fixture_planning_work_info(*, reset: bool = False) -> tuple[int, int]: ... def planner_identity_work_info(*, reset: bool = False) -> tuple[int, dict[str, int]]: ... +def compile_stage_work_info(*, reset: bool = False) -> dict[str, int]: ... def migration_fingerprint_work_info(*, reset: bool = False) -> tuple[int, int]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None @@ -4284,9 +4285,6 @@ def declared_relations( dict[str, dict[str, InferredNullability]], dict[str, dict[str, str]], ]: ... -def externally_described_models( - inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] -) -> list[Path]: ... def analyze_resolved_column_reads( *, query_sql: str, references: Sequence[CompileSqlReference], dialect: str | None ) -> tuple[CompiledLineageSourceFact, ...]: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f618ca5b23..a28023fb1d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -826,6 +826,7 @@ class NativeAnalysisSeamTestCase: description: str expected_native_returns: dict[str, list[object]] + expected_stage_work: dict[str, int] @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f02bbb8997..1747b7ccf6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4807,10 +4807,8 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_module, "infer_native_expression_source_shapes"), - (native_module, "assemble_native_sql_tests"), (native_module, "complete_semantic_project"), (native_module, "evaluate_model_contracts"), - (native_module, "promotion_conflict_diagnostics"), (native_module, "build_project_column_lineage"), (plan_artifacts_module, "plan_compiled_sql_test_artifacts"), ) @@ -4853,6 +4851,7 @@ def seam_recording_compile( }.items(): patch.setenv(name, value) seams: dict[str, list[object]] = record_analysis_seams(monkeypatch=patch) + _ = native_module.compile_stage_work_info(reset=True) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) return run, seams diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 4feed479e4..79e9018240 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -11,7 +11,6 @@ from sqlbuild._native import ( ContractValidationResult, NativeSqlTestArtifact, - NativeSqlTestAssembly, ProjectColumnLineage, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -30,13 +29,15 @@ description="models_contracts_lineage_and_sql_tests", expected_native_returns={ "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], - "assemble_native_sql_tests": [ANY], "complete_semantic_project": [ANY], "evaluate_model_contracts": [ANY], - "promotion_conflict_diagnostics": [()], "build_project_column_lineage": [ANY], "plan_compiled_sql_test_artifacts": [ANY], }, + expected_stage_work={ + "assemble_native_sql_tests": 1, + "promotion_conflict_diagnostics": 1, + }, ) ], ids=lambda case: case.description, @@ -56,13 +57,13 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_run.returncode == 0, native_run.stderr assert native_run.compiled assert native_seams == test_case.expected_native_returns + stage_work: dict[str, int] = native_module.compile_stage_work_info() + assert { + stage: stage_work.get(stage, 0) for stage in test_case.expected_stage_work + } == test_case.expected_stage_work for seams in (native_seams,): assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) - assembled_tests: object = seams["assemble_native_sql_tests"][0] - assert isinstance(assembled_tests, tuple) - assert assembled_tests - assert all(isinstance(assembled, NativeSqlTestAssembly) for assembled in assembled_tests) planned_artifacts: object = seams["plan_compiled_sql_test_artifacts"][0] assert isinstance(planned_artifacts, tuple) assert planned_artifacts diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 95eefdc761..d0d62ffc72 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,11 +17,8 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module from sqlbuild._native import ( CustomHostObserver, - DeclarationScopeBuild, DiscoveredMacroFile, - DiscoveredProjectInputs, LoadedMacro, - SqlLexicalSyntax, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main @@ -803,7 +800,6 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -824,31 +820,13 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil ) macro_finding: str = f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - scope_builds: list[Path] = [] - original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope - - def counting_build( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, - ) -> DeclarationScopeBuild: - scope_builds.append(tmp_path) - return original_build( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - - monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) + _ = native_module.compile_stage_work_info(reset=True) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert macro_finding in cold assert warm == cold - assert len(scope_builds) == 1 + assert native_module.compile_stage_work_info()["build_declaration_scope"] == 1 @pytest.mark.parametrize( @@ -879,38 +857,21 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr "SELECT 1\n", encoding="utf-8", ) - scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] - original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope original_load: Callable[..., dict[str, LoadedMacro]] = macros_module.load_project_macros - def counting_build( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, - ) -> DeclarationScopeBuild: - scope_builds.append(tmp_path) - return original_build( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, LoadedMacro]: loaded: dict[str, LoadedMacro] = original_load(macro_files) macro_loads.append(loaded) return loaded - monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) + _ = native_module.compile_stage_work_info(reset=True) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" in cold - assert len(scope_builds) == 1 + assert native_module.compile_stage_work_info()["build_declaration_scope"] == 1 assert len(macro_loads) == 2 assert macro_loads[0]["row_limit"].function is not macro_loads[1]["row_limit"].function diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 38befd7710..2cce7c00d1 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -47,7 +47,7 @@ from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry if TYPE_CHECKING: - from sqlbuild._native import CompileInputsView, ModelInput, ProofRow + from sqlbuild._native import CompileInputsView, ModelInput LINEAGE_NATIVE: str = "native" @@ -395,32 +395,22 @@ def pivot_project_files() -> dict[str, str]: } -@dataclass +@dataclass(frozen=True) class NativePivotProofs: - """Every native pivot proof assembly took, and how many a finished session proved.""" + """How many standalone pivot proofs assembly ran, and how many a session proved.""" - proofs: list[DynamicColumnContractProof | None] = field(default_factory=list) - session_proofs: int = 0 + proofs: int + session_proofs: int -def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs: - """Record every batch of native pivot proofs assembly takes.""" +def native_pivot_proofs() -> NativePivotProofs: + """The standalone pivot proofs assembly ran since the counters were last reset.""" - recorded_proofs: NativePivotProofs = NativePivotProofs() - native_proofs: Callable[..., tuple[ProofRow | None, ...]] = ( - native_module.prove_native_dynamic_contracts + work: dict[str, int] = native_module.compile_stage_work_info(reset=True) + return NativePivotProofs( + proofs=work.get("pivot_proofs", 0), session_proofs=work.get("session_pivot_proofs", 0) ) - def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: - proofs: tuple[ProofRow | None, ...] = native_proofs(**keywords) - recorded_proofs.proofs.extend(map(contract_proof, proofs)) - proven: int = sum(proof is not None for proof in proofs) - recorded_proofs.session_proofs += proven * (keywords["session"] is not None) - return proofs - - monkeypatch.setattr(native_module, "prove_native_dynamic_contracts", recorded) - return recorded_proofs - def generated_analysis_files(*, rng: random.Random, model_count: int) -> dict[str, str]: """Sources (typed, inferred and untyped), a pivot and models drawing on every template.""" diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index ca5f413ed6..8bbe7dd270 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -402,29 +402,28 @@ def test_given_cyclic_project_when_compiling_then_native_output_is_complete_and_ ids=lambda case: case.description, ) def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_without_the_wheel( - test_case: StandalonePivotProofTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: StandalonePivotProofTestCase, tmp_path: Path ) -> None: inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=pivot_project_files() ) - recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) + _ = native_pivot_proofs() project: CompiledProjectView = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), analysis_model_names=test_case.analysed_models, ) + recorded: NativePivotProofs = native_pivot_proofs() assert ( - all(proof is not None for proof in recorded.proofs), - len(recorded.proofs), + recorded.proofs, recorded.session_proofs, { model.name: getattr(model.dynamic_column_contract, "output_proven", None) for model in project.models }, ) == ( - True, test_case.expected_native_proofs, test_case.expected_session_proofs, test_case.expected_proven_by_model, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index b076b00c34..4055ecad6e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -13,14 +13,6 @@ class RetainedRulesRowsTestCase: expected_codes: tuple[str, ...] -@dataclass(frozen=True) -class UnretainedRulesRowsTestCase: - """A compile whose model facts were not retained, so rules cannot read them.""" - - description: str - expected_error: str - - @dataclass(frozen=True) class ZeroMaterializationTestCase: """An ordinary compile that must not construct the legacy Python project representation.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 72f66866e1..e45320423a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -6,10 +6,9 @@ import shutil from collections import Counter from collections.abc import Callable, Iterator -from dataclasses import replace from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING import pytest @@ -46,7 +45,7 @@ from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: - from sqlbuild._native import CompileInputsView, ModelInput + from sqlbuild._native import CompileInputsView TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( @@ -83,30 +82,6 @@ def retained_only(project: CompiledProjectView) -> _native.NativeCompiledProject monkeypatch.setattr(module, "compiled_project_facts", retained_only) -def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: - """Run the native model loop into a throwaway project instead of the compile's own. - - The compile's own project still runs the loop over no models, so it keeps the loop that - attaches its other resources. - """ - - build_inputs: Callable[..., tuple[ModelInput, ...]] = cast( - "Callable[..., tuple[ModelInput, ...]]", _native.build_model_inputs - ) - - def discarded(**kwargs: Any) -> tuple[ModelInput, ...]: - discovered: DiscoveredProjectInputs = kwargs["discovered_inputs"] - without_models: dict[str, Any] = { - **kwargs, - "discovered_inputs": replace(discovered, model_files=()), - } - _ = build_inputs(**without_models) - throwaway: dict[str, Any] = {**kwargs, "native_project": _native.NativeCompiledProject()} - return build_inputs(**throwaway) - - monkeypatch.setattr(_native, "build_model_inputs", discarded) - - LEGACY_PROJECT_TYPES: tuple[str, ...] = ( "CompileProjectInputs", "CompileModelInput", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 2c9d07b14f..0dd0ca946b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -27,7 +27,6 @@ ModelPathScopeErrorTestCase, RetainedRulesRowsTestCase, SharedInferredColumnsTestCase, - UnretainedRulesRowsTestCase, ZeroMaterializationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -48,7 +47,6 @@ expected_summary, forbid_hand_built_projects, inputs_summary, - skip_model_retention, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import ORDERS_PROJECT_FILES, write_files @@ -122,32 +120,6 @@ def test_given_compiled_project_when_evaluating_rules_then_retained_model_facts_ assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes -@pytest.mark.parametrize( - "test_case", - [ - UnretainedRulesRowsTestCase( - description="models the compile-input stage did not retain", - expected_error="compiled model 'orders' has no retained compile facts", - ) - ], - ids=lambda case: case.description, -) -def test_given_unretained_model_facts_when_evaluating_rules_then_compile_reports_it( - test_case: UnretainedRulesRowsTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) - skip_model_retention(monkeypatch=monkeypatch) - - exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) - output: str = "".join(capsys.readouterr()) - - assert exit_code == 1 - assert test_case.expected_error in output - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/graph/helpers.py b/tests/integration/src/sqlbuild/compiler/graph/helpers.py index bb647d0aac..77d4d681fc 100644 --- a/tests/integration/src/sqlbuild/compiler/graph/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/graph/helpers.py @@ -4,10 +4,6 @@ from pathlib import Path -import pytest - -import sqlbuild._native as native_module - ORDERS_PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/staging/stg_orders.sql": ( @@ -30,23 +26,6 @@ def write_files(*, project_dir: Path, files: dict[str, str]) -> None: path.write_text(text, encoding="utf-8") -def record_compile_owners( - *, monkeypatch: pytest.MonkeyPatch -) -> list[native_module.NativeCompiledProject]: - """Record every native compiled project the compile creates, to read its graph builds.""" - - owners: list[native_module.NativeCompiledProject] = [] - owner_type: type[native_module.NativeCompiledProject] = native_module.NativeCompiledProject - - def recorded() -> native_module.NativeCompiledProject: - owner: native_module.NativeCompiledProject = owner_type() - owners.append(owner) - return owner - - monkeypatch.setattr(native_module, "NativeCompiledProject", recorded) - return owners - - def compiled_models(*, stdout: str) -> dict[str, bool]: """Whether the compile report lists each project model.""" diff --git a/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py index 3efb4be77f..416390bb84 100644 --- a/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py +++ b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py @@ -14,7 +14,6 @@ MODEL_NAMES, ORDERS_PROJECT_FILES, compiled_models, - record_compile_owners, write_files, ) @@ -53,12 +52,9 @@ def test_given_compile_selection_when_compiling_then_native_graph_selects_models test_case: NativeGraphSelectionTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: write_files(project_dir=tmp_path, files=ORDERS_PROJECT_FILES) - owners: list[native_module.NativeCompiledProject] = record_compile_owners( - monkeypatch=monkeypatch - ) + _ = native_module.compile_stage_work_info(reset=True) exit_code: int = main( ["--project-dir", str(tmp_path), "--no-color", "compile", "--no-cache", *test_case.select] @@ -68,7 +64,10 @@ def test_given_compile_selection_when_compiling_then_native_graph_selects_models assert compiled_models(stdout=capsys.readouterr().out) == { name: name in test_case.expected_models for name in sorted(MODEL_NAMES) } - assert sum(owner.graph_builds() for owner in owners) == test_case.expected_graph_builds + assert ( + native_module.compile_stage_work_info()["lineage_graph_builds"] + == test_case.expected_graph_builds + ) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index c835c7305b..953eb66629 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -10,13 +10,14 @@ from pathlib import Path from typing import TYPE_CHECKING, cast -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, CompiledSqlTest, NativeSqlTestAssembly, NativeSqlTestFailure, + build_compile_inputs, build_project_graph, discover_project_inputs, ) @@ -438,6 +439,14 @@ def _helper_mock_test(*, rng: random.Random, index: int) -> str: return f'TEST (name "helper_mock_{index}");\n\nWITH\n{comment}{ctes}\nSELECT 1\n' +_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, +) + + def write_project(*, project_dir: Path, files: dict[str, str]) -> None: """Write a project's files below `project_dir`.""" @@ -461,29 +470,28 @@ def _compiled_tests_and_diagnostics(*, project_dir: Path) -> tuple[object, objec return project.sql_tests, project.diagnostics -def record_native_assemblies(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count tests compile takes from the native assembly, with diagnostics, cases or failures. +def native_assembly_answers(*, project_dir: Path) -> Counter[str]: + """Count tests the native assembly answers for a project, with diagnostics, cases or failures. A test whose assembly raises counts under `failed_`. """ + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=_ADAPTER_CONTEXT, + run_id="integration_run", + ) + assemblies: tuple[NativeSqlTestAssembly, ...] = native_module.assemble_native_sql_tests( + inputs=inputs + ) answers: Counter[str] = Counter() - seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = native_module.assemble_native_sql_tests - - def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: - assemblies: tuple[NativeSqlTestAssembly, ...] = seam(inputs=inputs) - failures: list[NativeSqlTestFailure | None] = [item.failure for item in assemblies] - answers.update(f"failed_{failure.kind}" for failure in filter(None, failures)) - tests: list[CompiledSqlTest | None] = [item.test for item in assemblies] - native: list[CompiledSqlTest] = list(filter(None, tests)) - answers["native_assembled"] += len(native) - answers["native_with_diagnostics"] += sum(bool(item.diagnostics) for item in assemblies) - answers["native_case_fingerprints"] += sum( - test.case_fingerprint is not None for test in native - ) - return assemblies - - monkeypatch.setattr(native_module, "assemble_native_sql_tests", counted_assemblies) + failures: list[NativeSqlTestFailure | None] = [item.failure for item in assemblies] + answers.update(f"failed_{failure.kind}" for failure in filter(None, failures)) + tests: list[CompiledSqlTest | None] = [item.test for item in assemblies] + native: list[CompiledSqlTest] = list(filter(None, tests)) + answers["native_assembled"] += len(native) + answers["native_with_diagnostics"] += sum(bool(item.diagnostics) for item in assemblies) + answers["native_case_fingerprints"] += sum(test.case_fingerprint is not None for test in native) return answers diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py index a553eeefa3..ff8dbab312 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py @@ -13,6 +13,7 @@ import pytest +import sqlbuild._native as native_module from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -39,8 +40,8 @@ assembly_outcome, edge_sql_test_files, generated_sql_test_assembly_files, + native_assembly_answers, outcome_kind, - record_native_assemblies, write_project, ) @@ -72,10 +73,10 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_match_python( test_case: GeneratedSqlTestAssemblyParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - native_assemblies: Counter[str] = record_native_assemblies(monkeypatch=monkeypatch) + native_assemblies: Counter[str] = Counter() + _ = native_module.compile_stage_work_info(reset=True) entries: list[GoldenEntry] = [] outcomes: Counter[str] = Counter() for index in range(test_case.count): @@ -87,6 +88,7 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_ ), ) actual: PlanningCallOutcome = assembly_outcome(project_dir=project_dir) + native_assemblies.update(native_assembly_answers(project_dir=project_dir)) entries.append(assembly_golden_entry(outcome=actual, project_dir=project_dir)) outcomes[outcome_kind(actual)] += 1 @@ -95,7 +97,8 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_ read_golden(golden_name("sql_test_assembly", test_case.description)), entries ), dict(native_assemblies), - ) == ([], test_case.expected_native_assemblies), outcomes + native_module.compile_stage_work_info()["assemble_native_sql_tests"], + ) == ([], test_case.expected_native_assemblies, test_case.count), outcomes @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 1b4112bbc6..4f4499d076 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -871,12 +871,6 @@ class ExpectedCountTestCase: expected_count: int -@dataclass(frozen=True) -class ExpectedMessageTestCase: - description: str - expected_message: str - - @dataclass(frozen=True) class LoadTargetRetentionPoliciesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index bf6107ed94..8c45a1122d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -4,7 +4,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, @@ -21,7 +20,6 @@ DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, ExpectedCountTestCase, - ExpectedMessageTestCase, ModelHeaderColumnLocationTestCase, ModelOutputColumnLocationTestCase, ParseModelSqlErrorTestCase, @@ -69,29 +67,6 @@ def test_given_unique_model_headers_when_discovering_then_header_values_and_loca ) -@pytest.mark.parametrize( - "test_case", - [ - ExpectedMessageTestCase( - description="native worker error remains authoritative", - expected_message="MODEL header worker pool construction failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_native_pool_construction_error_when_preparing_headers_then_error_is_authoritative( - monkeypatch: pytest.MonkeyPatch, - test_case: ExpectedMessageTestCase, -) -> None: - def rejecting_tokenize(_headers: list[str]) -> object: - raise ValueError("MODEL header worker pool construction failed") - - monkeypatch.setattr(native_module, "parse_model_headers", rejecting_tokenize) - - with pytest.raises(ValueError, match=test_case.expected_message): - native_module.prepare_model_header_tokens(["pool_error_marker value"]) - - @pytest.mark.parametrize( "test_case", [ExpectedBooleanTestCase(description="cached maps remain fresh", expected_result=True)], From ed2dd2ecdda8f390c7bbec224a6bc9e46533bf96 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:33:46 +0100 Subject: [PATCH 05/11] refactor(compile): read declared relations and known declarations natively during analysis setup assembly_analysis builds the declared relations, known function names and known declared types with direct Rust calls. The declared_relations, known_function_names and known_declared_types pyfunctions had no Python callers and are removed with their stubs and NativeAttr ids. --- .../compile_driver/assembly_analysis.rs | 32 +++++++++++-------- .../project_assembly/declared_relations.rs | 15 ++++----- .../_helpers/project_assembly/registration.rs | 3 +- .../src/bindings/_helpers/records/ids.rs | 3 -- .../_helpers/semantic_shapes/declarations.rs | 29 +---------------- .../_helpers/semantic_shapes/registration.rs | 5 ++- src/sqlbuild/_native.pyi | 11 ------- 7 files changed, 28 insertions(+), 70 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index f9b1c6463f..fcf2ed9a4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -18,8 +18,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::project_assembly::declared_relations::{ + DeclaredRelations, declared_relations, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; @@ -78,11 +82,17 @@ pub(crate) fn analysis_context<'py>( .getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let relations: Bound<'py, PyAny> = - registry::native(py, NativeAttr::DeclaredRelations)?.call1((inputs,))?; - let column_types: Bound<'py, PyAny> = relations.get_item(0)?; - let nullability: Bound<'py, PyAny> = relations.get_item(1)?; - let complete_schemas: Bound<'py, PyAny> = relations.get_item(2)?; + let (column_types, nullability, complete_schemas): DeclaredRelations<'py> = + declared_relations(inputs.clone())?; + let (column_types, nullability, complete_schemas): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = ( + column_types.into_any(), + nullability.into_any(), + complete_schemas.into_any(), + ); let dynamic_families: Bound<'py, PyAny> = dynamic_families_by_table(inputs)?.into_any(); let profile: Bound<'py, PyAny> = profile_with_catalog(inputs, request.inference_profile.bind(py), &column_types)?; @@ -141,15 +151,9 @@ pub(crate) fn declared_names<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = inputs.py(); let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; - let names: Bound<'py, PyAny> = - registry::native(py, NativeAttr::KnownFunctionNames)?.call1((functions.clone(),))?; - let types: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::KnownDeclaredTypes)?, - &[ - ("functions", functions), - ("column_types", column_types.clone()), - ], - )?; + let names: Bound<'py, PyAny> = PyTuple::new(py, function_names(&functions)?)?.into_any(); + let types: Bound<'py, PyAny> = + PyTuple::new(py, declared_types(&functions, column_types)?)?.into_any(); Ok((names, types)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index fda2adb7e8..e68586513e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -1,8 +1,7 @@ //! What each model, seed, source and table function declares about its columns before analysis. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_input_view::{ @@ -22,7 +21,8 @@ const NON_NULL_MEMBER: &str = "NON_NULL"; const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// The three per-relation declaration maps the analysis starts from. -type DeclaredRelations<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); +pub(crate) type DeclaredRelations<'py> = + (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. fn column_types<'py>( @@ -73,8 +73,9 @@ fn table_function_name(function: &Bound<'_, PyAny>) -> PyResult { } /// Declared column types, declared nullability and the authoritative binding schemas. -#[pyfunction] -fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn declared_relations<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = inputs.py(); let nullability: Bound<'py, PyAny> = registry::class(py, RecordId::InferredNullability)?; let types: Bound<'py, PyDict> = PyDict::new(py); @@ -146,7 +147,3 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(declared_relations, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 4b6c8aa7b5..160f660c9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::project_assembly::models::{ NativeModelFacts, NativeProjectFacts, NativeProjectResources, NativeResourceFailure, }; use crate::bindings::_helpers::project_assembly::{ - declared_relations, native_resources, resources, syntax_checks, syntax_validation, + native_resources, resources, syntax_checks, syntax_validation, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -14,7 +14,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - declared_relations::register(module)?; native_resources::register(module)?; syntax_validation::register(module)?; syntax_checks::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 4c0390d152..3155ffe9b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -758,7 +758,6 @@ native_ids! { DbtRunner => "DbtRunner", DebugDbtFromProject => "debug_dbt_from_project", DeclarationRows => "declaration_rows", - DeclaredRelations => "declared_relations", DescribeConnectionWarehouse => "describe_connection_warehouse", DetectChanges => "detect_changes", DetectModelChanges => "detect_model_changes", @@ -831,8 +830,6 @@ native_ids! { InferNativeExpressionSourceShapes => "infer_native_expression_source_shapes", InferNativeQueryColumns => "infer_native_query_columns", InstallRulesSkills => "install_rules_skills", - KnownDeclaredTypes => "known_declared_types", - KnownFunctionNames => "known_function_names", LaunchRecordOffered => "_launch_record_offered", LaunchRecordWithheld => "_launch_record_withheld", ListCostRuns => "list_cost_runs", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs index ba956e8688..a47736e73d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs @@ -2,11 +2,7 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::boundary::keyword_request::Request; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::bindings::_helpers::records::fields::field as native_field; const UNKNOWN: &str = "UNKNOWN"; @@ -50,26 +46,3 @@ pub(crate) fn declared_types( types.remove(UNKNOWN); Ok(types.into_iter().collect()) } - -/// `known_function_names` as a sorted tuple. -#[pyfunction] -fn known_function_names<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { - PyTuple::new(functions.py(), function_names(functions)?) -} - -/// `known_declared_types(*, functions, column_types)` as a sorted tuple. -#[pyfunction] -#[pyo3(signature = (**request))] -fn known_declared_types<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("known_declared_types", request)?; - let types: Vec = - declared_types(&request.get("functions")?, &request.get("column_types")?)?; - PyTuple::new(request.py(), types) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(known_function_names, module)?)?; - module.add_function(wrap_pyfunction!(known_declared_types, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs index 8d3deb3d24..5116b8f3a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs @@ -2,11 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_shapes::{coverage, declarations, shapes, source_bindings}; +use crate::bindings::_helpers::semantic_shapes::{coverage, shapes, source_bindings}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { shapes::register(module)?; coverage::register(module)?; - source_bindings::register(module)?; - declarations::register(module) + source_bindings::register(module) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4232b772cd..e521b1fc00 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4278,13 +4278,6 @@ def validate_sql_syntax( parsing: tuple[dict[str, str] | None, str | None] = (None, None), ) -> None: ... def hook_sql_statements(value: object) -> list[str]: ... -def declared_relations( - inputs: CompileInputsView, -) -> tuple[ - dict[str, dict[str, str]], - dict[str, dict[str, InferredNullability]], - dict[str, dict[str, str]], -]: ... def analyze_resolved_column_reads( *, query_sql: str, references: Sequence[CompileSqlReference], dialect: str | None ) -> tuple[CompiledLineageSourceFact, ...]: ... @@ -5402,10 +5395,6 @@ def semantic_coverage(project: Any, /) -> dict[str, tuple[str, ...]]: ... def source_binding_diagnostics( *, project: Any, columns: Mapping[str, Any], profile: Any, selected_keys: frozenset[Any] ) -> tuple[Any, ...]: ... -def known_function_names(functions: Sequence[Any], /) -> tuple[str, ...]: ... -def known_declared_types( - *, functions: Sequence[Any], column_types: Mapping[str, Mapping[str, str]] -) -> tuple[str, ...]: ... def complete_semantic_project( *, project: Any, From b22e39c05f5a1e7ab71e356f56507b06b5ef5aa1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:35:44 +0100 Subject: [PATCH 06/11] refactor(compile): run model analysis, expression shapes and semantic completion as direct calls Assembly calls the three analysis stages in Rust. Stage tests take the stage inputs from the test-support assembly_stage_inputs(stage, ...), which assembles like assemble_project and stops before the stage, and read stage counts from compile_stage_work_info (including expression shape batches). Removes the invariant-only test that replaced model analysis with an empty result. --- .../analysis_session/stage_functions.rs | 42 ++++--- .../_helpers/compile_driver/assembly.rs | 111 +++++++++++++++-- .../compile_driver/assembly_analysis.rs | 43 ++++--- .../_helpers/compile_driver/assembly_facts.rs | 24 ++-- .../compile_driver/assembly_project.rs | 64 ++++++---- .../_helpers/compile_driver/stage_work.rs | 5 + .../src/bindings/_helpers/records/ids.rs | 1 - .../_helpers/semantic_checks/stage.rs | 46 +++++-- src/sqlbuild/_native.pyi | 10 ++ .../cli/commands/main/compile/helpers.py | 1 - .../compile/test_native_analysis_seams.py | 9 +- .../compiler/analysis_session/helpers.py | 112 +++++++++--------- .../test_native_model_analysis.py | 3 - .../src/sqlbuild/compiler/pipeline/helpers.py | 33 +----- .../pipeline/test_expression_catalog_cache.py | 17 +-- .../compiler/semantic_checks/helpers.py | 32 ++--- .../test_native_metadata_checks.py | 2 - .../test_native_semantic_checks.py | 8 -- 18 files changed, 345 insertions(+), 218 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index ae3dfe9365..30dd166a28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -20,10 +20,13 @@ use crate::bindings::_helpers::analysis_session::session::{ infer_expression_source_shapes, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compile_driver::stage_work::{ + EXPRESSION_SHAPE_BATCHES, EXPRESSION_SHAPES_INFERRED, count_stage, +}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::records::fields::field as native_field; const BATCH_MODE: &str = "batch"; const TABLE_FUNCTION_KIND: &str = "table_fn"; @@ -105,9 +108,16 @@ fn infer_native_expression_source_shapes<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("infer_native_expression_source_shapes", request)?; - let py: Python<'py> = request.py(); - let expressions: Bound<'py, PyAny> = request.get("expressions")?; - let profile: Bound<'py, PyAny> = request.get("profile")?; + expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) +} + +/// One shape per expression, cached on `profile`'s binding catalog; a failure raises. +pub(crate) fn expression_source_shapes<'py>( + expressions: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = profile.py(); + let profile: Bound<'py, PyAny> = profile.clone(); let dialect: String = aliased_dialect(py, profile_dialect(&profile)?)?; let catalog: Bound<'py, PyAny> = profile_catalog(&profile, &dialect)?; let cache: Bound<'py, PyAny> = native_field(&catalog, "expression_shapes")?; @@ -119,6 +129,8 @@ fn infer_native_expression_source_shapes<'py>( } } if !pending.is_empty() { + count_stage(py, EXPRESSION_SHAPE_BATCHES, 1)?; + count_stage(py, EXPRESSION_SHAPES_INFERRED, pending.len() as u64)?; let (rules, adapter_rules) = nullability_rules(&profile)?; let native: PyRef<'py, ProjectCatalog> = native_field(&catalog, "native")?.extract()?; let pending_names: Vec = pending @@ -203,10 +215,7 @@ pub(crate) fn dynamic_contract_proofs<'py>( ( dialect_or_generic(&native_field(&tables, "dialect")?)?, shape_rows(&native_field(&tables, "column_types_by_table")?)?, - shape_rows(&native_field( - &tables, - "authoritative_column_types_by_table", - )?)?, + shape_rows(&native_field(&tables, "authoritative_column_types_by_table")?)?, shape_rows(&native_field(&tables, "column_nullability_by_table")?)?, families, rows, @@ -264,16 +273,13 @@ pub(crate) fn query_column_rows( let dialect: String = aliased_dialect(py, normalization_dialect.clone())?; let mut batch: bool = false; for query in &queries { - if native_field(query, "mode")?.eq(BATCH_MODE)? { + if native_field(&query, "mode")?.eq(BATCH_MODE)? { batch = true; break; } } let catalog: Option> = if batch { - Some(native_field( - &profile_catalog(profile, &dialect)?, - "native", - )?) + Some(native_field(&profile_catalog(profile, &dialect)?, "native")?) } else { None }; @@ -301,7 +307,7 @@ pub(crate) fn query_column_rows( } fn query_row(query: &Bound<'_, PyAny>) -> PyResult { - let placeholders: Bound<'_, PyAny> = native_field(query, "placeholders")?; + let placeholders: Bound<'_, PyAny> = native_field(&query, "placeholders")?; let placeholders: Pairs = if placeholders.is_truthy()? { placeholders .call_method0("items")? @@ -313,7 +319,7 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { }; let mut analysis_names: Vec = Vec::new(); let mut lineage: Vec<(String, String, String)> = Vec::new(); - for reference in native_field(query, "references")?.try_iter()? { + for reference in native_field(&query, "references")?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; let kind: String = native_field(&reference, "ref_kind")?.extract()?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -338,12 +344,12 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { } } Ok(( - native_field(query, "sql")?.extract()?, + native_field(&query, "sql")?.extract()?, placeholders, analysis_names, lineage, - native_field(query, "recover_cte_facts")?.extract()?, - native_field(query, "mode")?.extract()?, + native_field(&query, "recover_cte_facts")?.extract()?, + native_field(&query, "mode")?.extract()?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index f304cbf65d..2e0f513cd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -1,5 +1,8 @@ //! Planner-ready project assembly: diagnostics collection, the macro bridge and explicit references. +use std::sync::{Arc, Mutex, PoisonError}; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -17,6 +20,11 @@ use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +const STAGE_REACHED: &str = "assembly stopped before the requested stage"; +const EXPRESSION_SHAPES_STAGE: &str = "expression_shapes"; +const MODEL_FACTS_STAGE: &str = "model_facts"; +const SEMANTIC_COMPLETION_STAGE: &str = "semantic_completion"; + const OPTIONAL: [&str; 5] = [ "inference_profile", "skip_column_inference", @@ -33,6 +41,53 @@ pub(crate) struct AssemblyRequest { pub(crate) column_lineage_mode: Py, pub(crate) analysis_cache_dir: Py, pub(crate) analysis_model_names: Py, + /// The stage to stop before, keeping its inputs, when assembly runs for stage tests. + pub(crate) stop: Option, +} + +/// An assembly stage whose inputs tests can take instead of running it. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum AssemblyStage { + ExpressionShapes, + ModelFacts, + SemanticCompletion, +} + +/// Where an assembly stopped for a stage test keeps that stage's inputs. +pub(crate) struct StageStop { + stage: AssemblyStage, + inputs: Arc>>>, +} + +impl AssemblyRequest { + /// Stop with `inputs` before `stage` when the request stops there; else continue. + pub(crate) fn before_stage<'py>( + &self, + stage: AssemblyStage, + inputs: impl FnOnce() -> PyResult>, + ) -> PyResult<()> { + let Some(stop) = self.stop.as_ref().filter(|stop| stop.stage == stage) else { + return Ok(()); + }; + *stop.inputs.lock().unwrap_or_else(PoisonError::into_inner) = Some(inputs()?.unbind()); + Err(PyRuntimeError::new_err(STAGE_REACHED)) + } +} + +/// The assembly request of `arguments`, stopping at `stop` when given. +fn assembly_request( + arguments: &KeywordArguments<'_>, + stop: Option, +) -> PyResult { + Ok(AssemblyRequest { + inputs: arguments.value("inputs")?.unbind(), + inference_profile: arguments.value("inference_profile")?.unbind(), + skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), + column_lineage_mode: lineage_mode(arguments)?.unbind(), + analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), + analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + stop, + }) } /// Convert compile inputs into the planner-ready project view. @@ -44,14 +99,53 @@ pub(crate) fn assemble_project<'py>( ) -> PyResult> { let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "assemble_project", options, (&["inputs"], &OPTIONAL))?; - let request: AssemblyRequest = AssemblyRequest { - inputs: arguments.value("inputs")?.unbind(), - inference_profile: arguments.value("inference_profile")?.unbind(), - skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), - column_lineage_mode: lineage_mode(&arguments)?.unbind(), - analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), - analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + collected_project(py, assembly_request(&arguments, None)?) +} + +/// Test support: assemble like `assemble_project` but stop before `stage`, returning its inputs or None. +#[pyfunction] +#[pyo3(signature = (stage, **options))] +pub(crate) fn assembly_stage_inputs<'py>( + py: Python<'py>, + stage: &str, + options: Option>, +) -> PyResult>> { + let stage: AssemblyStage = match stage { + EXPRESSION_SHAPES_STAGE => AssemblyStage::ExpressionShapes, + MODEL_FACTS_STAGE => AssemblyStage::ModelFacts, + SEMANTIC_COMPLETION_STAGE => AssemblyStage::SemanticCompletion, + _ => { + return Err(PyValueError::new_err(format!( + "unknown assembly stage '{stage}'" + ))); + } }; + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "assembly_stage_inputs", + options, + (&["inputs"], &OPTIONAL), + )?; + let inputs: Arc>>> = Arc::new(Mutex::new(None)); + let stop: StageStop = StageStop { + stage, + inputs: Arc::clone(&inputs), + }; + let assembled: PyResult> = + collected_project(py, assembly_request(&arguments, Some(stop))?); + let captured: Option> = inputs.lock().unwrap_or_else(PoisonError::into_inner).take(); + match (captured, assembled) { + (Some(captured), _) => Ok(Some(captured.into_bound(py))), + (None, Ok(_)) => Ok(None), + (None, Err(error)) => Err(error), + } +} + +/// `request`'s project, with diagnostics collected while assembling appended. +fn collected_project<'py>( + py: Python<'py>, + request: AssemblyRequest, +) -> PyResult> { let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { @@ -155,5 +249,6 @@ fn with_native_project<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_project, module)?) + module.add_function(wrap_pyfunction!(assemble_project, module)?)?; + module.add_function(wrap_pyfunction!(assembly_stage_inputs, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index fcf2ed9a4c..0682fcd9ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -1,7 +1,7 @@ //! Project assembly's inference profile, binding catalog, source shapes and model analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; @@ -9,19 +9,23 @@ use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignor use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; +use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; -use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::compile_timings::{ record_compile_cpu_timing, record_compile_timing, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::stage_work::{ + EXPRESSION_SHAPES, MODEL_ANALYSIS, count_stage, +}; use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, declared_relations, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; @@ -106,7 +110,7 @@ pub(crate) fn analysis_context<'py>( analyses: None, }; if context.sql_analysis_enabled.is_truthy()? { - record_expression_shapes(inputs, &context)?; + record_expression_shapes(request, &context)?; } let catalog: Bound<'py, PyAny> = context.profile.getattr("binding_catalog")?; catalog @@ -259,10 +263,11 @@ fn function_return_types<'py>( /// Record each expression source's analysed shape as its declared relation. fn record_expression_shapes<'py>( - inputs: &Bound<'py, PyAny>, + request: &AssemblyRequest, context: &AnalysisContext<'py>, ) -> PyResult<()> { - let py: Python<'py> = inputs.py(); + let py: Python<'py> = context.profile.py(); + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let mut sources: Vec> = Vec::new(); let mut expressions: Vec> = Vec::new(); for source_input in inputs.getattr("source_inputs")?.try_iter()? { @@ -275,15 +280,20 @@ fn record_expression_shapes<'py>( expressions.push(expression); } } - let shapes: Vec> = call_with( - ®istry::native(py, NativeAttr::InferNativeExpressionSourceShapes)?, - &[ - ("expressions", PyTuple::new(py, expressions)?.into_any()), - ("profile", context.profile.clone()), - ], - )? - .try_iter()? - .collect::>()?; + let expressions: Bound<'py, PyAny> = PyTuple::new(py, expressions)?.into_any(); + request.before_stage(AssemblyStage::ExpressionShapes, || { + keywords( + py, + &[ + ("expressions", expressions.clone()), + ("profile", context.profile.clone()), + ], + ) + })?; + count_stage(py, EXPRESSION_SHAPES, 1)?; + let shapes: Vec> = expression_source_shapes(&expressions, &context.profile)? + .iter() + .collect(); if shapes.len() != sources.len() { let relation: &str = if shapes.len() < sources.len() { "shorter" @@ -362,6 +372,7 @@ fn model_analyses<'py>( model_inputs.push(model_input); } } + count_stage(py, MODEL_ANALYSIS, 1)?; analyze_models(&ModelAnalysisRequest { model_inputs, profile: context.profile.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index c56cb968ef..fe15167773 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -84,14 +84,10 @@ pub(crate) fn native_project_facts<'py>( .into_any()) } -/// Each standalone pivot model's dynamic contract proof, by model position. -fn pivot_proofs<'py>( - pivots: &Bound<'py, PyDict>, - context: &AnalysisContext<'py>, - session: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = pivots.py(); - let tables: Bound<'py, PyAny> = call_with( +/// The declared relation facts standalone pivot proofs read. +pub(crate) fn pivot_tables<'py>(context: &AnalysisContext<'py>) -> PyResult> { + let py: Python<'py> = context.profile.py(); + call_with( ®istry::class(py, RecordId::NativePivotTables)?, &[ ("dialect", context.profile.getattr("sql_analysis_dialect")?), @@ -106,7 +102,17 @@ fn pivot_proofs<'py>( context.dynamic_families.clone(), ), ], - )?; + ) +} + +/// Each standalone pivot model's dynamic contract proof, by model position. +fn pivot_proofs<'py>( + pivots: &Bound<'py, PyDict>, + context: &AnalysisContext<'py>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let tables: Bound<'py, PyAny> = pivot_tables(context)?; + let py: Python<'py> = pivots.py(); let proven: Vec> = dynamic_contract_proofs( session, &tables, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 1922d48a6c..dc1e79bc0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -5,17 +5,20 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTu use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; -use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, analysis_context, and_not, }; use crate::bindings::_helpers::compile_driver::assembly_checks::compiled_sql_scenarios; -use crate::bindings::_helpers::compile_driver::assembly_facts::native_project_facts; +use crate::bindings::_helpers::compile_driver::assembly_facts::{ + native_project_facts, pivot_tables, +}; use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::stage_work::{SEMANTIC_COMPLETION, count_stage}; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_effective_adapter_name, resolve_effective_scenario_config, @@ -23,10 +26,13 @@ use crate::bindings::_helpers::contracts::spec_resolution::{ use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; +use crate::bindings::_helpers::semantic_checks::stage::{ + SemanticCompletion, completed_semantic_project, +}; use pyo3::IntoPyObjectExt; const SOURCE_SLOT: &str = "source"; @@ -87,6 +93,16 @@ pub(crate) fn assembled_project<'py>( || py.None().into_bound(py), |session| session.clone().into_any(), ); + request.before_stage(AssemblyStage::ModelFacts, || { + keywords( + py, + &[ + ("profile", context.profile.clone()), + ("session", native_session.clone()), + ("tables", pivot_tables(&context)?), + ], + ) + })?; let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native.getattr("resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; @@ -102,22 +118,30 @@ pub(crate) fn assembled_project<'py>( )?; let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; let binding_results: Bound<'py, PyDict> = binding_rows(py, context.analyses.as_ref())?; - call_with( - ®istry::native(py, NativeAttr::CompleteSemanticProject)?, - &[ - ("project", project), - ("profile", context.profile.clone()), - ( - "resource_sql_analysis", - and_not( - &context.sql_analysis_enabled, - &inputs.getattr("no_sql_validation")?, - )?, - ), - ("session", native_session), - ("binding_results", binding_results.into_any()), - ], - ) + let resource_sql_analysis: Bound<'py, PyAny> = and_not( + &context.sql_analysis_enabled, + &inputs.getattr("no_sql_validation")?, + )?; + request.before_stage(AssemblyStage::SemanticCompletion, || { + keywords( + py, + &[ + ("project", project.clone()), + ("profile", context.profile.clone()), + ("resource_sql_analysis", resource_sql_analysis.clone()), + ("session", native_session.clone()), + ("binding_results", binding_results.clone().into_any()), + ], + ) + })?; + count_stage(py, SEMANTIC_COMPLETION, 1)?; + completed_semantic_project(SemanticCompletion { + project, + profile: context.profile.clone(), + binding_results: binding_results.into_any(), + resource_sql_analysis: resource_sql_analysis.extract()?, + session: Some(native_session), + }) } /// `value.strip()` when `value` is a string with non-blank text, else `None`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 502b80cded..3aa6ac0c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -6,6 +6,11 @@ use pyo3::prelude::{Py, PyRefMut, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::{pyclass, pyfunction}; +pub(crate) const EXPRESSION_SHAPES: &str = "infer_native_expression_source_shapes"; +pub(crate) const EXPRESSION_SHAPE_BATCHES: &str = "expression_shape_batches"; +pub(crate) const EXPRESSION_SHAPES_INFERRED: &str = "expression_shapes_inferred"; +pub(crate) const MODEL_ANALYSIS: &str = "analyze_native_model_sql"; +pub(crate) const SEMANTIC_COMPLETION: &str = "complete_semantic_project"; pub(crate) const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; pub(crate) const PROMOTION_CONFLICTS: &str = "promotion_conflict_diagnostics"; pub(crate) const PIVOT_PROOFS: &str = "pivot_proofs"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 3155ffe9b2..3d10f4a29c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -732,7 +732,6 @@ native_ids! { CompiledCodeIdentity => "compiled_code_identity", CompiledProjectFacts => "compiled_project_facts", CompiledProjectRecord => "compiled_project_record", - CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", ConnectionProgressReporter => "ConnectionProgressReporter", ContentDigest => "content_digest", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index b944fb5140..b89ec1db02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -29,17 +29,41 @@ fn complete_semantic_project<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("complete_semantic_project", request)?; - let project: Bound<'py, PyAny> = request.get("project")?; - let profile: Bound<'py, PyAny> = request.get("profile")?; - let binding_results: Bound<'py, PyAny> = request.get("binding_results")?; - let resource_sql_analysis: bool = request.get("resource_sql_analysis")?.extract()?; - let session: Option> = - match request.optional("session")? { - Some(session) if !session.is_none() => Some(session.extract()?), - _ => None, - }; + completed_semantic_project(SemanticCompletion { + project: request.get("project")?, + profile: request.get("profile")?, + binding_results: request.get("binding_results")?, + resource_sql_analysis: request.get("resource_sql_analysis")?.extract()?, + session: request.optional("session")?, + }) +} + +/// The semantic completion stage's inputs. +pub(crate) struct SemanticCompletion<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) profile: Bound<'py, PyAny>, + pub(crate) binding_results: Bound<'py, PyAny>, + pub(crate) resource_sql_analysis: bool, + pub(crate) session: Option>, +} + +/// The completed project; an internal failure raises `NativeCompilerError`. +pub(crate) fn completed_semantic_project<'py>( + stage: SemanticCompletion<'py>, +) -> PyResult> { + let SemanticCompletion { + project, + profile, + binding_results, + resource_sql_analysis, + session, + } = stage; + let session: Option> = match session { + Some(session) if !session.is_none() => Some(session.extract()?), + _ => None, + }; let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; - let classes = StageClasses::new(request.py())?; + let classes = StageClasses::new(project.py())?; let shared = StageShared { catalog: &catalog.inner, session: session.as_deref(), @@ -59,7 +83,7 @@ fn complete_semantic_project<'py>( &recovered, &[( "diagnostics", - &PyTuple::new(request.py(), diagnostics)?.into_any(), + &PyTuple::new(project.py(), diagnostics)?.into_any(), )], )?; completed_diagnostics(&classes, &combined, &shared) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e521b1fc00..ca768e90bf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4082,6 +4082,16 @@ def assemble_project( analysis_cache_dir: Path | None = None, analysis_model_names: frozenset[str] | None = None, ) -> CompiledProjectView: ... +def assembly_stage_inputs( + stage: Literal["expression_shapes", "model_facts", "semantic_completion"], + *, + inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile | None = None, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + analysis_cache_dir: Path | None = None, + analysis_model_names: frozenset[str] | None = None, +) -> dict[str, Any] | None: ... def build_project_graph_with_analysis_selection( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 1747b7ccf6..3a97fbda99 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4807,7 +4807,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_module, "infer_native_expression_source_shapes"), - (native_module, "complete_semantic_project"), (native_module, "evaluate_model_contracts"), (native_module, "build_project_column_lineage"), (plan_artifacts_module, "plan_compiled_sql_test_artifacts"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 79e9018240..7ca329ad8d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -28,13 +28,15 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_native_returns={ - "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], - "complete_semantic_project": [ANY], + "infer_native_expression_source_shapes": [ANY, ANY, ANY], "evaluate_model_contracts": [ANY], "build_project_column_lineage": [ANY], "plan_compiled_sql_test_artifacts": [ANY], }, expected_stage_work={ + "infer_native_expression_source_shapes": 1, + "analyze_native_model_sql": 1, + "complete_semantic_project": 1, "assemble_native_sql_tests": 1, "promotion_conflict_diagnostics": 1, }, @@ -68,9 +70,6 @@ def test_given_project_when_compiling_then_native_seams_answer( assert isinstance(planned_artifacts, tuple) assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) - completed: object = seams["complete_semantic_project"][0] - assert isinstance(completed, native_module.NativeProjectView) - assert any(model.inferred_columns for model in completed.models) assert isinstance(seams["evaluate_model_contracts"][0], ContractValidationResult) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 2cce7c00d1..a4f2c1993f 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -10,7 +10,7 @@ from itertools import chain from operator import attrgetter from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Literal, cast import pytest @@ -538,6 +538,9 @@ class NativeAnalysisRuns: sessions: list[Any] = field(default_factory=list) +type AssemblyStage = Literal["expression_shapes", "model_facts", "semantic_completion"] + + def analyse_natively( *, inputs: CompileInputsView, @@ -545,7 +548,6 @@ def analyse_natively( lineage_mode: ColumnLineageMode, runs: NativeAnalysisRuns, cache_root: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: """Assemble `inputs` with a cold, then a warm analysis cache, then uncached, and compare them. @@ -555,67 +557,67 @@ def analyse_natively( """ views: list[object] = [] - sinks: list[NativeAnalysisRuns] = [runs] - profiles: list[ExpressionInferenceProfile] = [] - proved: Callable[..., tuple[ProofRow | None, ...]] = ( - native_module.prove_native_dynamic_contracts - ) def recorded_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile - ) -> tuple[dict[str, str] | None, ...]: + sink: NativeAnalysisRuns, + *, + expressions: tuple[str, ...], + profile: ExpressionInferenceProfile, + ) -> None: catalog: Any = cast(Any, profile.binding_catalog) native: tuple[dict[str, str] | None, ...] = infer_native_expression_source_shapes( expressions=expressions, profile=profile ) - profiles.append(profile) - sinks[-1].expression_shapes += sum(shape is not None for shape in native) - sinks[-1].golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) - return native - - def recorded_analyses(**keywords: Any) -> tuple[ProofRow | None, ...]: - for profile in profiles[-1:]: - _record_analyses( - inputs=inputs, - catalog=profile.binding_catalog, - proof_inputs=(keywords["tables"], keywords["session"]), - views=views, - runs=sinks[-1], - ) - profiles.clear() - return proved(**keywords) - - completed: Callable[..., CompiledProjectView] = native_module.complete_semantic_project - - def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: - sinks[-1].golden.extend( + sink.expression_shapes += sum(shape is not None for shape in native) + sink.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) + + def recorded_analyses( + sink: NativeAnalysisRuns, + *, + profile: ExpressionInferenceProfile, + session: Any, + tables: NativePivotTables, + ) -> None: + _record_analyses( + inputs=inputs, + catalog=profile.binding_catalog, + proof_inputs=(tables, session), + views=views, + runs=sink, + ) + + def recorded_proofs( + sink: NativeAnalysisRuns, *, project: CompiledProjectView, **keywords: Any + ) -> CompiledProjectView: + sink.golden.extend( golden_entry("proof", model.dynamic_column_contract) for model in project.models ) - with monkeypatch.context() as completing: - completing.setattr( - native_module, - "infer_native_expression_source_shapes", - infer_native_expression_source_shapes, - ) - return completed(project=project, **keywords) - - with monkeypatch.context() as patch: - patch.setattr(native_module, "complete_semantic_project", recorded_proofs) - patch.setattr(native_module, "infer_native_expression_source_shapes", recorded_shapes) - patch.setattr(native_module, "prove_native_dynamic_contracts", recorded_analyses) - for cache_dir, sink in ( - (cache_root, NativeAnalysisRuns()), - (cache_root, NativeAnalysisRuns()), - (None, runs), - ): - sinks.append(sink) - with suppress(CompileInputError): - _ = native_module.assemble_project( - inputs=inputs, - inference_profile=replace(inference_profile), - column_lineage_mode=lineage_mode, - analysis_cache_dir=cache_dir, - ) + return native_module.complete_semantic_project(project=project, **keywords) + + def stage_inputs(stage: AssemblyStage, cache_dir: Path | None) -> list[dict[str, Any]]: + found: dict[str, Any] | None = native_module.assembly_stage_inputs( + stage, + inputs=inputs, + inference_profile=replace(inference_profile), + column_lineage_mode=lineage_mode, + analysis_cache_dir=cache_dir, + ) + return list(filter(None, [found])) + + for cache_dir, sink in ( + (cache_root, NativeAnalysisRuns()), + (cache_root, NativeAnalysisRuns()), + (None, runs), + ): + with suppress(CompileInputError): + shapes: list[dict[str, Any]] = stage_inputs("expression_shapes", cache_dir) + _ = [recorded_shapes(sink, **found) for found in shapes] + facts: list[dict[str, Any]] = stage_inputs("model_facts", cache_dir) + _ = [recorded_analyses(sink, **found) for found in facts[: len(shapes)]] + _ = [ + recorded_proofs(sink, **found) + for found in stage_inputs("semantic_completion", cache_dir) + ] for index, view in enumerate(views[:-1]): _append(runs, f"model analyses, cached run {index}", views[-1], view) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 8bbe7dd270..33ddbde8b1 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -199,7 +199,6 @@ def test_given_generated_projects_when_analysing_natively_then_cached_and_proven lineage_mode=test_case.lineage_mode, runs=runs, cache_root=tmp_path / f"cache_{index}_{profile_index}", - monkeypatch=monkeypatch, ) assert ( @@ -254,7 +253,6 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache lineage_mode=ColumnLineageMode.FAST, runs=runs, cache_root=tmp_path / f"cache_{dialect}", - monkeypatch=monkeypatch, ) for dialect in test_case.dialects ] @@ -312,7 +310,6 @@ def test_given_cyclic_models_when_analysing_natively_then_session_answers_and_ca lineage_mode=lineage_mode, runs=runs, cache_root=tmp_path / f"cache_{dialect}_{lineage_mode.value}", - monkeypatch=monkeypatch, ) for dialect, lineage_mode in product(test_case.dialects, test_case.lineage_modes) ] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 9487e17ec9..e5f8e00ff5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -1,13 +1,10 @@ from __future__ import annotations -import functools import json from collections.abc import Callable from pathlib import Path from typing import TYPE_CHECKING, Any -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( CollectionRendering, @@ -24,7 +21,6 @@ semantic_shapes, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView @@ -215,32 +211,11 @@ def apply_warehouse_setup_sql(*, project_dir: Path, statements: tuple[str, ...]) connection.close() -def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ...]]: - """Record each shape request's expressions the binding catalog's shape cache misses.""" - - batches: list[tuple[str, ...]] = [] - original: Callable[..., Any] = native_module.infer_native_expression_source_shapes - monkeypatch.setattr( - native_module, - "infer_native_expression_source_shapes", - functools.partial(_traced_shapes, original=original, batches=batches), - ) - return batches - +def shape_batches(*, reset: bool = False) -> tuple[int, int]: + """Native shape batches the binding catalog's shape cache missed, and expressions they held.""" -def _traced_shapes( - *, - original: Callable[..., Any], - batches: list[tuple[str, ...]], - expressions: tuple[str, ...], - profile: ExpressionInferenceProfile, -) -> tuple[dict[str, str] | None, ...]: - cache: dict[str, object] = getattr(profile.binding_catalog, "expression_shapes", {}) - pending: tuple[str, ...] = tuple( - dict.fromkeys(filter(lambda item: item not in cache, expressions)) - ) - batches.extend((pending,) * bool(pending)) - return original(expressions=expressions, profile=profile) + work: dict[str, int] = native_module.compile_stage_work_info(reset=reset) + return work.get("expression_shape_batches", 0), work.get("expression_shapes_inferred", 0) _OFFLINE_POSTGRES_CONNECTION: dict[str, object] = { diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index 314244e82d..5326d7e785 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -18,7 +18,7 @@ ExpressionMemoCase, RestoredShapeCase, ) -from tests.integration.src.sqlbuild.compiler.pipeline.helpers import traced_shape_batches +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import shape_batches @pytest.mark.parametrize( @@ -30,7 +30,6 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva test_case: ExpressionMemoCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -44,17 +43,17 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva source.write_text( f"sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.expression}\n - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.expression}\n" ) - calls: list[tuple[str, ...]] = traced_shape_batches(monkeypatch) + _ = shape_batches(reset=True) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] assert main(args) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert payload["diagnostics"] == [] - assert len(calls) == test_case.expected_analysis_calls + assert shape_batches()[0] == test_case.expected_analysis_calls source.write_text(source.read_text().replace("AS id", "AS quantity")) assert main(args) == 1 changed: dict[str, Any] = json.loads(capsys.readouterr().out) assert [item["code"] for item in changed["diagnostics"]] == ["B002"] - assert len(calls) == test_case.expected_analysis_calls * 2 + assert shape_batches()[0] == test_case.expected_analysis_calls * 2 @pytest.mark.parametrize( @@ -73,7 +72,6 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on test_case: ExpressionBatchCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -88,11 +86,14 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on f" - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.orders_expression}\n" f" - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.customers_expression}\n" ) - batches: list[tuple[str, ...]] = traced_shape_batches(monkeypatch) + _ = shape_batches(reset=True) assert main(["--project-dir", str(tmp_path), "compile", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert payload["diagnostics"] == [] - assert tuple(batches) == test_case.expected_batches + assert shape_batches() == ( + len(test_case.expected_batches), + sum(len(batch) for batch in test_case.expected_batches), + ) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 54f83d4884..a0488f1fa2 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -10,19 +10,16 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( CompilerDiagnostic, - build_project_graph, - discover_project_inputs, project_with, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( + compile_inputs, generated_analysis_files, ) @@ -188,29 +185,16 @@ def captured_semantic_inputs( *, project_dir: Path, files: dict[str, str], - monkeypatch: pytest.MonkeyPatch, ) -> SemanticInputs: """Compile and keep the semantic completion stage's inputs.""" - for relative_path, contents in files.items(): - path: Path = project_dir / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - _ = path.write_text(contents, encoding="utf-8") - captured: list[SemanticInputs] = [] - - complete: Callable[..., CompiledProjectView] = native_module.complete_semantic_project - - def capture(**arguments: Any) -> CompiledProjectView: - captured.append(SemanticInputs(**arguments)) - return complete(**arguments) - - with monkeypatch.context() as patch: - patch.setattr(native_module, "complete_semantic_project", capture) - _ = build_project_graph( - discovered_inputs=discover_project_inputs(project_dir=project_dir), - adapter=DuckDbAdapter(), - ) - return captured[0] + stage: dict[str, Any] | None = native_module.assembly_stage_inputs( + "semantic_completion", + inputs=compile_inputs(project_dir=project_dir, files=files), + inference_profile=DuckDbAdapter().expression_inference_profile(), + ) + assert stage is not None + return SemanticInputs(**stage) def with_dialect(inputs: SemanticInputs, dialect: str | None) -> SemanticInputs: diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py index dd452c37c5..de7023ba50 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py @@ -68,7 +68,6 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_recorded_python( test_case: GeneratedMetadataParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) families: Counter[str] = Counter() @@ -82,7 +81,6 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r rng=rng, non_ascii=test_case.non_ascii, ), - monkeypatch=monkeypatch, ) views: dict[str, object] = { str(dialect): semantic_outcome( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 37a7d81982..1f2c4f7dcb 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -77,7 +77,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_recorded_python( test_case: GeneratedSemanticParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) statuses: Counter[str] = Counter() @@ -90,7 +89,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ files=generated_semantic_files( rng=rng, model_count=test_case.model_count, require_analysis=index % 2 == 0 ), - monkeypatch=monkeypatch, ) views: dict[str, object] = {} for dialect in test_case.dialects: @@ -156,7 +154,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_recorded_python( test_case: FormerlyDeferredSemanticTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) golden: list[GoldenEntry] = [] @@ -169,7 +166,6 @@ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_ captured: SemanticInputs = captured_semantic_inputs( project_dir=project_dir, files=files_with_non_ascii(files=files, rng=rng, non_ascii=test_case.non_ascii), - monkeypatch=monkeypatch, ) views: dict[str, object] = {} for dialect in test_case.dialects: @@ -206,12 +202,10 @@ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_ def test_given_project_without_catalog_when_completing_then_raises_native_compiler_error( test_case: MissingCatalogTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: captured: SemanticInputs = captured_semantic_inputs( project_dir=tmp_path / "project", files=generated_semantic_files(rng=random.Random(5), model_count=8, require_analysis=False), - monkeypatch=monkeypatch, ) with pytest.raises(native_module.NativeCompilerError) as raised: @@ -253,7 +247,6 @@ def test_given_project_without_catalog_when_completing_then_raises_native_compil def test_given_compiles_when_completing_from_the_session_then_matches_payload_and_python( test_case: SessionCompletionTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) captured: list[SemanticInputs] = [ @@ -262,7 +255,6 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an files=session_corpus_files( corpus=test_case.corpus, rng=rng, model_count=test_case.model_count ), - monkeypatch=monkeypatch, ) for index in range(test_case.count) ] From 0d2fe7cd9878366ab7b0a9c9c7c812b92a39b354 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:07:59 +0100 Subject: [PATCH 07/11] refactor(compile): call the compile driver stages and project discovery directly from commands compile_driver analysis, pipeline, command and phases call lineage, graph selection, unit-test selector checks, adapter-name resolution, contracts, rules, connection config, manifest and reuse-store stages in Rust, and every command calls discover_project_inputs / build_compiled_project directly. Tests that replaced those module attributes now use stage counters (project discoveries, lineage, contracts), a failing provider module, adapter hooks on a manual clock, or a PreparedCompileArtifacts subclass; the compile case of the injected rich-lineage panic test is dropped because compile no longer reads the module attribute. --- .../_helpers/build_command/defer_clone.rs | 131 +++++++--------- .../_helpers/build_command/invocation.rs | 17 +- .../bindings/_helpers/clone/clone_command.rs | 8 +- .../bindings/_helpers/clone/clone_pipeline.rs | 56 +++---- .../_helpers/command_runtime/invocation.rs | 8 +- .../sql_expansion_context.rs | 23 +-- .../_helpers/compile_driver/analysis.rs | 139 ++++++++-------- .../_helpers/compile_driver/command.rs | 53 ++++--- .../_helpers/compile_driver/discovery.rs | 2 + .../_helpers/compile_driver/input_reads.rs | 4 +- .../_helpers/compile_driver/phases.rs | 22 +-- .../_helpers/compile_driver/pipeline.rs | 99 ++++++------ .../_helpers/compile_driver/project.rs | 2 +- .../compile_driver/selection_phases.rs | 4 +- .../_helpers/compile_driver/stage_work.rs | 3 + .../_helpers/contract_command/command.rs | 84 +++++----- .../_helpers/contract_command/write.rs | 50 +++--- .../contracts/contract_diagnostics.rs | 2 +- .../bindings/_helpers/cost_command/refresh.rs | 35 ++--- .../bindings/_helpers/dbt_interop/pipeline.rs | 10 +- .../_helpers/dbt_interop/pipeline_prologue.rs | 73 ++++----- .../_helpers/debug_command/run_debug.rs | 11 +- .../bindings/_helpers/diagnostics/trackers.rs | 5 + .../bindings/_helpers/diff_report/command.rs | 8 +- .../_helpers/execution_history/sqlite.rs | 26 ++- .../_helpers/format_command/command.rs | 19 ++- .../_helpers/format_command/selection.rs | 19 ++- .../_helpers/lineage/lineage_command.rs | 3 +- .../_helpers/lineage/project_lineage.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 8 +- .../bindings/_helpers/plan_command/plan.rs | 17 +- .../bindings/_helpers/plan_command/rules.rs | 8 +- .../_helpers/query_command/project.rs | 8 +- .../src/bindings/_helpers/records/ids.rs | 8 - .../bindings/_helpers/rules/fix_session.rs | 148 +++++++----------- .../bindings/_helpers/rules/lint_expansion.rs | 23 +-- .../_helpers/scenario_command/setup.rs | 8 +- .../bindings/_helpers/scopes/scope_loading.rs | 8 +- .../fast_lineage/test_native_fast_lineage.py | 14 -- .../cli/commands/main/compile/helpers.py | 41 ++--- .../compile/test_native_analysis_seams.py | 8 +- .../commands/_helpers/cost/test_refresh.py | 5 +- .../main/compile/test_compile_command.py | 23 +-- .../commands/main/execution/_test_types.py | 1 + .../cli/commands/main/execution/helpers.py | 28 ++-- .../execution/test_build_process_resources.py | 15 +- .../main/execution/test_scenario_command.py | 10 +- .../compiler/pipeline/main/helpers.py | 30 +++- .../main/test_compile_phase_timings.py | 31 +--- .../unit/src/sqlbuild/lint/test_cli_format.py | 5 +- .../src/sqlbuild/lint/test_expanded_lint.py | 26 +-- 51 files changed, 672 insertions(+), 719 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index a4914d0697..ece5d008ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -13,12 +13,13 @@ use sqlbuild_core::build_command::errors::BuildCommandError; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const VIEW_MATERIALIZATION: &str = "view"; const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; @@ -41,9 +42,7 @@ fn user_error(py: Python<'_>, message: String, code: &str) -> PyErr { } fn resource_type(key: &Bound<'_, PyAny>) -> PyResult { - native_field(key, "resource_type")? - .getattr("value")? - .extract() + native_field(&key, "resource_type")?.getattr("value")?.extract() } /// Functions and view models are recreated over cloned boundaries instead of cloned. @@ -53,7 +52,7 @@ fn recreated_over_clones(scope: &Bound<'_, PyAny>, key: &Bound<'_, PyAny>) -> Py } let model: Bound<'_, PyAny> = scope .getattr("models_by_name")? - .call_method1("get", (native_field(key, "name")?,))?; + .call_method1("get", (native_field(&key, "name")?,))?; if model.is_none() { return Ok(false); } @@ -109,8 +108,8 @@ fn traversed_names<'py>( let keys: Bound<'py, PyAny> = call_with( ®istry::native(py, function)?, &[ - ("selected", native_field(scope, "selected_keys")?), - ("upstream", native_field(scope, "upstream_deps")?), + ("selected", native_field(&scope, "selected_keys")?), + ("upstream", native_field(&scope, "upstream_deps")?), ("is_clonable", is_clonable.clone()), ("is_view", is_view.clone()), ], @@ -125,9 +124,9 @@ fn traversed_names<'py>( /// Selected models except those explicitly disabled. fn selected_executable_model_count(scope: &Bound<'_, PyAny>) -> PyResult { - let models: Bound<'_, PyAny> = native_field(scope, "models_by_name")?; + let models: Bound<'_, PyAny> = native_field(&scope, "models_by_name")?; let mut count: usize = 0; - for key in native_field(scope, "selected_keys")?.try_iter()? { + for key in native_field(&scope, "selected_keys")?.try_iter()? { let key: Bound<'_, PyAny> = key?; if resource_type(&key)? != MODEL_TYPE { continue; @@ -219,9 +218,8 @@ fn clone_input<'py>( on_item: Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = context.py(); - let project: Bound<'py, PyAny> = native_field(pipeline, "destination_project")?; - let destination_models: Bound<'py, PyAny> = - native_field(pipeline, "destination_model_entries")?; + let project: Bound<'py, PyAny> = native_field(&pipeline, "destination_project")?; + let destination_models: Bound<'py, PyAny> = native_field(&pipeline, "destination_model_entries")?; let target: Bound<'py, PyAny> = native( py, NativeAttr::ResolveTargetConfig, @@ -230,10 +228,7 @@ fn clone_input<'py>( "project_config", native_field(&prephase.discovered, "project_config")?, ), - ( - "local_config", - native_field(&prephase.discovered, "local_config")?, - ), + ("local_config", native_field(&prephase.discovered, "local_config")?), ("target_name", prephase.destination.clone()), ], )?; @@ -245,10 +240,7 @@ fn clone_input<'py>( &host::entry(py, ExecutorHost::BuildDestinationRetentionRequests)?, &[ ("project", project.clone()), - ( - "adapter_name", - native_field(&prephase.adapter, "adapter_name")?, - ), + ("adapter_name", native_field(&prephase.adapter, "adapter_name")?), ( "selected_model_names", PyFrozenSet::new(py, &selected)?.into_any(), @@ -265,10 +257,10 @@ fn clone_input<'py>( let sources: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::CloneSourceEntries)?, &[ - ("origin", native_field(pipeline, "origin_source_entries")?), + ("origin", native_field(&pipeline, "origin_source_entries")?), ( "destination", - native_field(pipeline, "destination_source_entries")?, + native_field(&pipeline, "destination_source_entries")?, ), ], )?; @@ -278,24 +270,24 @@ fn clone_input<'py>( ("source_entries", sources), ( "origin_model_entries", - native_field(pipeline, "origin_model_entries")?, + native_field(&pipeline, "origin_model_entries")?, ), ("destination_model_entries", destination_models), ( "origin_seed_entries", - native_field(pipeline, "origin_seed_entries")?, + native_field(&pipeline, "origin_seed_entries")?, ), ( "destination_seed_entries", - native_field(pipeline, "destination_seed_entries")?, + native_field(&pipeline, "destination_seed_entries")?, ), ( "destination_function_entries", - native_field(pipeline, "destination_function_entries")?, + native_field(&pipeline, "destination_function_entries")?, ), ( "execution_order", - native_field(pipeline, "clone_plan")?.getattr("execution_order")?, + native_field(&pipeline, "clone_plan")?.getattr("execution_order")?, ), ("adapter", prephase.adapter.clone()), ("destination_connection", connection.clone()), @@ -386,12 +378,9 @@ fn executed_clone<'py>( fn failed_items(result: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = result.py(); let status: Bound<'_, PyAny> = host::entry(py, ExecutorHost::CloneStatus)?; - let (failed, warning) = ( - native_field(&status, "FAILED")?, - native_field(&status, "WARNING")?, - ); + let (failed, warning) = (native_field(&status, "FAILED")?, native_field(&status, "WARNING")?); let mut items: Vec = Vec::new(); - for item in native_field(result, "item_results")?.try_iter()? { + for item in native_field(&result, "item_results")?.try_iter()? { let item: Bound<'_, PyAny> = item?; let item_status: Bound<'_, PyAny> = native_field(&item, "status")?; if !(item_status.eq(&failed)? || item_status.eq(&warning)?) { @@ -438,10 +427,7 @@ fn clone_boundaries<'py>( call_with( ®istry::class(py, RecordId::ClonePipelineConnection)?, &[ - ( - "config", - native_field(&context.invocation, "connection_config")?, - ), + ("config", native_field(&context.invocation, "connection_config")?), ("handle", connection.clone()), ], )?, @@ -451,13 +437,10 @@ fn clone_boundaries<'py>( call_with( ®istry::class(py, RecordId::ClonePipelineOptions)?, &[ - ( - "no_sql_validation", - native_field(request, "no_sql_validation")?, - ), - ("no_cache", native_field(request, "no_cache")?), + ("no_sql_validation", native_field(&request, "no_sql_validation")?), + ("no_cache", native_field(&request, "no_cache")?), ("select", PyTuple::new(py, &prephase.select)?.into_any()), - ("cli_vars", native_field(request, "cli_vars")?), + ("cli_vars", native_field(&request, "cli_vars")?), ], )?, ), @@ -567,13 +550,13 @@ pub(crate) fn run_defer_clone_boundary_prephase( ) -> PyResult<()> { let py: Python<'_> = context.py(); let (request, invocation) = (&context.request, &context.invocation); - let discovered: Bound<'_, PyAny> = native_field(invocation, "discovered_inputs")?; + let discovered: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")?; let split: Bound<'_, PyAny> = native( py, NativeAttr::SplitUnitTestSelectors, &[ - ("select", native_field(request, "select")?), - ("exclude", native_field(request, "exclude")?), + ("select", native_field(&request, "select")?), + ("exclude", native_field(&request, "exclude")?), ("discovered_inputs", discovered.clone()), ], )?; @@ -584,32 +567,31 @@ pub(crate) fn run_defer_clone_boundary_prephase( { return Ok(()); } - let adapter: Bound<'_, PyAny> = native_field(invocation, "adapter")?; - let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; - let project: Bound<'_, PyAny> = native( + let adapter: Bound<'_, PyAny> = native_field(&invocation, "adapter")?; + let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let project: Bound<'_, PyAny> = build_compiled_project( py, - NativeAttr::BuildCompiledProject, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", native_field(request, "selected_target")?), - ( - "no_sql_validation", - native_field(request, "no_sql_validation")?, - ), - ( - "analysis_selection", - call_with( - ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(request, "no_cache")?)], - )?, - ), - ("cli_vars", native_field(request, "cli_vars")?), - ( - "external_sql_reference_resolver", - external_resolver(&project_dir, &discovered)?, - ), - ], + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", native_field(&request, "selected_target")?), + ("no_sql_validation", native_field(&request, "no_sql_validation")?), + ( + "analysis_selection", + call_with( + ®istry::class(py, RecordId::CompileAnalysisSelection)?, + &[("no_cache", native_field(&request, "no_cache")?)], + )?, + ), + ("cli_vars", native_field(&request, "cli_vars")?), + ( + "external_sql_reference_resolver", + external_resolver(&project_dir, &discovered)?, + ), + ], + )?), )?; let scope: Bound<'_, PyAny> = native( py, @@ -620,7 +602,7 @@ pub(crate) fn run_defer_clone_boundary_prephase( ("exclude", native_field(&split, "exclude")?), ( "auto_load_sources", - native_field(invocation, "should_load_sources")?, + native_field(&invocation, "should_load_sources")?, ), ], )?; @@ -631,11 +613,8 @@ pub(crate) fn run_defer_clone_boundary_prephase( NativeAttr::EnforceModelExecutionLimit, &[ ("model_count", model_count.into_pyobject(py)?.into_any()), - ( - "target_name", - native_field(invocation, "effective_target_name")?, - ), - ("limits", native_field(invocation, "execution_limits")?), + ("target_name", native_field(&invocation, "effective_target_name")?), + ("limits", native_field(&invocation, "execution_limits")?), ( "refusal_note", EXECUTION_LIMIT_BUILD_NOTE.into_bound_py_any(py)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 89a6712f1f..d7756831c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -10,6 +10,8 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; @@ -105,12 +107,15 @@ pub(crate) fn resolve_build_context<'py>( let project_dir: Bound<'py, PyAny> = or_none(native_field(request, "project_dir")?, || { imported(py, "pathlib", "Path")?.call_method0("cwd") })?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", boolean(py, false)), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", boolean(py, false)), + ], + )?), )?; let selected_target: Bound<'py, PyAny> = native_field(request, "selected_target")?; let target_name: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 1ddcfdbf89..0861ce3202 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -24,6 +24,8 @@ use crate::bindings::_helpers::clone::clone_execution::{ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -132,9 +134,9 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = py.import("sys")?.getattr("stdout")?; let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; progress.call_method1("start", ("Discovering project...",))?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; progress.call_method1("complete", ("Discovered project.",))?; call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 4f2c067f60..10356cbd2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -5,11 +5,13 @@ use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; /// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. const PLANNED_FIELDS: [&str; 8] = [ @@ -39,30 +41,30 @@ fn compiled_environment<'py>( ) -> PyResult> { let py: Python<'py> = target_name.py(); let options: &Bound<'py, PyAny> = &pipeline.options; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", pipeline.discovered.clone()), - ("adapter", pipeline.adapter.clone()), - ("selected_target", target_name.clone()), - ( - "resolved_connection", - native_field(&pipeline.connection, "config")?, - ), - ( - "no_sql_validation", - native_field(options, "no_sql_validation")?, - ), - ( - "analysis_selection", - call_with( - ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(options, "no_cache")?)], - )?, - ), - ("cli_vars", native_field(options, "cli_vars")?), - ("external_sql_reference_resolver", pipeline.resolver.clone()), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", pipeline.discovered.clone()), + ("adapter", pipeline.adapter.clone()), + ("selected_target", target_name.clone()), + ( + "resolved_connection", + native_field(&pipeline.connection, "config")?, + ), + ("no_sql_validation", native_field(&options, "no_sql_validation")?), + ( + "analysis_selection", + call_with( + ®istry::class(py, RecordId::CompileAnalysisSelection)?, + &[("no_cache", native_field(&options, "no_cache")?)], + )?, + ), + ("cli_vars", native_field(&options, "cli_vars")?), + ("external_sql_reference_resolver", pipeline.resolver.clone()), + ], + )?), ) } @@ -74,8 +76,8 @@ fn read_target_name<'py>( let py: Python<'py> = target_name.py(); let deferred: Bound<'py, PyAny> = resolve_target_config( py, - &native_field(discovered, "project_config")?, - &native_field(discovered, "local_config")?, + &native_field(&discovered, "project_config")?, + &native_field(&discovered, "local_config")?, target_name, )? .getattr("defer_sources_to")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index e8b1dee75f..a43dcb0451 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -4,6 +4,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; @@ -73,9 +75,9 @@ fn invocation_context<'py>( } else { requested }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( &discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 7c4ed3d9c3..0788d4989d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -6,6 +6,8 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_vars; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; @@ -156,15 +158,18 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult discovered, - None => call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", sources.project_dir.clone()), - ( - "sql_analysis_enabled_override", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + None => discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", sources.project_dir.clone()), + ( + "sql_analysis_enabled_override", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?), )?, }; let project_config: Bound<'_, PyAny> = native_field(&discovered, "project_config")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index cb6fbaf5bf..faf5cce965 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -7,16 +7,27 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMetho use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::compile_driver::stage_work::{ + COLUMN_LINEAGE, MODEL_CONTRACTS, count_stage, +}; +use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; +use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; const RICH_LINEAGE: &str = "rich"; @@ -46,11 +57,16 @@ fn compile_lineage<'py>( } else { return Ok(py.None().into_bound(py)); }; - registry::native(py, NativeAttr::BuildProjectColumnLineage)?.call1(( - graph.getattr("project")?, - dialect, - column_lineage_mode(py, member)?, - py.None(), + count_stage(py, COLUMN_LINEAGE, 1)?; + Ok(build_project_column_lineage( + &graph.getattr("project")?, + Some(dialect), + &column_lineage_mode(py, member)?, + None, + )? + .map_or_else( + || py.None().into_bound(py), + |lineage| lineage.into_bound(py), )) } @@ -186,14 +202,8 @@ fn analyzed<'py>( } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; - let selected_keys: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::SelectProjectGraph)?, - &[ - ("graph", graph.clone()), - ("select", driver.select.clone()), - ("exclude", driver.exclude.clone()), - ], - )?; + let selected_keys: Bound<'py, PyAny> = + select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( py, project @@ -283,22 +293,18 @@ impl<'py> Driver<'_, 'py> { } else { py.None().into_bound(py) }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", self.options.project_dir.clone()), - ("extract_output_column_locations", false_value), - ("sql_analysis_enabled_override", override_analysis), - ], - )?; - call_with( - ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, - &[ - ("select", self.select.clone()), - ("exclude", self.exclude.clone()), - ("discovered_inputs", discovered.clone()), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", self.options.project_dir.clone()), + ("extract_output_column_locations", false_value), + ("sql_analysis_enabled_override", override_analysis), + ], + )?), )?; + reject_unit_test_selectors(&self.select, &self.exclude, &discovered)?; let discover_ms: u64 = elapsed_ms(started); self.status .complete(&format!("Discovered project. ({}s)", seconds(discover_ms)))?; @@ -307,12 +313,9 @@ impl<'py> Driver<'_, 'py> { /// The adapter the project and local configuration select. fn adapter(&self, discovered: &Bound<'py, PyAny>) -> PyResult> { - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(self.py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, )?; resolve_cli_adapter(&adapter_name, &self.options.project_dir) } @@ -354,21 +357,24 @@ impl<'py> Driver<'_, 'py> { }, )?; let graph: Bound<'py, PyAny> = within(&lifecycle, |_| { - call_with( - ®istry::native(py, NativeAttr::BuildProjectGraphWithAnalysisSelection)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", options.selected_target.clone()), - ("no_sql_validation", options.no_sql_validation.clone()), - ( - "skip_column_inference", - options.profile_flags.getattr("skip_column_inference")?, - ), - ("column_lineage_mode", lineage_mode), - ("cli_vars", options.cli_vars.clone()), - ("analysis_selection", analysis_selection), - ], + build_project_graph_with_analysis_selection( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.selected_target.clone()), + ("no_sql_validation", options.no_sql_validation.clone()), + ( + "skip_column_inference", + options.profile_flags.getattr("skip_column_inference")?, + ), + ("column_lineage_mode", lineage_mode), + ("cli_vars", options.cli_vars.clone()), + ("analysis_selection", analysis_selection), + ], + )?), ) })?; let graph_ms: u64 = elapsed_ms(started); @@ -417,10 +423,8 @@ impl<'py> Driver<'_, 'py> { )? } else { self.status.start("Validating model contracts...")?; - call_with( - ®istry::native(py, NativeAttr::EvaluateModelContracts)?, - &[("project", project.clone()), ("dialect", dialect.clone())], - )? + count_stage(py, MODEL_CONTRACTS, 1)?; + evaluate_model_contracts(py, project, Some(dialect))? }; let wall_ms: u64 = elapsed_ms(started); let cpu_ended: f64 = thread_time.call0()?.extract()?; @@ -476,18 +480,21 @@ impl<'py> Driver<'_, 'py> { } else { py.None().into_bound(py) }; - let result: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::EvaluateRules)?, - &[ - ("graph", graph.clone()), - ("discovered_inputs", discovered.clone()), - ("config", rules_config), - ("project_dir", project_dir.clone()), - ("dialect", lint_dialect.clone()), - ("selected_keys", selected), - ("early_lint", self.early_lint.clone().into_any()), - ("no_sql_analysis", options.no_sql_validation.clone()), - ], + let result: Bound<'py, PyAny> = evaluate_rules( + py, + Some(keywords( + py, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", rules_config), + ("project_dir", project_dir.clone()), + ("dialect", lint_dialect.clone()), + ("selected_keys", selected), + ("early_lint", self.early_lint.clone().into_any()), + ("no_sql_analysis", options.no_sql_validation.clone()), + ], + )?), )?; let ended: f64 = monotonic.call0()?.extract()?; let progress: String = rules_progress(ended - started, &result, false)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 7e1e5c63e4..92ffc2298a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; +use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_compile; use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::compile_timings::{ CompileTimingsHandle, collect_compile_timings, timing_milliseconds, }; +use crate::bindings::_helpers::compile_driver::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::phases::{ elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, target_dir, write_artifacts, write_dag_artifact, @@ -151,7 +153,7 @@ fn recorded_compile<'py>( let compiled: PyResult<(CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = collect_compile_timings(|detailed_timings| { within( - ®istry::native(py, NativeAttr::RecordCompileInputReads)?.call0()?, + &Bound::new(py, record_compile_input_reads())?.into_any(), |input_reads| { let artifact_writes: Py = new_artifact_recorder(py)?; let manager: Bound<'py, PyAny> = call_with( @@ -192,29 +194,32 @@ fn recorded_compile<'py>( }); close_status(status)?; let (outcome, input_reads, artifact_writes) = compiled?; - let stored: PyResult> = call_with( - ®istry::native(py, NativeAttr::WriteReusableCompile)?, - &[ - ("attempt", attempt.clone()), - ("output", output.clone()), - ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), - ("input_reads", input_reads), - ("artifact_writes", artifact_writes), - ( - "compile_cache_enabled", - PyBool::new(py, outcome.compile_cache_enabled) - .to_owned() - .into_any(), - ), - ( - "artifacts_written", - PyBool::new(py, outcome.artifacts_written) - .to_owned() - .into_any(), - ), - ("dag_artifact_path", outcome.dag_artifact_path), - ("json_output", request.getattr("json_output")?), - ], + let stored: PyResult<()> = write_reusable_compile( + py, + Some(keywords( + py, + &[ + ("attempt", attempt.clone()), + ("output", output.clone()), + ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), + ("input_reads", input_reads), + ("artifact_writes", artifact_writes), + ( + "compile_cache_enabled", + PyBool::new(py, outcome.compile_cache_enabled) + .to_owned() + .into_any(), + ), + ( + "artifacts_written", + PyBool::new(py, outcome.artifacts_written) + .to_owned() + .into_any(), + ), + ("dag_artifact_path", outcome.dag_artifact_path), + ("json_output", request.getattr("json_output")?), + ], + )?), ); let printed: PyResult> = output.call_method0("print_stdout"); printed?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index f48b6de73b..81a1634eb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::compile_driver::integration_loaders::integration_ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, within, }; +use crate::bindings::_helpers::compile_driver::stage_work::{PROJECT_DISCOVERIES, count_stage}; use crate::bindings::_helpers::contracts::project_config::{ load_local_config_file, load_project_config_file, }; @@ -203,6 +204,7 @@ pub(crate) fn discover_project_inputs<'py>( ], ), )?; + count_stage(py, PROJECT_DISCOVERIES, 1)?; let request: DiscoveryRequest = DiscoveryRequest { project_dir: arguments.value("project_dir")?.unbind(), sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs index ca98271b0a..cc7bc8a336 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs @@ -78,7 +78,7 @@ pub(crate) fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> P /// Records environment variables and run-identity reads while entered. #[pyclass(frozen, module = "sqlbuild._native")] -struct CompileInputRecording { +pub(crate) struct CompileInputRecording { recorder: Mutex>>, } @@ -111,7 +111,7 @@ impl CompileInputRecording { /// Record template input reads from every thread until the returned block exits. #[pyfunction] -fn record_compile_input_reads() -> CompileInputRecording { +pub(crate) fn record_compile_input_reads() -> CompileInputRecording { CompileInputRecording { recorder: Mutex::new(None), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 234514b0e5..1af7553338 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -9,10 +9,13 @@ use crate::bindings::_helpers::compile_command::reuse_recording::ArtifactRecorde use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, write_static_target, }; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::manifest::builder::build_manifest; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const EMPTY_DAG_PATH: &str = ""; @@ -55,15 +58,10 @@ pub(crate) fn manifest_payload<'py>( let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; - let adapter_type: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", discovered.getattr("local_config")?), - ], - )?; - let payload: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildManifest)?, + let adapter_type: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config, &discovered.getattr("local_config")?)?; + let payload: Bound<'py, PyAny> = build_manifest(Some(&keywords( + py, &[ ("project", graph.getattr("project")?), ("plan_output", py.None().into_bound(py)), @@ -71,7 +69,9 @@ pub(crate) fn manifest_payload<'py>( ("adapter_type", adapter_type), ("graph", graph.getattr("native")?), ], - )?; + )?))? + .into_bound(py) + .into_any(); status.complete(&format!( "Built manifest. ({}s)", seconds_since(py, started)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 0be1890ab6..603ca0ff0c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::{ compile_analysis_selection, project_graph, @@ -11,15 +12,21 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::{ use crate::bindings::_helpers::compile_driver::plan_result::{ PlanningInputs, connection_with_hooks, pipeline_result, }; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::compile_driver::selection_phases::{ + apply_unit_test_selectors, resolve_configured_rule_selection, +}; +use crate::bindings::_helpers::diagnostics::trackers::current_tracker; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; +use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -27,7 +34,7 @@ const GENERIC_DIALECT: &str = "generic"; /// The current build's phase timing tracker, when one is recording. fn timing_tracker(py: Python<'_>) -> PyResult> { - registry::native(py, NativeAttr::BuildPhaseTimingTracker)?.call_method0("current") + current_tracker(py) } /// Report `message` through `on_progress` when a callback is set. @@ -78,13 +85,10 @@ fn compiled_phase<'py>( }; let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; let connection_config: Bound<'py, PyAny> = if configured.is_none() { - call_with( - ®istry::native(py, NativeAttr::BuildEffectiveConnectionConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", options.getattr("selected_target")?), - ("cli_vars", options.getattr("cli_vars")?), - ], + build_effective_connection_config( + discovered, + Some(options.getattr("selected_target")?), + Some(options.getattr("cli_vars")?), )? } else { configured @@ -93,21 +97,24 @@ fn compiled_phase<'py>( let started: f64 = monotonic(py)?; let compiled: PyResult> = (|| { let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", options.getattr("selected_target")?), - ("no_sql_validation", options.getattr("no_sql_validation")?), - ("cli_vars", options.getattr("cli_vars")?), - ( - "external_sql_reference_resolver", - options.getattr("external_sql_reference_resolver")?, - ), - ("resolved_connection", connection_config.clone()), - ("analysis_selection", selection), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.getattr("selected_target")?), + ("no_sql_validation", options.getattr("no_sql_validation")?), + ("cli_vars", options.getattr("cli_vars")?), + ( + "external_sql_reference_resolver", + options.getattr("external_sql_reference_resolver")?, + ), + ("resolved_connection", connection_config.clone()), + ("analysis_selection", selection), + ], + )?), ) })(); let compile_seconds: f64 = monotonic(py)? - started; @@ -208,13 +215,8 @@ fn run_compile_pipeline<'py>( )?; let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let options: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ApplyUnitTestSelectors)?, - &[ - ("options", arguments.value("options")?), - ("discovered_inputs", discovered.clone()), - ], - )?; + let options: Bound<'py, PyAny> = + apply_unit_test_selectors(&arguments.value("options")?, &discovered)?; let hooks: Bound<'py, PyAny> = arguments.value("hooks")?; let hooks: Bound<'py, PyAny> = if hooks.is_none() { host::entry(py, ExecutorHost::ConnectionHooks)?.call0()? @@ -266,25 +268,22 @@ fn run_rules_gate<'py>( } else { GENERIC_DIALECT.into_pyobject(py)?.into_any() }; - let selected_keys: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveConfiguredRuleSelection)?, - &[ - ("options", options.clone()), - ("discovered_inputs", discovered.clone()), - ("project", project.clone()), - ], - )?; - let result: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::EvaluateRules)?, - &[ - ("graph", graph.clone()), - ("discovered_inputs", discovered.clone()), - ("config", config), - ("project_dir", project_dir), - ("dialect", dialect), - ("selected_keys", selected_keys), - ("no_sql_analysis", options.getattr("no_sql_validation")?), - ], + let selected_keys: Bound<'py, PyAny> = + resolve_configured_rule_selection(options, discovered, project)?; + let result: Bound<'py, PyAny> = evaluate_rules( + py, + Some(keywords( + py, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", config), + ("project_dir", project_dir), + ("dialect", dialect), + ("selected_keys", selected_keys), + ("no_sql_analysis", options.getattr("no_sql_validation")?), + ], + )?), )?; if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index c68c057499..850ff27a5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -293,7 +293,7 @@ fn frontier_compile_inputs<'py>( /// Build a static graph with an explicit deep-analysis selection. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph_with_analysis_selection<'py>( +pub(crate) fn build_project_graph_with_analysis_selection<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index c750e7c002..f94c859c10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -92,7 +92,7 @@ fn scoped_models<'py>(scopes: &Bound<'py, PyAny>) -> PyResult> /// Return options whose selectors name only resources, with unit tests selected apart. #[pyfunction] #[pyo3(signature = (*, options, discovered_inputs))] -fn apply_unit_test_selectors<'py>( +pub(crate) fn apply_unit_test_selectors<'py>( options: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { @@ -102,7 +102,7 @@ fn apply_unit_test_selectors<'py>( /// Align configured-rule subjects with models that received deep SQL analysis. #[pyfunction] #[pyo3(signature = (*, options, discovered_inputs, project))] -fn resolve_configured_rule_selection<'py>( +pub(crate) fn resolve_configured_rule_selection<'py>( options: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 3aa6ac0c3b..2537090419 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -9,6 +9,9 @@ use pyo3::{pyclass, pyfunction}; pub(crate) const EXPRESSION_SHAPES: &str = "infer_native_expression_source_shapes"; pub(crate) const EXPRESSION_SHAPE_BATCHES: &str = "expression_shape_batches"; pub(crate) const EXPRESSION_SHAPES_INFERRED: &str = "expression_shapes_inferred"; +pub(crate) const COLUMN_LINEAGE: &str = "build_project_column_lineage"; +pub(crate) const MODEL_CONTRACTS: &str = "evaluate_model_contracts"; +pub(crate) const PROJECT_DISCOVERIES: &str = "discover_project_inputs"; pub(crate) const MODEL_ANALYSIS: &str = "analyze_native_model_sql"; pub(crate) const SEMANTIC_COMPLETION: &str = "complete_semantic_project"; pub(crate) const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index cadaab4d69..30d45446e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -20,12 +20,15 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const GENERATE_ACTION: &str = "generate"; @@ -80,19 +83,19 @@ fn select_resources<'py>( ®istry::class(py, RecordId::CompileAnalysisSelection)?, &[("no_cache", PyBool::new(py, true).to_owned().into_any())], )?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", native_field(context, "adapter")?), - ("selected_target", native_field(request, "from_target")?), - ("cli_vars", native_field(request, "cli_vars")?), - ( - "resolved_connection", - native_field(context, "connection_config")?, - ), - ("analysis_selection", analysis), - ], + let project: Bound<'py, PyAny> = build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", native_field(&context, "adapter")?), + ("selected_target", native_field(&request, "from_target")?), + ("cli_vars", native_field(&request, "cli_vars")?), + ("resolved_connection", native_field(&context, "connection_config")?), + ("analysis_selection", analysis), + ], + )?), )?; let graph: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::BuildProjectGraphFromCompiledProject)?, @@ -102,8 +105,8 @@ fn select_resources<'py>( ®istry::native(py, NativeAttr::SelectProjectGraph)?, &[ ("graph", graph.clone()), - ("select", native_field(request, "select")?), - ("exclude", native_field(request, "exclude")?), + ("select", native_field(&request, "select")?), + ("exclude", native_field(&request, "exclude")?), ], )?; let compiled: Bound<'py, PyAny> = native_field(&graph, "project")?; @@ -162,11 +165,11 @@ fn compare<'py>( progress: &Progress<'py>, ) -> PyResult> { let py: Python<'py> = request.py(); - let adapter: Bound<'py, PyAny> = native_field(context, "adapter")?; + let adapter: Bound<'py, PyAny> = native_field(&context, "adapter")?; let reporter: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ConnectionProgressReporter)?, &[ - ("adapter_name", native_field(context, "adapter_name")?), + ("adapter_name", native_field(&context, "adapter_name")?), ("stream", progress.stream.clone()), ( "use_color", @@ -177,7 +180,7 @@ fn compare<'py>( let connection_started: Instant = Instant::now(); reporter.call_method1("on_connection_start", (1,))?; let connection: Bound<'py, PyAny> = - match adapter.call_method1("connect", (native_field(context, "connection_config")?,)) { + match adapter.call_method1("connect", (native_field(&context, "connection_config")?,)) { Ok(connection) => connection, Err(error) => { if exception(py, &error) { @@ -189,7 +192,7 @@ fn compare<'py>( let compared: PyResult> = (|| { connection_timing(&reporter, "on_connection_complete", connection_started)?; let count: usize = selection.models.len() + selection.sources.len(); - let target: String = native_field(request, "from_target")?.str()?.to_string(); + let target: String = native_field(&request, "from_target")?.str()?.to_string(); let status: Bound<'py, PyAny> = progress.status()?; let started: Instant = Instant::now(); status.call_method1( @@ -256,10 +259,10 @@ fn write_updates<'py>( status.call_method1("start", ("Updating repository contracts...",))?; let write: ContractWrite<'py> = ContractWrite { project_dir: project_dir.clone(), - overwrite: native_field(request, "overwrite")?, - adapter: native_field(context, "adapter")?, - resolved_connection: native_field(context, "connection_config")?, - cli_vars: native_field(request, "cli_vars")?, + overwrite: native_field(&request, "overwrite")?, + adapter: native_field(&context, "adapter")?, + resolved_connection: native_field(&context, "connection_config")?, + cli_vars: native_field(&request, "cli_vars")?, }; let updated: Bound<'py, PyAny> = match write_contracts(&write, graph, &result) { Ok(updated) => updated, @@ -320,7 +323,7 @@ fn resource_text(item: Bound<'_, PyAny>) -> PyResult { fn report(result: &Bound<'_, PyAny>) -> PyResult { Ok(ContractReport { - from_target: native_field(result, "from_target")?.str()?.to_string(), + from_target: native_field(&result, "from_target")?.str()?.to_string(), resources: result .getattr("evidence")? .try_iter()? @@ -336,13 +339,12 @@ fn report(result: &Bound<'_, PyAny>) -> PyResult { fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); - let json_output: bool = native_field(request, "json_output")?.is_truthy()?; + let json_output: bool = native_field(&request, "json_output")?.is_truthy()?; let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let use_color: bool = !native_field(request, "no_color")?.is_truthy()? - && !json_output - && stdout_supports_color(py)?; + let use_color: bool = + !native_field(&request, "no_color")?.is_truthy()? && !json_output && stdout_supports_color(py)?; Ok(Progress { stream, use_color }) } @@ -350,35 +352,35 @@ fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { #[pyfunction] #[pyo3(signature = (*, request))] fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let action: String = native_field(request, "action")?.str()?.to_string(); + let action: String = native_field(&request, "action")?.str()?.to_string(); let generate_write: bool = - action == GENERATE_ACTION && native_field(request, "write")?.is_truthy()?; - if native_field(request, "overwrite")?.is_truthy()? && !generate_write { + action == GENERATE_ACTION && native_field(&request, "write")?.is_truthy()?; + if native_field(&request, "overwrite")?.is_truthy()? && !generate_write { return Err(user_error( py, "contract --overwrite requires generate --write", "C471", )); } - let requested_dir: Bound<'_, PyAny> = native_field(request, "project_dir")?; + let requested_dir: Bound<'_, PyAny> = native_field(&request, "project_dir")?; let project_dir: Bound<'_, PyAny> = if requested_dir.is_truthy()? { requested_dir } else { imported(py, "pathlib", "Path")?.call_method0("cwd")? }; - let discovered: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; call_with( ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, &[ - ("select", native_field(request, "select")?), - ("exclude", native_field(request, "exclude")?), + ("select", native_field(&request, "select")?), + ("exclude", native_field(&request, "exclude")?), ("discovered_inputs", discovered.clone()), ], )?; - let from_target: Bound<'_, PyAny> = native_field(request, "from_target")?; + let from_target: Bound<'_, PyAny> = native_field(&request, "from_target")?; if !discovered .getattr("project_config")? .getattr("targets")? @@ -393,7 +395,7 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< let context: Bound<'_, PyAny> = resolve_adapter_connection_context( &discovered, &project_dir, - (py.None().into_bound(py), native_field(request, "cli_vars")?), + (py.None().into_bound(py), native_field(&request, "cli_vars")?), true, )?; let selection: Selection<'_> = select_resources(request, &discovered, &context)?; @@ -412,7 +414,7 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< )?; } let report: ContractReport = report(&result)?; - let rendered: String = if native_field(request, "json_output")?.is_truthy()? { + let rendered: String = if native_field(&request, "json_output")?.is_truthy()? { render_contract_json(&report) .map_err(|error| PyValueError::new_err(format!("{error:?}")))? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index 2233529140..22f84e46f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -4,10 +4,13 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const UTF8: &str = "utf-8"; @@ -28,14 +31,14 @@ pub(crate) fn write_contracts<'py>( result: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); - let project: Bound<'py, PyAny> = native_field(graph, "project")?; + let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let models: Bound<'py, PyDict> = by_name(&native_field(&project, "models")?)?; let sources: Bound<'py, PyDict> = by_name(&native_field(&project, "sources")?)?; let model_type: Bound<'py, PyAny> = imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; let updates: Bound<'py, PyDict> = PyDict::new(py); let updated_evidence: Bound<'py, PyList> = PyList::empty(py); - for evidence in native_field(result, "evidence")?.try_iter()? { + for evidence in native_field(&result, "evidence")?.try_iter()? { let evidence: Bound<'py, PyAny> = evidence?; let physical_columns: Bound<'py, PyAny> = native_field(&evidence, "physical_columns")?; let findings: Bound<'py, PyAny> = native_field(&evidence, "findings")?; @@ -56,9 +59,7 @@ pub(crate) fn write_contracts<'py>( ], )?; ( - write - .project_dir - .div(native_field(&model, "relative_path")?)?, + write.project_dir.div(native_field(&model, "relative_path")?)?, edited, ) } else { @@ -130,27 +131,30 @@ fn applied<'py>( ) -> PyResult<()> { let py: Python<'py> = result.py(); write_all(updates)?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", write.project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", write.project_dir.clone())])?), )?; if write.adapter.is_none() { return Ok(()); } - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("no_cache", true)?; - let selection: Bound<'py, PyAny> = - registry::class(py, RecordId::CompileAnalysisSelection)?.call((), Some(&keywords))?; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered), - ("adapter", write.adapter.clone()), - ("selected_target", native_field(result, "from_target")?), - ("resolved_connection", write.resolved_connection.clone()), - ("cli_vars", write.cli_vars.clone()), - ("analysis_selection", selection), - ], + let selection_keywords: Bound<'py, PyDict> = PyDict::new(py); + selection_keywords.set_item("no_cache", true)?; + let selection: Bound<'py, PyAny> = registry::class(py, RecordId::CompileAnalysisSelection)? + .call((), Some(&selection_keywords))?; + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("adapter", write.adapter.clone()), + ("selected_target", native_field(&result, "from_target")?), + ("resolved_connection", write.resolved_connection.clone()), + ("cli_vars", write.cli_vars.clone()), + ("analysis_selection", selection), + ], + )?), )?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 64b2d21b60..98c2cac979 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -163,7 +163,7 @@ fn model_resource_type(py: Python<'_>) -> PyResult> { /// Every model's column contract diagnostics, in model order. #[pyfunction] #[pyo3(signature = (*, project, dialect = None))] -fn evaluate_model_contracts<'py>( +pub(crate) fn evaluate_model_contracts<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 4d0feb89e9..0b885bfd28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -10,22 +10,22 @@ use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refres use sqlbuild_core::cost_command::models::CostCoverage; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { Ok(CostCoverage { - status: native_field(summary, "status")? - .getattr("value")? - .extract()?, - expected_statement_count: native_field(summary, "expected_statement_count")?.extract()?, - observed_statement_count: native_field(summary, "observed_statement_count")?.extract()?, - query_count: native_field(summary, "query_count")?.extract()?, + status: native_field(&summary, "status")?.getattr("value")?.extract()?, + expected_statement_count: native_field(&summary, "expected_statement_count")?.extract()?, + observed_statement_count: native_field(&summary, "observed_statement_count")?.extract()?, + query_count: native_field(&summary, "query_count")?.extract()?, limitations: summary .getattr("limitations")? .try_iter()? @@ -72,7 +72,7 @@ fn target_database<'py>( ) -> PyResult> { let py: Python<'py> = target_config.py(); if !target_config.is_none() { - let database: Bound<'py, PyAny> = native_field(target_config, "database")?; + let database: Bound<'py, PyAny> = native_field(&target_config, "database")?; if !database.is_none() { return Ok(database); } @@ -85,11 +85,11 @@ fn target_database<'py>( } fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { - let selector: Bound<'_, PyAny> = native_field(request, "selector")?; + let selector: Bound<'_, PyAny> = native_field(&request, "selector")?; if selector.extract::()? == HISTORY_SELECTOR { return Ok(()); } - let requested: Bound<'_, PyAny> = native_field(request, "project_dir")?; + let requested: Bound<'_, PyAny> = native_field(&request, "project_dir")?; let project_dir: Bound<'_, PyAny> = if requested.is_none() { imported(py, "pathlib", "Path")?.call_method0("cwd")? } else { @@ -100,16 +100,14 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let current: Bound<'_, PyAny> = native_field(&record, "cost")?; - let status: String = native_field(¤t, "status")? - .getattr("value")? - .extract()?; + let status: String = native_field(¤t, "status")?.getattr("value")?.extract()?; if !REFRESHABLE_STATUSES.contains(&status.as_str()) { return Ok(()); } let target_name: Bound<'_, PyAny> = native_field(&record, "target_name")?; - let discovered: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let target_config: Bound<'_, PyAny> = call_with( ®istry::native(py, NativeAttr::BuildEffectiveTargetConfig)?, @@ -121,10 +119,7 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { let adapter_name: Bound<'_, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ( - "project_config", - native_field(&discovered, "project_config")?, - ), + ("project_config", native_field(&discovered, "project_config")?), ("local_config", native_field(&discovered, "local_config")?), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index efd3badb26..5a66b8c059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -9,6 +9,8 @@ use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; use std::time::Instant; use crate::bindings::_helpers::build_command::support::{boolean, none, text}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::runtime_config::config_overrides; @@ -322,10 +324,12 @@ fn plan_dbt_interop_from_project<'py>( #[pyo3(signature = (**options))] fn debug_dbt_from_project(py: Python<'_>, options: Option>) -> PyResult { let args = KeywordArguments::read(py, "debug_dbt_from_project", options, DEBUG_OPTIONS)?; - let discovered = native( + let discovered = discover_project_inputs( py, - NativeAttr::DiscoverProjectInputs, - &[("project_dir", args.value("project_dir")?)], + Some(keywords( + py, + &[("project_dir", args.value("project_dir")?)], + )?), )?; let dbt_args: Vec = args .value("args")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index a6c4a7b2ac..c587de26bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -1,5 +1,8 @@ //! Discover, compile and resolve the combined dbt pipeline through its existing boundaries. +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -10,11 +13,11 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::DbtCompileRefere use crate::bindings::_helpers::dbt_interop::runtime_config::{ interop_adapter, manifest_path, parse_dbt_execution_args, plan_options, vars_mapping, }; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; +use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_OPTIONS: (&[&str], &[&str]) = ( &[ @@ -63,24 +66,24 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( request: &Bound<'py, PyAny>, ) -> PyResult> { let py = request.py(); - let mut executable = native_field(request, "dbt_executable")?; + let mut executable = native_field(&request, "dbt_executable")?; if !executable.is_truthy()? { executable = registry::native(py, NativeAttr::ResolveDbtExecutable)?.call0()?; } - let mut stream = native_field(request, "progress_stream")?; + let mut stream = native_field(&request, "progress_stream")?; if !stream.is_truthy()? { stream = py.import("sys")?.getattr("stdout")?; } - let command = native_field(request, "command")?; - let parsed = parse_dbt_execution_args(&command, &native_field(request, "args")?)?; + let command = native_field(&request, "command")?; + let parsed = parse_dbt_execution_args(&command, &native_field(&request, "args")?)?; let routed = call_with( ®istry::native(py, NativeAttr::RouteDbtInteropArgs)?, &[("command", command), ("parsed", parsed)], )?; - let project_dir = native_field(request, "project_dir")?; - let discovered = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let project_dir = native_field(&request, "project_dir")?; + let discovered = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let dbt_args: Vec = routed .getattr("dbt_args")? @@ -94,14 +97,14 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( &dbt_args, )? .into_any(); - let mut runner = native_field(request, "dbt_runner")?; + let mut runner = native_field(&request, "dbt_runner")?; if !runner.is_truthy()? { runner = call_with( ®istry::native(py, NativeAttr::DbtRunner)?, &[("dbt_executable", executable.clone())], )?; } - let dbt_stream = native_field(request, "dbt_stdout_stream")?; + let dbt_stream = native_field(&request, "dbt_stdout_stream")?; call_with( ®istry::class(py, RecordId::DbtInteropInvocation)?, &[ @@ -115,10 +118,7 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( stream }, ), - ( - "effective_sqlbuild_args", - native_field(&routed, "sqlbuild_args")?, - ), + ("effective_sqlbuild_args", native_field(&routed, "sqlbuild_args")?), ("routed", routed), ("discovered_inputs", discovered), ("dbt_options", options), @@ -150,12 +150,7 @@ fn load_compiled_dbt_manifest<'py>( let detail = registry::native(py, NativeAttr::DbtFailureDetail)?.call1((result,))?; let error = registry::error(py, ErrorId::DbtInteropRuntime)?.call( ("dbt compile failed",), - Some( - &crate::bindings::_helpers::compile_driver::python_calls::keywords( - py, - &[("help", detail)], - )?, - ), + Some(&keywords(py, &[("help", detail)])?), )?; return Err(PyErr::from_value(error)); } @@ -186,10 +181,7 @@ fn compile_dbt_interop_project<'py>( let name = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ( - "project_config", - native_field(&discovered, "project_config")?, - ), + ("project_config", native_field(&discovered, "project_config")?), ("local_config", native_field(&discovered, "local_config")?), ], )?; @@ -199,15 +191,18 @@ fn compile_dbt_interop_project<'py>( DbtCompileReferenceResolver::new(Some(args.value("manifest")?.unbind())), )? .into_any(); - let project = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered), - ("adapter", adapter.clone()), - ("no_sql_validation", args.value("no_sql_validation")?), - ("cli_vars", args.value("dbt_vars")?), - ("external_sql_reference_resolver", resolver), - ], + let project = build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("no_sql_validation", args.value("no_sql_validation")?), + ("cli_vars", args.value("dbt_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?), )?; completed(&callback, "Compiled SQLBuild project.", start)?; call_with( @@ -249,10 +244,7 @@ fn resolve_dbt_interop_plan<'py>( ®istry::class(py, RecordId::DbtInteropCommandArgs)?, &[ ("select", tuple.call1((native_field(&routed, "select")?,))?), - ( - "exclude", - tuple.call1((native_field(&routed, "exclude")?,))?, - ), + ("exclude", tuple.call1((native_field(&routed, "exclude")?,))?), ( "dbt_command_args", tuple.call1((native_field(&routed, "dbt_args")?,))?, @@ -261,10 +253,7 @@ fn resolve_dbt_interop_plan<'py>( "sqlbuild_command_args", tuple.call1((native_field(&invocation, "effective_sqlbuild_args")?,))?, ), - ( - "dbt_executable", - native_field(&invocation, "dbt_executable")?, - ), + ("dbt_executable", native_field(&invocation, "dbt_executable")?), ("sqlbuild_executable", args.value("sqlbuild_executable")?), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 364c3915cf..c5c6fb0c02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -30,6 +30,8 @@ use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehou use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; @@ -259,9 +261,12 @@ fn build_report( ) -> PyResult { let project_dir_object: Bound<'_, PyAny> = imported(py, "pathlib", "Path")?.call1((project_dir,))?; - let discovered_inputs: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir_object.clone())], + let discovered_inputs: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[("project_dir", project_dir_object.clone())], + )?), )?; let files: DebugConfigFiles = debug_config_files(project_dir); let adapter_name: Bound<'_, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs index d1455f1450..4dab1204b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -30,6 +30,11 @@ fn current_variable(py: Python<'_>) -> PyResult> { context_variable(py, &CURRENT_TRACKER, TRACKER_VARIABLE) } +/// The active build timing tracker, or None. +pub(crate) fn current_tracker(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} + /// `monotonic`, or `time.monotonic` when none is given. fn clock<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index a147a603f4..27dc07328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -17,7 +17,9 @@ use sqlbuild_cli::diff_report::types::QueryDiffPhase; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::diff_report::facts::diff_facts; use crate::bindings::_helpers::diff_report::query::{ execute_query_diff, execution_options, prepare_query_diff, project_adapter, @@ -83,9 +85,9 @@ fn resolve_invocation<'py>( let discovered: Bound<'py, PyAny> = if query_mode { discover_project_configuration(py, &project_dir)? } else { - call( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", &project_dir)], + discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )? }; call( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs index 3eb9cdf6ee..b7717fc98c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs @@ -40,6 +40,7 @@ use sqlbuild_state::execution_history::models::{ use sqlbuild_state::execution_history::types::UtcTime; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::execution_history::appends::prepared_batch; use crate::bindings::_helpers::execution_history::conversion::{ checked, event_filter_spec, exact_int, history_error_of_kind, new_namespace, now, @@ -498,20 +499,17 @@ impl SqliteExecutionHistory { let sqlite = py.import(SQLITE_MODULE)?; let opened: Bound<'_, PyAny> = native_field(&sqlite, "connect")?.call( (database.as_str(),), - Some( - &crate::bindings::_helpers::compile_driver::python_calls::keywords( - py, - &[ - ( - "timeout", - (busy_timeout_ms as f64 / MILLIS_PER_SECOND) - .into_bound_py_any(py)?, - ), - ("isolation_level", py.None().into_bound(py)), - ("check_same_thread", false.into_bound_py_any(py)?), - ], - )?, - ), + Some(&keywords( + py, + &[ + ( + "timeout", + (busy_timeout_ms as f64 / MILLIS_PER_SECOND).into_bound_py_any(py)?, + ), + ("isolation_level", py.None().into_bound(py)), + ("check_same_thread", false.into_bound_py_any(py)?), + ], + )?), )?; connection = Some(opened.clone()); slf.setattr("_connection", &opened)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index 33a3fde9bc..65a22d98f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -16,6 +16,8 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -27,8 +29,6 @@ use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_l use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; @@ -97,8 +97,7 @@ fn resolve_inputs<'py>( selected, discovered, }) => { - resolved.has_fixture_candidates = - native_field(&discovered, "test_files")?.is_truthy()?; + resolved.has_fixture_candidates = native_field(&discovered, "test_files")?.is_truthy()?; let adapter_dialect: Bound<'py, PyAny> = native_field(&adapter, "sql_analysis_dialect_name")?; resolved.dialect = if adapter_dialect.is_truthy()? { @@ -117,9 +116,9 @@ fn resolve_inputs<'py>( } } if fix { - resolved.discovered = Some(call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + resolved.discovered = Some(discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?); } Ok(resolved) @@ -133,14 +132,14 @@ fn fixture_inputs<'py>(resolved: &Resolved<'py>) -> PyResult = native_field(discovered, "local_config")?; + let local: Bound<'py, PyAny> = native_field(&discovered, "local_config")?; let settings_owner: Bound<'py, PyAny> = if local .getattr("setting_overrides")? .contains(SQL_ANALYSIS_SETTING_KEY)? { local } else { - native_field(discovered, "project_config")? + native_field(&discovered, "project_config")? }; let enabled: bool = settings_owner .getattr("settings")? @@ -196,7 +195,7 @@ fn complete_status(status: &Bound<'_, PyAny>, report: &LintReport, elapsed: f64) status.call_method1("error", (message,))?; } else { call_with( - &native_field(status, "complete")?, + &native_field(&status, "complete")?, &[("message", PyString::new(status.py(), &message).into_any())], )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 202e716c67..cac7a691a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -15,6 +15,8 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -334,13 +336,16 @@ pub(crate) fn resolve_lint_inputs<'py>( return path_scoped_inputs(project_dir, &directory, paths); } let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", disabled.clone()), - ("extract_output_column_locations", disabled), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", disabled.clone()), + ("extract_output_column_locations", disabled), + ], + )?), )?; call_with( ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 3b4b82ff5f..463de3a49d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -33,6 +33,7 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::build_progress::transient_status::maybe_status; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; @@ -136,7 +137,7 @@ fn discover<'py>( options.set_item("project_dir", project_dir)?; options.set_item("sql_analysis_enabled_override", analysis_override)?; options.set_item("extract_output_column_locations", false)?; - registry::native(py, NativeAttr::DiscoverProjectInputs)?.call((), Some(&options)) + discover_project_inputs(py, Some(options)) } fn reject_unit_tests( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 3cd6139974..6441bd888a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -35,7 +35,7 @@ const GENERIC_DIALECT: &str = "generic"; /// The project's column lineage graph in `mode`, or `None` without SQL analysis. #[pyfunction] -fn build_project_column_lineage<'py>( +pub(crate) fn build_project_column_lineage<'py>( project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, mode: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index cec1b2e845..5ef2ef5906 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -17,7 +17,9 @@ use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, @@ -75,9 +77,9 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index ed3a623375..35517b405f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -9,6 +9,8 @@ use pyo3::wrap_pyfunction; use crate::bindings::_helpers::build_command::support::keyword_relay; use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -64,12 +66,15 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", flag(py, false)), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?), )?; let preview: Bound<'py, PyDict> = PyDict::new(py); preview.set_item("discovered_inputs", &discovered)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index b3bfdd4ee5..b5a3ecbde1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -7,6 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -154,9 +156,9 @@ fn rule_inputs<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 257e448d25..4bd16132a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -3,6 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; @@ -23,9 +25,9 @@ pub(crate) fn project_adapter<'py>( Some(path) => path, None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let adapter_name: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 3d10f4a29c..9bc737c7f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -660,7 +660,6 @@ native_ids! { ApplyCommandWarehouse => "apply_command_warehouse", ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", - ApplyUnitTestSelectors => "apply_unit_test_selectors", AssembleProject => "assemble_project", AttachChecks => "attach_checks", AttachResources => "attach_resources", @@ -670,7 +669,6 @@ native_ids! { BuildAuditCommandPlan => "build_audit_command_plan", BuildCompileDbtManifestIndex => "build_compile_dbt_manifest_index", BuildCompileInputs => "build_compile_inputs", - BuildCompiledProject => "build_compiled_project", BuildCursorInputRelations => "build_cursor_input_relations", BuildDagJson => "build_dag_json", BuildDbtCombinedGraph => "build_dbt_combined_graph", @@ -700,7 +698,6 @@ native_ids! { BuildProjectColumnLineage => "build_project_column_lineage", BuildProjectGraph => "build_project_graph", BuildProjectGraphFromCompiledProject => "build_project_graph_from_compiled_project", - BuildProjectGraphWithAnalysisSelection => "build_project_graph_with_analysis_selection", BuildProjectRelationTargets => "build_project_relation_targets", BuildPythonRelationTargets => "build_python_relation_targets", BuildPythonRunPlanOutputs => "build_python_run_plan_outputs", @@ -764,7 +761,6 @@ native_ids! { DirectorySnapshotScope => "directory_snapshot_scope", DiscoverModelFiles => "discover_model_files", DiscoverModelSchemaFiles => "discover_model_schema_files", - DiscoverProjectInputs => "discover_project_inputs", DiscoverSchemaFiles => "discover_schema_files", DiscoverSourceFiles => "discover_source_files", DiscoverTestFiles => "discover_test_files", @@ -776,7 +772,6 @@ native_ids! { EffectiveConfigVars => "effective_config_vars", EnforceModelExecutionLimit => "enforce_model_execution_limit", EnforcePlanSafetyPolicies => "enforce_plan_safety_policies", - EvaluateModelContracts => "evaluate_model_contracts", EvaluateRules => "evaluate_rules", EvaluateRulesRequest => "evaluate_rules_request", EventExporterCommandScope => "EventExporterCommandScope", @@ -883,7 +878,6 @@ native_ids! { ReadOldNameViewHistories => "read_old_name_view_histories", ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", - RecordCompileInputReads => "record_compile_input_reads", RecordTemplateReads => "record_template_reads", RejectUnitTestSelectors => "reject_unit_test_selectors", RejectUnparseableHeaderRewrites => "reject_unparseable_header_rewrites", @@ -901,7 +895,6 @@ native_ids! { ReportCompileDiagnostic => "report_compile_diagnostic", ResolveCloneBoundary => "resolve_clone_boundary", ResolveCompileAnalysisSelection => "resolve_compile_analysis_selection", - ResolveConfiguredRuleSelection => "resolve_configured_rule_selection", ResolveCostRun => "resolve_cost_run", ResolveDbtConfig => "resolve_dbt_config", ResolveDbtConnectionConfig => "resolve_dbt_connection_config", @@ -1018,7 +1011,6 @@ native_ids! { WriteDbtExecutionSummary => "write_dbt_execution_summary", WriteExecutionHeader => "write_execution_header", WriteExecutionJsonOutput => "write_execution_json_output", - WriteReusableCompile => "write_reusable_compile", WriteRuntimeTarget => "write_runtime_target", WriteSourceFreshnessRecords => "write_source_freshness_records", WriteSqlbuildSkipNotice => "write_sqlbuild_skip_notice", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index f5a82effb1..592e5e280c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -6,24 +6,17 @@ use std::path::PathBuf; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, -}; -use sqlbuild_core::text::main::python_strip::python_strip; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyTuple}; use sqlbuild_rules::fix_verification::main::refusal::refusal; use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; @@ -37,6 +30,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ }; use crate::bindings::_helpers::rules::relation_keys::{RelationKeys, project_relation_keys}; use crate::bindings::_helpers::rules::run_lint::given_files; +use crate::bindings::_helpers::records::fields::field as native_field; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; @@ -118,9 +112,11 @@ impl<'py> FixSession<'py> { fn run(&mut self) -> PyResult> { let py = self.original.py(); let mut current = self.original.copy()?; - let paths: Vec> = field(&self.inputs, "model_files")? + let paths: Vec> = self + .inputs + .getattr("model_files")? .try_iter()? - .map(|model| model_file_path(&model?)) + .map(|model| native_field(&model?, "file_path")) .collect::>()?; let model_paths = PyFrozenSet::new(py, paths)?; let mut seen: BTreeSet = BTreeSet::new(); @@ -211,14 +207,15 @@ impl<'py> FixSession<'py> { let mut pending: Vec> = Vec::new(); let mut remaining: Vec> = Vec::new(); for violation in violations { - if !field(&violation, "code")? + if !violation + .getattr("code")? .extract::()? .starts_with("SQBRSQL") { continue; } let reason = self.refusal(&violation, model_paths)?; - let edit = field(&violation, "fix")?; + let edit = native_field(&violation, "fix")?; if reason.is_some() || edit.is_none() { remaining.push(report( &violation, @@ -233,7 +230,7 @@ impl<'py> FixSession<'py> { )?); continue; } - let path = field(&edit, "file_path")?; + let path = native_field(&edit, "file_path")?; let chosen = if let Some(chosen) = edits.get_item(&path)? { chosen.cast_into::()? } else { @@ -241,12 +238,12 @@ impl<'py> FixSession<'py> { edits.set_item(&path, &chosen)?; chosen }; - let start: usize = field(&edit, "start")?.extract()?; - let end: usize = field(&edit, "end")?.extract()?; + let start: usize = native_field(&edit, "start")?.extract()?; + let end: usize = native_field(&edit, "end")?.extract()?; let mut overlaps = false; for other in chosen.iter() { - if start < field(&other, "end")?.extract::()? - && field(&other, "start")?.extract::()? < end + if start < native_field(&other, "end")?.extract::()? + && native_field(&other, "start")?.extract::()? < end { overlaps = true; break; @@ -265,20 +262,20 @@ impl<'py> FixSession<'py> { violation: &Bound<'py, PyAny>, model_paths: &Bound<'py, PyAny>, ) -> PyResult> { - let code = field(violation, "code")?; - let path = field(violation, "file_path")?; - let has_fix = !field(violation, "fix")?.is_none(); + let code = native_field(&violation, "code")?; + let path = native_field(&violation, "file_path")?; + let has_fix = !native_field(&violation, "fix")?.is_none(); let facts = FixEligibility { model_body: model_paths.contains(&path)?, has_fix, safe_code: SAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), conditionless_join: code.eq("SQBRSQL003")?, - snowflake: field(&self.config, "dialect")?.eq("snowflake")?, + snowflake: self.config.getattr("dialect")?.eq("snowflake")?, }; if let Some(reason) = refusal(&facts) { return Ok(Some(reason.to_owned())); } - let unavailable = field(violation, "fix_unavailable_reason")?; + let unavailable = native_field(&violation, "fix_unavailable_reason")?; if !unavailable.is_none() || !has_fix { return unavailable.extract(); } @@ -323,7 +320,7 @@ impl<'py> FixSession<'py> { let mut reports: Vec> = Vec::new(); for report in self.reports.iter().chain(pending) { reports.push( - if let Some(reason) = self.blocked.get_item(field(report, "file_path")?)? { + if let Some(reason) = self.blocked.get_item(native_field(&report, "file_path")?)? { refused(report, &reason.extract::()?)? } else { report.clone() @@ -478,8 +475,8 @@ impl<'py> FixSession<'py> { let affected = self.blocked.copy()?; let mut reports: Vec> = Vec::new(); for report in &self.reports { - if field(report, "status")?.eq("applied")? { - affected.set_item(field(report, "file_path")?, py.None())?; + if native_field(&report, "status")?.eq("applied")? { + affected.set_item(native_field(&report, "file_path")?, py.None())?; } reports.push(refused(report, reason)?); } @@ -496,61 +493,12 @@ fn replace<'py>( object: &Bound<'py, PyAny>, changes: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - replaced(object, changes) -} - -/// A discovered model file's contents, read in place from a native model file. -fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - match model.cast::() { - Ok(native) => Ok(native.get().texts(model.py()).1.clone().into_any()), - Err(_) => native_field(model, "contents"), - } -} - -/// A discovered model file's path, read in place from a native model file. -fn model_file_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - match model.cast::() { - Ok(native) => Ok(native.get().paths(model.py()).0.clone()), - Err(_) => native_field(model, "file_path"), - } -} - -/// `model` with its edited text and the query `text` holds where the old query started. -fn edited_model<'py>( - model: &Bound<'py, PyAny>, - text: Bound<'py, PyAny>, -) -> PyResult> { - let py = model.py(); - let Ok(native) = model.cast::() else { - let current = native_field(model, "contents")?; - let start = current.call_method1("index", (native_field(model, "query_sql")?,))?; - let query = text - .get_item(pyo3::types::PySlice::new( - py, - start.extract()?, - isize::MAX, - 1, - ))? - .call_method0("strip")?; - return model.call_method1("with_text", (text, query)); - }; - let native: &NativeModelFile = native.get(); - let (query_sql, contents) = native.texts(py); - let contents: String = contents.to_str()?.to_owned(); - let Some(offset) = contents.find(query_sql.to_str()?) else { - return Err(pyo3::exceptions::PyValueError::new_err( - "substring not found", - )); - }; - let start: usize = contents[..offset].chars().count(); - let text: Bound<'py, PyString> = text.cast_into::()?; - let edited: String = text.to_str()?.to_owned(); - let tail: &str = edited - .char_indices() - .nth(start) - .map_or("", |(index, _)| &edited[index..]); - let query: Bound<'py, PyString> = PyString::new(py, python_strip(tail)); - Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind()))?.into_any()) + let kwargs = keywords(object.py(), changes)?; + object + .py() + .import("dataclasses")? + .getattr("replace")? + .call((object,), Some(&kwargs)) } fn updated_inputs<'py>( @@ -559,13 +507,22 @@ fn updated_inputs<'py>( ) -> PyResult> { let py = inputs.py(); let mut models: Vec> = Vec::new(); - for model in field(inputs, "model_files")?.try_iter()? { + for model in native_field(&inputs, "model_files")?.try_iter()? { let model = model?; - let current = model_contents(&model)?; - if let Some(text) = files.get_item(model_file_path(&model)?)? + let current = native_field(&model, "contents")?; + if let Some(text) = files.get_item(native_field(&model, "file_path")?)? && !text.eq(¤t)? { - models.push(edited_model(&model, text)?); + let start = current.call_method1("index", (native_field(&model, "query_sql")?,))?; + let query = text + .get_item(pyo3::types::PySlice::new( + py, + start.extract()?, + isize::MAX, + 1, + ))? + .call_method0("strip")?; + models.push(model.call_method1("with_text", (text, query))?); } else { models.push(model); } @@ -590,9 +547,9 @@ fn edited_files<'py>( .map(|edit| { let edit = edit?; Ok(FixEdit { - start: field(&edit, "start")?.extract()?, - end: field(&edit, "end")?.extract()?, - replacement: field(&edit, "replacement")?.extract()?, + start: native_field(&edit, "start")?.extract()?, + end: native_field(&edit, "end")?.extract()?, + replacement: native_field(&edit, "replacement")?.extract()?, }) }) .collect::>()?; @@ -617,7 +574,7 @@ fn failing_edits<'py>( for (path, _) in edits.iter() { resolved.set_item(path.call_method0("resolve")?, path)?; } - for (path, reason) in field(verdict, "failing")?.cast_into::()?.iter() { + for (path, reason) in native_field(&verdict, "failing")?.cast_into::()?.iter() { if let Some(original) = resolved.get_item(path)? { result.set_item(original, reason)?; } @@ -660,9 +617,9 @@ fn report<'py>( call_with( ®istry::class(py, RecordId::RuleFixResult)?, &[ - ("file_path", field(source, "file_path")?), - ("code", field(source, "code")?), - ("line", field(source, "line")?), + ("file_path", native_field(&source, "file_path")?), + ("code", native_field(&source, "code")?), + ("line", native_field(&source, "line")?), ("status", status.into_pyobject(py)?.into_any()), ("reason", reason.into_pyobject(py)?.into_any()), ], @@ -670,7 +627,7 @@ fn report<'py>( } fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult> { - if !field(report, "status")?.eq("applied")? { + if !native_field(&report, "status")?.eq("applied")? { return Ok(report.clone()); } replace( @@ -743,5 +700,6 @@ fn result<'py>( /// Whether a fix verdict proved every edit: nothing failed and nothing was unattributed. fn verified(verdict: &Bound<'_, PyAny>) -> PyResult { - Ok(!field(verdict, "failing")?.is_truthy()? && !field(verdict, "unattributed")?.is_truthy()?) + Ok(!native_field(&verdict, "failing")?.is_truthy()? + && !native_field(&verdict, "unattributed")?.is_truthy()?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index f648ff9397..bba9cb572d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -9,13 +9,15 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ ExpansionSources, SqlExpansionInputs, present, sql_expansion_inputs, }; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; @@ -76,13 +78,16 @@ fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult discovered, None => { let no: Bound<'_, PyAny> = PyBool::new(py, false).to_owned().into_any(); - call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", no.clone()), - ("extract_output_column_locations", no), - ], + discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )?), )? } }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index bfc075e488..3f5f1bb1fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -21,6 +21,8 @@ use sqlbuild_core::scenario_command::models::{ }; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -114,9 +116,9 @@ pub(crate) fn discover_project<'py>( directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let selectors: Bound<'py, PyAny> = native_field(request, "selectors")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 271d0ccd10..5588c9f8b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -19,8 +20,7 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridg use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, @@ -30,6 +30,7 @@ use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use pyo3::IntoPyObjectExt; +use crate::bindings::_helpers::records::fields::field as native_field; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; @@ -57,8 +58,7 @@ fn project_adapter<'py>( let discover_keywords: Bound<'py, PyDict> = PyDict::new(py); discover_keywords.set_item("project_dir", project_dir)?; discover_keywords.set_item("extract_output_column_locations", false)?; - let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&discover_keywords))?; + let discovered: Bound<'py, PyAny> = discover_project_inputs(py, Some(discover_keywords))?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, &native_field(&discovered, "local_config")?, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 1161eb5b16..930da556bc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -116,13 +116,6 @@ def _panicked(*arguments): native.build_project_column_lineage = _panicked """ -_RICH_COMPILE: tuple[str, ...] = ( - "compile", - "--json", - "--no-cache", - "--lineage-mode", - "rich", -) @pytest.mark.parametrize( @@ -247,13 +240,6 @@ def test_given_project_when_tracing_rich_lineage_then_it_is_traced( @pytest.mark.parametrize( "test_case", [ - NativeRichLineageFailureCliTestCase( - description="compile --lineage-mode rich", - files=_RICH_FILES, - command=_RICH_COMPILE, - expected_returncode=1, - expected_error_line=f"_native.NativeCompilerError: {_NATIVE_PANIC_ERROR}", - ), NativeRichLineageFailureCliTestCase( description="lineage --mode rich", files=_RICH_FILES, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 3a97fbda99..94d94f9f18 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -17,7 +17,6 @@ import time import zipfile from bisect import bisect_left -from collections import Counter from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout from dataclasses import replace @@ -44,7 +43,7 @@ from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter -from sqlbuild.adapter.contract.types import TypeDialect +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent @@ -3350,16 +3349,16 @@ def write_before_reuse_store( ) -> None: """Let another writer change target/ once, after a compile wrote it and before it stores.""" - original: Callable[..., None] = native_module.write_reusable_compile pending: list[Callable[[Path], None]] = [write] - def write_then_store(**kwargs: Any) -> None: - for action in tuple(pending): - action(project_dir) - pending.clear() - original(**kwargs) + class WrittenThenChanged(native_module.PreparedCompileArtifacts): + def __exit__(self, *exception: object) -> None: + super().__exit__(*exception) + for action in tuple(pending): + action(project_dir) + pending.clear() - monkeypatch.setattr(native_module, "write_reusable_compile", write_then_store) + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", WrittenThenChanged) def rewrite_compiled_model_unchanged(project_dir: Path) -> None: @@ -4807,8 +4806,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_module, "infer_native_expression_source_shapes"), - (native_module, "evaluate_model_contracts"), - (native_module, "build_project_column_lineage"), (plan_artifacts_module, "plan_compiled_sql_test_artifacts"), ) @@ -4872,35 +4869,31 @@ def type_system_compile( """Compile in this process; return the run and what native types answered.""" normalized: list[bool] = [] - statuses: Counter[str] = Counter() normalize: Callable[[str, str], object] = native_module.normalize_type - evaluate_contracts: Callable[..., ContractValidationResult] = ( - native_module.evaluate_model_contracts - ) def recorded(type_sql: str, dialect: str) -> object: result: object = normalize(type_sql, dialect) normalized.append(result is not None) return result - def recorded_contracts( - *, project: CompiledProjectView, dialect: TypeDialect | str | None - ) -> ContractValidationResult: - result: ContractValidationResult = evaluate_contracts(project=project, dialect=dialect) - statuses.update(contract_statuses(project=project, result=result)) - return result - type_normalization.normalize_type.cache_clear() with monkeypatch.context() as patch: patch.setattr(native_module, "normalize_type", recorded) - patch.setattr(native_module, "evaluate_model_contracts", recorded_contracts) for name, value in {REUSE_DISABLE_ENV_VAR: "1"}.items(): patch.setenv(name, value) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) type_normalization.normalize_type.cache_clear() + adapter: DuckDbAdapter = DuckDbAdapter() + project: CompiledProjectView = native_module.build_project_graph( + discovered_inputs=native_module.discover_project_inputs(project_dir=project_dir), + adapter=adapter, + ).project + result: ContractValidationResult = native_module.evaluate_model_contracts( + project=project, dialect=adapter.sql_analysis_dialect() + ) return run, NativeTypeAnswers( normalized=normalized, - typed_comparisons=statuses["typed_comparisons"], + typed_comparisons=contract_statuses(project=project, result=result)["typed_comparisons"], ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 7ca329ad8d..64e5273d6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -9,9 +9,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( - ContractValidationResult, NativeSqlTestArtifact, - ProjectColumnLineage, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, @@ -29,11 +27,11 @@ description="models_contracts_lineage_and_sql_tests", expected_native_returns={ "infer_native_expression_source_shapes": [ANY, ANY, ANY], - "evaluate_model_contracts": [ANY], - "build_project_column_lineage": [ANY], "plan_compiled_sql_test_artifacts": [ANY], }, expected_stage_work={ + "evaluate_model_contracts": 1, + "build_project_column_lineage": 1, "infer_native_expression_source_shapes": 1, "analyze_native_model_sql": 1, "complete_semantic_project": 1, @@ -64,13 +62,11 @@ def test_given_project_when_compiling_then_native_seams_answer( stage: stage_work.get(stage, 0) for stage in test_case.expected_stage_work } == test_case.expected_stage_work for seams in (native_seams,): - assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) planned_artifacts: object = seams["plan_compiled_sql_test_artifacts"][0] assert isinstance(planned_artifacts, tuple) assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) - assert isinstance(seams["evaluate_model_contracts"][0], ContractValidationResult) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index 9a86d50d88..19883313a2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -86,9 +86,8 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh project_dir=tmp_path, record=replace(record, cost=replace(record.cost, status=test_case.initial_status)), ) - monkeypatch.setattr( - "sqlbuild._native.discover_project_inputs", - lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), + _ = (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "shop"\nadapter = "snowflake"\n', encoding="utf-8" ) monkeypatch.setattr( "sqlbuild._native.resolve_effective_adapter_name", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 5ec96f130e..1d901e9c4d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -498,14 +498,7 @@ def test_given_lineage_mode_when_compiling_then_lineage_runs_in_that_mode( capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) - modes: list[str] = [] - build_lineage: Callable[..., object] = discovery_entry.build_project_column_lineage - - def recorded_lineage(*args: Any) -> object: - modes.append(ColumnLineageMode(args[2]).value) - return build_lineage(*args) - - monkeypatch.setattr(discovery_entry, "build_project_column_lineage", recorded_lineage) + _ = discovery_entry.compile_stage_work_info(reset=True) exit_code: int = run_compile( CompileCommandRequest( @@ -518,7 +511,9 @@ def recorded_lineage(*args: Any) -> object: _ = capsys.readouterr() assert exit_code == 0 - assert tuple(modes) == test_case.expected_lineage_mode_values + assert discovery_entry.compile_stage_work_info().get("build_project_column_lineage", 0) == len( + test_case.expected_lineage_mode_values + ) @pytest.mark.parametrize( @@ -626,16 +621,12 @@ def test_given_collection_state_when_compiling_then_cycles_are_paused_and_state_ ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) enabled_during: list[bool] = [] - discover: Callable[..., Any] = discovery_entry.discover_project_inputs - def observed_discover(**kwargs: Any) -> Any: + def observed_adapter(*args: Any, **kwargs: Any) -> NoConnectDuckDbAdapter: enabled_during.append(gc.isenabled()) - return discover(**kwargs) + return NoConnectDuckDbAdapter() - monkeypatch.setattr( - runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() - ) - monkeypatch.setattr(discovery_entry, "discover_project_inputs", observed_discover) + monkeypatch.setattr(runtime_adapters, "resolve_adapter", observed_adapter) set_collection: dict[bool, Callable[[], None]] = {True: gc.enable, False: gc.disable} was_enabled: bool = gc.isenabled() set_collection[test_case.collection_enabled_before]() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py index dbcc9a35a7..1b54c4b844 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py @@ -6,6 +6,7 @@ @dataclass(frozen=True) class BuildProcessReportTestCase: description: str + provider_source: str error_type: type[BaseException] verbose: bool expected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 18a48bbc06..53b04a222f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -13,10 +13,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledSqlScenario, - DiscoveredProjectInputs, DiscoveredSqlScenarioFile, - LocalConfig, - ProjectConfig, ) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module from sqlbuild.compiler.compile.types import CompiledResourceType @@ -25,14 +22,18 @@ from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal -def fail_build_discovery(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> Mock: - """Make project discovery raise and record the debug resource reporter calls.""" - - def discover(**_kwargs: object) -> DiscoveredProjectInputs: - raise error +def fail_build_discovery( + *, monkeypatch: pytest.MonkeyPatch, project_dir: Path, provider_source: str +) -> Mock: + """Make project discovery fail in a provider module and record debug resource reporting.""" + providers: Path = project_dir / "providers" + providers.mkdir(parents=True) + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "shop"\nadapter = "duckdb"\n', encoding="utf-8" + ) + _ = (providers / "failing.py").write_text(provider_source, encoding="utf-8") reporter: Mock = Mock(return_value=nullcontext()) - monkeypatch.setattr(native, "discover_project_inputs", discover) monkeypatch.setattr(resource_reporting_module, "process_resource_reporting", reporter) return reporter @@ -53,16 +54,15 @@ def configure_native_scenario_command( monkeypatch: pytest.MonkeyPatch, stream: ScenarioProgressStream, compile_project: Any, + project_dir: Path, ) -> None: - """Isolate the native scenario command from discovery, adapters and the warehouse.""" + """Isolate the native scenario command from adapters and the warehouse in an empty project.""" - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="project", adapter="duckdb"), - local_config=LocalConfig(), + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "project"\nadapter = "duckdb"\n', encoding="utf-8" ) monkeypatch.setattr(sys, "stdout", stream) use_plain_terminal(monkeypatch) - monkeypatch.setattr(native, "discover_project_inputs", lambda **_: discovered_inputs) monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) monkeypatch.setattr(adapters_module, "resolve_adapter", lambda **_: Mock()) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py index 429bd1358e..b3284c2102 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py @@ -7,7 +7,12 @@ import pytest -from sqlbuild._native import BuildCommandRequest, CliUserError, run_build_command +from sqlbuild._native import ( + BuildCommandRequest, + CliUserError, + ProviderDiscoveryError, + run_build_command, +) from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( BuildDurationLimitTestCase, BuildProcessReportTestCase, @@ -20,13 +25,15 @@ [ BuildProcessReportTestCase( description="failed debug build reports resources and partial timings", - error_type=RuntimeError, + provider_source='raise RuntimeError("provider import failed")\n', + error_type=ProviderDiscoveryError, verbose=False, expected_fragments=("Phase timings", "total"), expected_absent_fragments=("compile", "planning", "execution"), ), BuildProcessReportTestCase( description="interrupted debug build reports resources and partial timings", + provider_source="raise KeyboardInterrupt\n", error_type=KeyboardInterrupt, verbose=True, expected_fragments=("Phase timings", "total"), @@ -41,7 +48,9 @@ def test_given_failed_debug_build_when_unwinding_then_reports_resources_and_part capsys: pytest.CaptureFixture[str], tmp_path: Path, ) -> None: - reporter: Mock = fail_build_discovery(monkeypatch=monkeypatch, error=test_case.error_type()) + reporter: Mock = fail_build_discovery( + monkeypatch=monkeypatch, project_dir=tmp_path, provider_source=test_case.provider_source + ) with pytest.raises(test_case.error_type): run_build_command( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index 340918201b..4adb978cb5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -59,7 +59,10 @@ def compile_project(**kwargs: Any) -> CompilePipelineResult: return pipeline_result configure_native_scenario_command( - monkeypatch=monkeypatch, stream=stream, compile_project=compile_project + monkeypatch=monkeypatch, + stream=stream, + compile_project=compile_project, + project_dir=tmp_path, ) events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() @@ -119,7 +122,10 @@ def fail_compile(**kwargs: Any) -> CompilePipelineResult: raise RuntimeError("compile failed") configure_native_scenario_command( - monkeypatch=monkeypatch, stream=stream, compile_project=fail_compile + monkeypatch=monkeypatch, + stream=stream, + compile_project=fail_compile, + project_dir=tmp_path, ) events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index cb01e9b97b..6a88987264 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -17,6 +17,7 @@ SourceEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType @@ -104,12 +105,35 @@ def advance(self, seconds: float) -> None: class SlowConnectionDuckDbAdapter(DuckDbAdapter): - """An in-memory DuckDB adapter whose connections take `connection_seconds` on a manual clock.""" - - def __init__(self, *, clock: ManualClock, connection_seconds: float) -> None: + """An in-memory DuckDB adapter whose connections, compile profile and rules dialect take + `connection_seconds`, `compile_seconds` and `rules_seconds` on a manual clock. + + The compile profile takes its time on its first call; the dialect on its second, which the + rules gate makes after the compile's own dialect read.""" + + def __init__( + self, + *, + clock: ManualClock, + connection_seconds: float, + compile_seconds: float = 0.0, + rules_seconds: float = 0.0, + ) -> None: super().__init__() self._clock: ManualClock = clock self._connection_seconds: float = connection_seconds + self._compile_seconds: float = compile_seconds + self._dialect_seconds: list[float] = [0.0, rules_seconds] + + def expression_inference_profile(self) -> ExpressionInferenceProfile: + self._clock.advance(self._compile_seconds) + self._compile_seconds = 0.0 + return super().expression_inference_profile() + + def sql_analysis_dialect(self) -> str | None: + self._clock.advance(sum(self._dialect_seconds[:1])) + del self._dialect_seconds[:1] + return super().sql_analysis_dialect() def connect(self, config: dict[str, Any]) -> Any: del config diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 3bc86e35aa..ff3ee0d109 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -4,22 +4,17 @@ import time from pathlib import Path -from unittest.mock import Mock import pytest import sqlbuild._native as _native from sqlbuild._native import ( BuildPhaseTimingTracker, - CompiledProject, CompilePipelineOptions, CompilePipelineResult, PartialBuildPhaseTimings, - RulesRunResult, discover_project_inputs, ) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import ( @@ -63,31 +58,10 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di tmp_path: Path, ) -> None: clock: ManualClock = ManualClock() - project: CompiledProjectView = CompiledProject( - run_id="timing-run", - effective_target_name="test", - effective_connection={}, - effective_vars={}, - ) - - def compiled(**kwargs: object) -> CompiledProjectView: - del kwargs - clock.advance(test_case.compile_seconds) - return project - - def evaluated(*args: object, **kwargs: object) -> RulesRunResult: - del args, kwargs - clock.advance(test_case.rules_seconds) - return RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) - monkeypatch.setattr(time, "monotonic", clock) - monkeypatch.setattr(_native, "build_compiled_project", compiled) - monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) (tmp_path / "sqlbuild_project.toml").write_text( 'name = "timing"\nadapter = "duckdb"\n', encoding="utf-8" ) - monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) - monkeypatch.setattr(_native, "evaluate_rules", evaluated) tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=clock) progress: list[str] = [] progress_hooks: dict[bool, ConnectionHooks] = { @@ -99,7 +73,10 @@ def evaluated(*args: object, **kwargs: object) -> RulesRunResult: result: CompilePipelineResult = _native.run_compile_pipeline( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SlowConnectionDuckDbAdapter( - clock=clock, connection_seconds=test_case.connection_seconds + clock=clock, + connection_seconds=test_case.connection_seconds, + compile_seconds=test_case.compile_seconds, + rules_seconds=test_case.rules_seconds, ), options=CompilePipelineOptions(connection_config={}), hooks=progress_hooks[test_case.report_progress], diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 3ef0cd87cf..364dda5bf2 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -549,10 +549,9 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select unselected_file.parent.mkdir() unselected_file.write_text(fixture_sql, encoding="utf-8") native_format: Mock = Mock(wraps=native_module.format_lint_sql_bodies) - full_discovery: Mock = Mock(side_effect=AssertionError("full discovery must not run")) project_graph: Mock = Mock(side_effect=AssertionError("project graph must not build")) monkeypatch.setattr(native_module, "format_lint_sql_bodies", native_format) - monkeypatch.setattr(native_module, "discover_project_inputs", full_discovery) + _ = native_module.compile_stage_work_info(reset=True) monkeypatch.setattr(native_module, "build_project_graph", project_graph) exit_code: int = main( @@ -570,7 +569,7 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select assert f"{test_case.selected_file_count} files checked" in captured.err assert native_format.call_count == test_case.expected_native_batch_count assert len(native_format.call_args.args[0]) == test_case.expected_native_file_count - assert full_discovery.call_count == 0 + assert native_module.compile_stage_work_info().get("discover_project_inputs", 0) == 0 assert project_graph.call_count == 0 assert all( "CAST(NULL AS VARCHAR)" not in path.read_text(encoding="utf-8") for path in selected_files diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index d9f90d1c8a..557d46f66a 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -2,14 +2,12 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path import pytest import sqlbuild._native as native_module from sqlbuild._native import ( - DiscoveredProjectInputs, LintBody, LintConfig, LintRunResult, @@ -44,30 +42,16 @@ def test_given_lint_context_when_building_then_project_is_discovered_once( test_case: LintBehaviorTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - discovery_count: list[int] = [0] - discover: Callable[..., DiscoveredProjectInputs] = native_module.discover_project_inputs - - def counted_discovery( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None = None, - extract_output_column_locations: bool = True, - ) -> DiscoveredProjectInputs: - discovery_count[0] += 1 - return discover( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ) - - monkeypatch.setattr(native_module, "discover_project_inputs", counted_discovery) + _ = native_module.compile_stage_work_info(reset=True) _ = build_lint_expansion_context(project_dir=tmp_path) - assert discovery_count[0] == test_case.expected_value + assert ( + native_module.compile_stage_work_info()["discover_project_inputs"] + == test_case.expected_value + ) @pytest.mark.parametrize( From 6c71049400bac15f5af3fdbeeec5c87ec69c7db7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:46:46 +0100 Subject: [PATCH 08/11] refactor(compile): host CompiledResourceType as a native string enum CompiledResourceType is registered with the native string enums in sqlbuild._native (same members, values, str, repr and StrEnum behaviour) instead of being defined in sqlbuild.compiler.compile.types. Rust reads it through the record registry rather than importing the Python module or the facade host entry; Python importers take it from sqlbuild._native. Stage captures keep its previous qualified name through the hosted shape table, so captures are unchanged. --- .../src/hosted_shapes/constants.rs | 6 +- .../_helpers/build_command/lifecycle.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 5 +- .../_helpers/clone/clone_execution.rs | 5 +- .../compile_driver/assembly_checks.rs | 4 +- .../bindings/_helpers/compile_driver/diff.rs | 9 +- .../_helpers/compile_driver/pipeline.rs | 3 +- .../_helpers/compiler_enums/constants.rs | 16 ++++ .../_helpers/contract_command/write.rs | 3 +- .../contracts/contract_diagnostics.rs | 5 +- .../_helpers/format_command/selection.rs | 21 ++--- .../_helpers/freshness/freshness_command.rs | 57 +++++------- .../bindings/_helpers/freshness/planning.rs | 7 +- .../_helpers/graph/execution_gates.rs | 13 ++- .../src/bindings/_helpers/graph/keys.rs | 6 +- .../_helpers/hand_built/compiled_model.rs | 51 +++++------ .../_helpers/lineage/lineage_objects.rs | 5 +- .../bindings/_helpers/native_host/facade.rs | 6 -- .../_helpers/planner_changes/actions.rs | 15 ++-- .../_helpers/planner_changes/detect.rs | 46 +++++----- .../planner_entries/execution_plan.rs | 17 ++-- .../_helpers/planner_entries/model_entries.rs | 7 +- .../_helpers/planner_entries/values.rs | 1 - .../_helpers/planner_phases/objects.rs | 7 +- .../_helpers/planner_scenario/execution.rs | 7 +- .../planner_strategy/audit_entries.rs | 2 +- .../_helpers/planner_strategy/audits.rs | 2 +- .../_helpers/record_behaviours/compiled.rs | 69 ++++++-------- .../src/bindings/_helpers/records/ids.rs | 1 + .../_helpers/refactor/contract_compare.rs | 5 +- .../src/bindings/_helpers/rules/rule_plan.rs | 90 ++++++++++--------- .../_helpers/semantic_checks/stage_objects.rs | 4 +- .../semantic_shapes/source_bindings.rs | 48 +++++----- .../_helpers/sql_test_glue/compiled_tests.rs | 49 ++++------ .../_helpers/sql_test_glue/constants.rs | 1 - .../sql_test_glue/fixture_validation.rs | 19 ++-- .../_helpers/sql_test_glue/plan_entries.rs | 37 +++----- .../_helpers/varied_statistics.py | 2 +- src/sqlbuild/_native.pyi | 12 ++- src/sqlbuild/_native_host.py | 1 - src/sqlbuild/_native_host.pyi | 1 - src/sqlbuild/compiler/compile/types.py | 12 --- .../executor/build/_helpers/source_audits.py | 9 +- .../executor/build/classes/build_scheduler.py | 2 +- .../_helpers/fixtures/test_type_scope.py | 3 +- .../compiler/planner/_helpers/helpers.py | 2 +- .../planner/_helpers/resolve/helpers.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 2 +- .../sql_test_assembly/test_execute_chain.py | 3 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../planner/_helpers/test_fixture_planning.py | 3 +- .../_helpers/test_warehouse_cursor_reads.py | 3 +- .../_helpers/test_warehouse_snapshot.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../executor/build/concurrent/helpers.py | 3 +- .../sqlbuild/executor/build/custom/helpers.py | 2 +- .../src/sqlbuild/executor/run/helpers.py | 2 +- .../executor/run/incremental/helpers.py | 2 +- .../executor/run/microbatch/helpers.py | 2 +- .../src/sqlbuild/executor/run/view/helpers.py | 2 +- .../src/sqlbuild/executor/scenario/helpers.py | 2 +- .../scenario/test_fixture_lifecycle.py | 2 +- .../src/sqlbuild/executor/testing/helpers.py | 2 +- .../_helpers/build_planning/helpers.py | 2 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 2 +- .../cli/commands/main/build/helpers.py | 3 +- .../commands/main/build/test_defer_clone.py | 3 +- .../cli/commands/main/compile/helpers.py | 2 +- .../main/compile/test_compile_output.py | 3 +- .../compile/test_compile_target_writer.py | 2 +- .../cli/commands/main/execution/helpers.py | 2 +- .../cli/commands/shared/_helpers/helpers.py | 2 +- .../sqlbuild/cli/output/main/plan/helpers.py | 3 +- .../classes/test_build_sql_test_projection.py | 2 +- .../compiler/auditing/main/helpers.py | 2 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../compile/_helpers/test_assembly.py | 2 +- .../_helpers/test_parameterized_sql_tests.py | 2 +- .../compile/_helpers/test_required_columns.py | 2 +- .../_helpers/test_sql_analysis_columns.py | 2 +- .../_helpers/test_repository_edits.py | 3 +- .../sqlbuild/compiler/contracts/helpers.py | 3 +- .../src/sqlbuild/compiler/dag/main/helpers.py | 2 +- .../src/sqlbuild/compiler/lineage/helpers.py | 2 +- .../sqlbuild/compiler/lineage/test_columns.py | 2 +- .../src/sqlbuild/compiler/manifest/helpers.py | 2 +- .../compiler/pipeline/_helpers/helpers.py | 2 +- .../_helpers/target_validation/helpers.py | 3 +- .../compiler/pipeline/main/helpers.py | 3 +- .../planner/_helpers/changes/_test_helpers.py | 2 +- .../compiler/planner/_helpers/helpers.py | 2 +- .../test_column_migration_candidates.py | 3 +- .../planner/_helpers/planning/helpers.py | 2 +- .../planner/_helpers/resolve/helpers.py | 3 +- .../planner/_helpers/resolve/test_refs.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 2 +- .../sql_test_assembly/test_plan_test.py | 3 +- .../_helpers/test_fixture_planning_work.py | 2 +- .../planner/_helpers/test_microbatch_limit.py | 2 +- .../planner/_helpers/test_model_closure.py | 7 +- .../test_plan_entry_cursor_overrides.py | 3 +- .../planner/_helpers/test_scenario_graph.py | 2 +- .../_helpers/test_selection_staleness.py | 2 +- .../planner/_helpers/test_source_columns.py | 2 +- .../_helpers/test_source_load_nodes.py | 2 +- .../planner/_helpers/warehouse/helpers.py | 3 +- .../planner/classes/test_sql_test_planning.py | 2 +- .../main/execution/test_display_plan.py | 3 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../compiler/planner/main/identity/helpers.py | 2 +- .../test_version_identity_function_hashes.py | 2 +- .../compiler/python_nodes/_helpers/helpers.py | 2 +- .../_helpers/test_run_lifecycle.py | 2 +- .../compiler/source_freshness/main/helpers.py | 2 +- .../executor/auditing/main/helpers.py | 2 +- .../executor/build/_helpers/helpers.py | 2 +- .../_helpers/test_external_source_loads.py | 2 +- .../executor/build/_helpers/test_indexes.py | 4 +- .../executor/build/_helpers/test_scheduler.py | 2 +- .../build/_helpers/test_source_node.py | 3 +- .../test_build_scheduler_diagnostics.py | 3 +- .../classes/test_build_scheduler_identity.py | 3 +- .../test_build_scheduler_test_order.py | 3 +- .../clone/_helpers/fingerprinting/helpers.py | 2 +- .../executor/clone/_helpers/helpers.py | 2 +- .../sqlbuild/executor/janitor/main/helpers.py | 2 +- .../executor/pipeline/_helpers/helpers.py | 2 +- .../pipeline/_helpers/test_auditing.py | 3 +- .../pipeline/_helpers/test_testing.py | 2 +- .../executor/pipeline/main/helpers.py | 3 +- .../pipeline/main/test_schema_preflight.py | 2 +- .../sqlbuild/executor/run/_helpers/helpers.py | 2 +- .../run/_helpers/test_cursor_bounds.py | 2 +- .../test_snapshot_adapter_boundary.py | 2 +- .../executor/scenario/_helpers/helpers.py | 2 +- .../executor/scenario/main/helpers.py | 2 +- .../sqlbuild/executor/seed/main/helpers.py | 2 +- .../sqlbuild/executor/testing/main/helpers.py | 9 +- .../src/sqlbuild/integrations/dbt/helpers.py | 3 +- .../rule_engine/_helpers/engine/helpers.py | 3 +- .../_helpers/engine/test_native_facts.py | 3 +- .../classes/audit_facts/helpers.py | 3 +- .../rule_engine/main/evaluate/helpers.py | 2 +- 143 files changed, 433 insertions(+), 500 deletions(-) diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 3a3381bbd8..e5bf80b787 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -1,7 +1,7 @@ //! The Python shape names of the natively hosted compiler shapes. /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -446,6 +446,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ "sqlbuild._native:DeclarationVisibilityIndex", "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", ), + ( + "sqlbuild._native:CompiledResourceType", + "sqlbuild.compiler.compile.types:CompiledResourceType", + ), ( "sqlbuild._native:DefaultsConfig", "sqlbuild.spec.contracts.models:DefaultsConfig", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index b86d3fed28..5637fc9624 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const SUCCESS_STATUS: &str = "success"; /// The cursor window and output settings the Python lifecycle runs with. @@ -323,7 +322,7 @@ fn lifecycle_plan<'py>( &[("selection", selection), ("python_graph", graph.clone())], )?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let mut keys: Vec> = Vec::new(); for name in native_field(&plan, "ingress_loader_names")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index e3f5a1d425..dcfd670474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -36,7 +36,7 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::graph::python_selection::python_node_names; @@ -48,7 +48,6 @@ use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ select checks directly without '+', '~', or path-between expansion"; @@ -202,7 +201,7 @@ fn compile_pipeline<'py>( let loaders: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, loaders)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; let mut keys: Vec> = Vec::new(); for reference in required_refs.try_iter()? { keys.push(sql_ref_key(py, &reference?)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 96c7645755..5b3bb5f04e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -12,14 +12,13 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", "destination_model_entries", @@ -125,7 +124,7 @@ fn dependency_locations<'py>( let location_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledRelationLocation)?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; for pair in source_map.call_method0("items")?.try_iter()? { let (name, source): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; let table: Bound<'py, PyAny> = native_field(&source, "table")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 8e635a0f6c..2f7df00b53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -6,8 +6,6 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::native_host::facade::FacadeHost; -use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; @@ -144,7 +142,7 @@ pub(crate) fn compiled_sql_scenarios<'py>( let scenario_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledSqlScenario)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let resource_type: Bound<'py, PyAny> = - host::entry(py, FacadeHost::CompiledResourceType)?.getattr("SQL_SCENARIO")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_SCENARIO")?; let mut scenarios: Vec> = Vec::new(); for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { let scenario_input: Bound<'py, PyAny> = scenario_input?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index f01496a11d..1ef39a7861 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -14,15 +14,12 @@ use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::graph::views::project_lineage_views; use crate::bindings::_helpers::records::ids::ErrorId; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. struct KeyIndexes<'py> { indexes: Bound<'py, PyDict>, @@ -163,7 +160,7 @@ fn diff_model_names<'py>( let selected: HashSet = resolve_diff_selection(&graph, (&select, &exclude)) .map_err(|error| selector_error(py, error))?; let model_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut names: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 603ca0ff0c..924e59d7b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -29,7 +29,6 @@ use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progr use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const GENERIC_DIALECT: &str = "generic"; /// The current build's phase timing tracker, when one is recording. @@ -150,7 +149,7 @@ fn reject_compile_errors( let py: Python<'_> = project.py(); let plan_sql_tests: bool = options.getattr("plan_sql_tests")?.is_truthy()?; let sql_test: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_TEST")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_TEST")?; let mut errors: Vec> = Vec::new(); for diagnostic in project.getattr("diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index 17cb132b3c..f31b913f53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -79,6 +79,21 @@ const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { ("UNKNOWN", "unknown"), ], }; +const E_COMPILED_RESOURCE_TYPE: EnumSpec = EnumSpec { + name: "CompiledResourceType", + doc: "", + members: &[ + ("MODEL", "model"), + ("SOURCE", "source"), + ("SEED", "seed"), + ("UDF", "udf"), + ("TABLE_FN", "table_fn"), + ("DBT_REF", "dbt_ref"), + ("AUDIT", "audit"), + ("SQL_TEST", "sql_test"), + ("SQL_SCENARIO", "sql_scenario"), + ], +}; const E_INFERRED_NULLABILITY: EnumSpec = EnumSpec { name: "InferredNullability", doc: "Conservative nullability state inferred for an output column.", @@ -725,6 +740,7 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_COLUMN_LINEAGE_CONFIDENCE, E_COLUMN_LINEAGE_MODE, E_INFERRED_NULLABILITY, + E_COMPILED_RESOURCE_TYPE, E_POLYGLOT_ANALYSIS_DIALECT, E_MIGRATION_DISCOVERY, E_MIGRATION_COMPATIBILITY, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index 22f84e46f1..e298080c11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -12,7 +12,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const UTF8: &str = "utf-8"; /// The repository the edits apply to and how they apply. @@ -35,7 +34,7 @@ pub(crate) fn write_contracts<'py>( let models: Bound<'py, PyDict> = by_name(&native_field(&project, "models")?)?; let sources: Bound<'py, PyDict> = by_name(&native_field(&project, "sources")?)?; let model_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let updates: Bound<'py, PyDict> = PyDict::new(py); let updated_evidence: Bound<'py, PyList> = PyList::empty(py); for evidence in native_field(&result, "evidence")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 98c2cac979..0a8084a1e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location_rows}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PATHLIB_MODULE: &str = "pathlib"; const IMPLICIT_COLUMN_CONTRACTS: &str = "implicit"; const GENERIC_DIALECT: &str = "generic"; @@ -155,9 +154,7 @@ fn contract_diagnostic<'py>( } fn model_resource_type(py: Python<'_>) -> PyResult> { - py.import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? - .call1((MODEL_RESOURCE,)) + registry::class(py, RecordId::CompiledResourceType)?.call1((MODEL_RESOURCE,)) } /// Every model's column contract diagnostics, in model order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index cac7a691a4..542e0cf62d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -17,12 +17,11 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SELECTION_CODE: &str = "S007"; /// The adapter, the selected files when a selector scopes the run, and the discovered inputs. @@ -78,11 +77,8 @@ fn discovered_adapter<'py>( let adapter_name: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ( - "project_config", - native_field(discovered, "project_config")?, - ), - ("local_config", native_field(discovered, "local_config")?), + ("project_config", native_field(&discovered, "project_config")?), + ("local_config", native_field(&discovered, "local_config")?), ], )?; resolve_cli_adapter(&adapter_name, project_dir) @@ -95,7 +91,7 @@ fn selectors<'py>(py: Python<'py>, values: &[String]) -> PyResult) -> PyResult> { let mut pairs: Vec<(String, PathBuf)> = Vec::new(); - for model_file in native_field(scope.discovered, "model_files")?.try_iter()? { + for model_file in native_field(&scope.discovered, "model_files")?.try_iter()? { let model_file: Bound<'_, PyAny> = model_file?; let relative_path: Bound<'_, PyAny> = native_field(&model_file, "relative_path")?; let configured: Bound<'_, PyAny> = model_file @@ -173,7 +169,7 @@ fn model_paths( ], )?; let model_type: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut names: BTreeSet = BTreeSet::new(); for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; @@ -182,10 +178,7 @@ fn model_paths( } } let mut paths: BTreeSet = BTreeSet::new(); - for model in native_field(&graph, "project")? - .getattr("models")? - .try_iter()? - { + for model in native_field(&graph, "project")?.getattr("models")?.try_iter()? { let model: Bound<'_, PyAny> = model?; if names.contains(&native_field(&model, "name")?.extract::()?) { let relative: PathBuf = native_field(&model, "relative_path")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 67a8d50ee1..44b0a6b18d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -27,15 +27,15 @@ use crate::bindings::_helpers::planner_selection::selectors::{ }; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; +use crate::bindings::_helpers::records::fields::field as native_field; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; @@ -81,8 +81,8 @@ fn source_names_for_keys( let mut names: BTreeSet = BTreeSet::new(); let py: Python<'_> = graph.py(); let source_type: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SOURCE")?; - let upstream_deps: Bound<'_, PyAny> = native_field(graph, "upstream_deps")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; + let upstream_deps: Bound<'_, PyAny> = native_field(&graph, "upstream_deps")?; for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; let upstream: Bound<'_, PyAny> = @@ -114,7 +114,7 @@ fn selected_source_names( let selected_keys: Bound<'_, PyAny> = if select.is_truthy()? { graph_keys(graph, select)? } else { - native_field(graph, "all_keys")?.call_method0("values")? + native_field(&graph, "all_keys")?.call_method0("values")? }; let selected: BTreeSet = source_names_for_keys(graph, &selected_keys)?; let excluded: BTreeSet = if exclude.is_truthy()? { @@ -174,9 +174,9 @@ fn previous_records<'py>( )?; let read: StateRead<'py> = StateRead { connection: connection.clone(), - execute: native_field(adapter, "execute")?, - render_qualified_name: native_field(adapter, "render_qualified_name")?, - render_read_latest_sql: native_field(adapter, "render_read_latest_source_freshness_sql")?, + execute: native_field(&adapter, "execute")?, + render_qualified_name: native_field(&adapter, "render_qualified_name")?, + render_read_latest_sql: native_field(&adapter, "render_read_latest_source_freshness_sql")?, }; let records: Bound<'py, PyDict> = PyDict::new(py); for schema in &schemas { @@ -211,19 +211,16 @@ fn observe( &[ ("discovered_inputs", discovered), ("project_dir", project_dir), - ( - "selected_target", - &native_field(request, "selected_target")?, - ), - ("cli_vars", &native_field(request, "cli_vars")?), + ("selected_target", &native_field(&request, "selected_target")?), + ("cli_vars", &native_field(&request, "cli_vars")?), ], )?), )?; let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; let observed = (|| { - let project: Bound<'_, PyAny> = native_field(graph, "project")?; + let project: Bound<'_, PyAny> = native_field(&graph, "project")?; let by_identity: Option> = - if native_field(request, "compare_state")?.extract::()? { + if native_field(&request, "compare_state")?.extract::()? { Some(previous_records(adapter, &connection, &project)?.into_any()) } else { None @@ -270,22 +267,22 @@ fn write_json_output( #[pyfunction] #[pyo3(signature = (*, request))] fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let fail_on_stale: bool = native_field(request, "fail_on_stale")?.extract()?; - let fail_on_error: bool = native_field(request, "fail_on_error")?.extract()?; - if fail_on_stale && !native_field(request, "compare_state")?.extract::()? { + let fail_on_stale: bool = native_field(&request, "fail_on_stale")?.extract()?; + let fail_on_error: bool = native_field(&request, "fail_on_error")?.extract()?; + if fail_on_stale && !native_field(&request, "compare_state")?.extract::()? { return Err(user_error( py, (STALE_REQUIRES_STATE_MESSAGE, STALE_REQUIRES_STATE_CODE), )); } - let project_dir: Bound<'_, PyAny> = match native_field(request, "project_dir")? { + let project_dir: Bound<'_, PyAny> = match native_field(&request, "project_dir")? { directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; let discovered: Bound<'_, PyAny> = discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; - let select: Bound<'_, PyAny> = native_field(request, "select")?; - let exclude: Bound<'_, PyAny> = native_field(request, "exclude")?; + let select: Bound<'_, PyAny> = native_field(&request, "select")?; + let exclude: Bound<'_, PyAny> = native_field(&request, "exclude")?; reject_unit_test_selectors(&select, &exclude, &discovered)?; let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, @@ -299,15 +296,9 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult &[ ("discovered_inputs", &discovered), ("adapter", &adapter), - ( - "selected_target", - &native_field(request, "selected_target")?, - ), - ( - "no_sql_validation", - &native_field(request, "no_sql_validation")?, - ), - ("cli_vars", &native_field(request, "cli_vars")?), + ("selected_target", &native_field(&request, "selected_target")?), + ("no_sql_validation", &native_field(&request, "no_sql_validation")?), + ("cli_vars", &native_field(&request, "cli_vars")?), ], )?), )?; @@ -324,15 +315,15 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult }; let payload: String = freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}")))?; - let json_output_path: Bound<'_, PyAny> = native_field(request, "json_output_path")?; - if native_field(request, "json_output")?.extract::()? { + let json_output_path: Bound<'_, PyAny> = native_field(&request, "json_output_path")?; + if native_field(&request, "json_output")?.extract::()? { write_json_output(py, &payload, (true, &json_output_path))?; } else { if !json_output_path.is_none() { write_json_output(py, &payload, (false, &json_output_path))?; } let use_color: bool = - !native_field(request, "no_color")?.extract::()? && stdout_supports_color(py)?; + !native_field(&request, "no_color")?.extract::()? && stdout_supports_color(py)?; write(py, &render_freshness_text(&results, CliStyle { use_color }))?; } Ok(freshness_exit_code( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 5eeffdde1b..9319287001 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -7,9 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTup use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_state::freshness::main::configuration_error::configuration_error; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::freshness::exception_types::SourceFreshnessObservationError; use crate::bindings::_helpers::freshness::observation::{ Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, @@ -26,7 +24,6 @@ use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use crate::bindings::_helpers::records::fields::field as native_field; const ADAPTER_STRATEGY: &str = "adapter"; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; /// The message of an `AdapterUserError` or a source freshness observation error, if it is one. @@ -306,7 +303,7 @@ fn downstream_models<'py>( &[ ( "resource_type", - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?, + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?, ), ("name", source_name.clone()), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index a439a58255..44ec984b2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -8,11 +8,10 @@ use crate::bindings::_helpers::analysis::explicit_references::{ GateRow, HookRow, native_execution_gate_edges, }; use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -93,7 +92,7 @@ fn model_python_hook_names<'py>( model: &Bound<'py, PyAny>, ) -> PyResult> { let entry_type: Bound<'_, PyAny> = registry::class(py, RecordId::PythonHookEntry)?; - let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; + let values: Bound<'_, PyAny> = native_field(&model, "config")?.getattr("values")?; let mut names: Vec = Vec::new(); for hook_key in MODEL_HOOK_KEYS { let entries: Bound<'_, PyAny> = values.call_method1("get", (hook_key,))?; @@ -123,15 +122,13 @@ pub(crate) fn sql_ref_key<'py>( py: Python<'py>, reference: &Bound<'py, PyAny>, ) -> PyResult> { - let resource_type: Bound<'py, PyAny> = py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? - .call1((native_field(reference, "kind")?.getattr("value")?,))?; + let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)? + .call1((native_field(&reference, "kind")?.getattr("value")?,))?; keyword_call( ®istry::class(py, RecordId::CompiledObjectKey)?, &[ ("resource_type", resource_type), - ("name", native_field(reference, "name")?), + ("name", native_field(&reference, "name")?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs index 704ef6c721..a5d71ca98f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs @@ -6,8 +6,6 @@ use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - /// One `CompiledObjectKey` per native pair, so shared keys convert once. pub(crate) struct KeyCache<'py> { key_type: Bound<'py, PyAny>, @@ -19,9 +17,7 @@ impl<'py> KeyCache<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { Ok(Self { key_type: registry::class(py, RecordId::CompiledObjectKey)?, - resource_type: py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")?, + resource_type: registry::class(py, RecordId::CompiledResourceType)?, keys: PyDict::new(py), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index f05a66d68b..3a377d56f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,12 +1,10 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::native_host::facade::FacadeHost; -use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; +use crate::bindings::_helpers::records::fields::field as native_field; type Object<'py> = Bound<'py, PyAny>; @@ -18,7 +16,7 @@ pub(crate) fn compiled_model<'py>( macro_deps: &Object<'py>, ) -> PyResult> { let py: Python<'py> = model_input.py(); - let model_file: Object<'py> = native_field(model_input, "model_file")?; + let model_file: Object<'py> = native_field(&model_input, "model_file")?; let model_name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let none: Object<'py> = py.None().into_bound(py); let mut inferred: Object<'py> = none.clone(); @@ -28,8 +26,8 @@ pub(crate) fn compiled_model<'py>( let mut diagnostics: Object<'py> = PyTuple::empty(py).into_any(); let mut validated: Object<'py> = false_object(py); let mut proof: Object<'py> = registry::native(py, NativeAttr::DynamicContractProof)? - .call1((native_field(analysis, "standalone_proof")?,))?; - let session: Object<'py> = native_field(analysis, "session")?; + .call1((native_field(&analysis, "standalone_proof")?,))?; + let session: Object<'py> = native_field(&analysis, "session")?; if !session.is_none() { let (session, position): (Object<'py>, Object<'py>) = session.extract()?; let arguments: Bound<'py, PyDict> = PyDict::new(py); @@ -45,7 +43,7 @@ pub(crate) fn compiled_model<'py>( has_star = native_field(&result, "has_star")?; star_resolved = native_field(&result, "star_resolved")?; } else { - inferred = fallback_columns(&native_field(analysis, "fallback")?)?; + inferred = fallback_columns(&native_field(&analysis, "fallback")?)?; } let session_proof: Object<'py> = native_field(&outcome, "dynamic_column_contract")?; if session_proof.is_truthy()? { @@ -68,7 +66,7 @@ pub(crate) fn compiled_model<'py>( inferred = native_field(&proof, "fixed_columns")?; has_star = true_object(py); } - let resource_type: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?; + let resource_type: Object<'py> = registry::class(py, RecordId::CompiledResourceType)?; let key_class: Object<'py> = registry::class(py, RecordId::CompiledObjectKey)?; let mut dependency_keys: Vec> = Vec::new(); for dependency in deps.try_iter()? { @@ -81,7 +79,7 @@ pub(crate) fn compiled_model<'py>( ], )?); } - let config: Object<'py> = native_field(model_input, "config")?; + let config: Object<'py> = native_field(&model_input, "config")?; let key: Object<'py> = call( &key_class, &[ @@ -96,20 +94,17 @@ pub(crate) fn compiled_model<'py>( ("deps", PyTuple::new(py, dependency_keys)?.into_any()), ("name", model_name.clone()), ("relative_path", native_field(&model_file, "relative_path")?), - ("query_sql", native_field(model_input, "query_sql")?), + ("query_sql", native_field(&model_input, "query_sql")?), ("config", config.clone()), ("destination", destination(&config, &model_name)?), - ("references", native_field(model_input, "references")?), - ("schema_entry", native_field(model_input, "schema_entry")?), + ("references", native_field(&model_input, "references")?), + ("schema_entry", native_field(&model_input, "schema_entry")?), ("inferred_columns", inferred), ("fast_lineage_columns", lineage), ("fast_lineage_has_star", has_star), ("fast_lineage_star_resolved", star_resolved), ("authored_sql", native_field(&model_file, "contents")?), - ( - "authored_query_sql", - native_field(&model_file, "query_sql")?, - ), + ("authored_query_sql", native_field(&model_file, "query_sql")?), ( "output_column_locations", native_field(&model_file, "output_column_locations")?, @@ -121,19 +116,19 @@ pub(crate) fn compiled_model<'py>( ("macro_deps", macro_deps.clone()), ( "enum_declarations", - native_field(model_input, "enum_declarations")?, + native_field(&model_input, "enum_declarations")?, ), ( "constant_declarations", - native_field(model_input, "constant_declarations")?, + native_field(&model_input, "constant_declarations")?, ), - ("enum_columns", native_field(model_input, "enum_columns")?), + ("enum_columns", native_field(&model_input, "enum_columns")?), ("binding_diagnostics", diagnostics), ("binding_validated", validated), ("dynamic_column_contract", proof), ( "rejected_sql_analysis_opt_out", - native_field(model_input, "rejected_sql_analysis_opt_out")?, + native_field(&model_input, "rejected_sql_analysis_opt_out")?, ), ], ) @@ -150,7 +145,7 @@ fn true_object(py: Python<'_>) -> Object<'_> { /// A model's configured physical and logical destination. fn destination<'py>(config: &Object<'py>, model_name: &Object<'py>) -> PyResult> { let py: Python<'py> = config.py(); - let values: Object<'py> = native_field(config, "values")?; + let values: Object<'py> = native_field(&config, "values")?; let text = |key: &str| -> PyResult>> { let value: Object<'py> = values.call_method1("get", (key,))?; Ok(value.is_instance_of::().then_some(value)) @@ -166,11 +161,8 @@ fn destination<'py>(config: &Object<'py>, model_name: &Object<'py>) -> PyResult< ("schema", text("schema")?.unwrap_or_else(|| none.clone())), ("name", text("alias")?.unwrap_or_else(|| model_name.clone())), ("qualified_name", none.clone()), - ("logical_schema", native_field(config, "logical_schema")?), - ( - "logical_database", - native_field(config, "logical_database")?, - ), + ("logical_schema", native_field(&config, "logical_schema")?), + ("logical_database", native_field(&config, "logical_database")?), ], ) } @@ -208,7 +200,7 @@ fn located<'py>( if !diagnostics.is_truthy()? { return Ok(PyTuple::empty(py).into_any()); } - let model_file: Object<'py> = native_field(model_input, "model_file")?; + let model_file: Object<'py> = native_field(&model_input, "model_file")?; let path: Object<'py> = native_field(&model_file, "relative_path")?; let mut rows: Vec> = Vec::new(); for item in diagnostics.try_iter()? { @@ -230,7 +222,7 @@ fn located<'py>( ], )? .into_any(); - let expansion: Object<'py> = native_field(model_input, "sql_expansion")?; + let expansion: Object<'py> = native_field(&model_input, "sql_expansion")?; let expansion_row: Object<'py> = if expansion.is_none() { expansion } else { @@ -262,7 +254,8 @@ fn located<'py>( let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = registry::class(py, RecordId::SourceLocation)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?.getattr("MODEL")?; + let model: Object<'py> = + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); for row in located.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs index 0afc432355..e8bee6abb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -9,7 +9,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LOGGING_MODULE: &str = "logging"; const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; @@ -35,9 +34,7 @@ pub(crate) struct LineageClasses<'py> { impl<'py> LineageClasses<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { - let resource_type: Bound<'py, PyAny> = py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")?; + let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?; let nullability: Bound<'py, PyAny> = registry::class(py, RecordId::InferredNullability)?; Ok(Self { qualified_column: registry::class(py, RecordId::QualifiedLineageColumn)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs index d40ae9e79f..c46ee8fe55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs @@ -15,7 +15,6 @@ pub(crate) enum FacadeHost { AuditSeverity, ThresholdOperator, AttachedAuditTargetKind, - CompiledResourceType, FunctionLanguage, SqlTestMode, CursorAlgebraError, @@ -74,11 +73,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Facade, }, - HostEntry { - name: "CompiledResourceType", - kind: HostKind::Class, - edge: HostEdge::Facade, - }, HostEntry { name: "FunctionLanguage", kind: HostKind::Class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index f587171c84..dd05716525 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -4,12 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CREATE_TABLE_ACTION: &str = "create_table"; @@ -22,13 +21,13 @@ pub(crate) fn resolve_model_actions<'py>( ) -> PyResult> { let py: Python<'py> = scope.py(); let model_kind: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let action_class: Bound<'py, PyAny> = registry::class(py, RecordId::ResolvedModelAction)?; - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; - let model_changes: Bound<'py, PyAny> = native_field(changes, "models")?; + let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; + let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; + let model_changes: Bound<'py, PyAny> = native_field(&changes, "models")?; let resolved: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(scope, "execution_order")?.try_iter()? { + for key in native_field(&scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index d1606ba79f..bf8ccf1507 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -43,12 +43,11 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; +use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; @@ -102,15 +101,14 @@ pub(crate) fn detect_changes<'py>( scope_model_changes((project, scope, snapshot), None, expected)?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = registry::class(py, RecordId::FunctionChangeResult)?; - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; - let recorded: Bound<'py, PyAny> = - native_field(snapshot, "fingerprints")?.getattr("functions")?; + let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; + let recorded: Bound<'py, PyAny> = native_field(&snapshot, "fingerprints")?.getattr("functions")?; let query_change_tracking: bool = project .getattr("settings")? .getattr("query_change_tracking")? .is_truthy()?; - let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; - for function in native_field(project, "functions")?.try_iter()? { + let dialect: Option = native_field(&snapshot, "column_dialect")?.extract()?; + for function in native_field(&project, "functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = native_field(&function, "name")?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; @@ -131,9 +129,7 @@ pub(crate) fn detect_changes<'py>( &name.extract::()?, ( &fingerprint_sql, - &native_field(&function, "language")? - .str()? - .extract::()?, + &native_field(&function, "language")?.str()?.extract::()?, dialect.as_deref(), ), )? != fingerprint @@ -250,7 +246,7 @@ pub(crate) fn scope_model_changes<'py>( .extract()?, }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &native_field(project, "functions")?, + &native_field(&project, "functions")?, project .getattr("sql_analysis_dialect")? .extract::>()? @@ -259,7 +255,7 @@ pub(crate) fn scope_model_changes<'py>( .into_any(); let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = host::entry(py, UserCodeHost::BuildHookIdentities)? - .call1((native_field(project, "hook_functions")?,))?; + .call1((native_field(&project, "hook_functions")?,))?; for item in identities.call_method0("items")?.try_iter()? { let item: Bound<'py, PyAny> = item?; hook_version_hashes.set_item( @@ -275,11 +271,11 @@ pub(crate) fn scope_model_changes<'py>( hook_version_hashes: hook_version_hashes.into_any(), }; let model_kind: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; + let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; + let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; let changes: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(scope, "execution_order")?.try_iter()? { + for key in native_field(&scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { continue; @@ -347,7 +343,7 @@ pub(crate) fn model_change<'py>( .getattr("models")? .call_method1("get", (&model_name,))?; let recorded: Option = recorded_fingerprint(&fingerprint)?; - let dialect: Bound<'py, PyAny> = native_field(snapshot, "column_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(&snapshot, "column_dialect")?; let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { Some(model_query_hash( py, @@ -367,7 +363,7 @@ pub(crate) fn model_change<'py>( (Some(hash), Some(recorded)) => *hash != recorded.definition_hash, _ => false, }; - if !relation_exists && native_field(snapshot, "renamed_models")?.contains(&model_name)? { + if !relation_exists && native_field(&snapshot, "renamed_models")?.contains(&model_name)? { result.query_changed = definition_changed; let backfill: Backfill = if definition_changed { replay_policy(model)? @@ -394,7 +390,7 @@ pub(crate) fn model_change<'py>( .as_ref() .map(|recorded| recorded.metadata_json.as_str()) .filter(|text| *text != EMPTY_FINGERPRINT_METADATA_JSON); - let renames: Bound<'py, PyAny> = native_field(snapshot, "reference_renames")?; + let renames: Bound<'py, PyAny> = native_field(&snapshot, "reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( (model_reads::query_sql(model)?.extract()?, &fingerprint), @@ -445,7 +441,7 @@ pub(crate) fn model_change<'py>( fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = model.py(); let raw: Option = authored_config_str( - &native_field(model, "config")?.getattr("values")?, + &native_field(&model, "config")?.getattr("values")?, REPLAY_CONFIG_KEY, )? .map(|value| value.extract()) @@ -465,9 +461,9 @@ fn recorded_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult( if warehouse.is_none() { return Ok(Vec::new()); } - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = native_field(&model, "schema_entry")?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index c1d782f835..4bf2b5d6d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, + KeywordArguments, call_with, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -11,7 +11,7 @@ use crate::bindings::_helpers::planner_changes::detect::detect_changes; use crate::bindings::_helpers::planner_entries::test_planning::{ PlanningRequest, SqlTestPlanning, start_planning, }; -use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; +use crate::bindings::_helpers::planner_entries::values::{native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::execution_changes::{ @@ -31,12 +31,13 @@ use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; +use crate::bindings::_helpers::records::fields::field as native_field; const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; /// Selected scopes, and the overrides with cursor overrides typed for the selected models. @@ -52,14 +53,12 @@ fn scopes_and_cursor_overrides<'py>( )?; let selected = class_field(&scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let models_by_name = native_field(&selected, "models_by_name")?; - let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; + let model_type = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut models: Vec> = Vec::new(); for key in native_field(&selected, "selected_keys")?.try_iter()? { let key = key?; let name = native_field(&key, "name")?; - if native_field(&key, "resource_type")?.eq(&model_type)? - && models_by_name.contains(&name)? - { + if native_field(&key, "resource_type")?.eq(&model_type)? && models_by_name.contains(&name)? { models.push(models_by_name.get_item(&name)?); } } @@ -88,13 +87,13 @@ fn change_results<'py>( &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, snapshot, ), - native_field(overrides, "full_refresh")?.is_truthy()?, + native_field(&overrides, "full_refresh")?.is_truthy()?, ( native_field(&versions, "model_version_hashes")?, native_field(&versions, "model_metadata_jsons")?, ), )?; - if !native_field(policies, "selection_diagnostics")?.is_truthy()? { + if !native_field(&policies, "selection_diagnostics")?.is_truthy()? { let empty = native( py, RecordId::PlannerChangeResults, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index e9f5542fd0..a79330227f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,8 +1,8 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; @@ -10,8 +10,7 @@ use crate::bindings::_helpers::planner_entries::models::{ ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, }; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, - string, + MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, }; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::{ @@ -225,7 +224,7 @@ fn check_binding_diagnostics<'py>( selected_keys: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = project.py(); - let sql_test = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SQL_TEST")?; + let sql_test = registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_TEST")?; let mut errors: Vec> = Vec::new(); for diagnostic in native_field(project, "diagnostics")?.try_iter()? { let diagnostic = diagnostic?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 0bdb855ac3..2e0bdc8483 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -9,7 +9,6 @@ use pyo3::types::PyString; pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; -pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; /// `sqlbuild._native.name(**pairs)`. pub(super) fn native<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index fa2922368e..e1198f52f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -172,12 +172,7 @@ pub(crate) fn model_key<'py>( &[ ( "resource_type", - imported( - py, - "sqlbuild.compiler.compile.types", - "CompiledResourceType", - )? - .getattr("MODEL")?, + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?, ), ("name", name.clone()), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 4b6548cff7..f7a0043f37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -205,12 +205,7 @@ fn complete_fixtures<'py>( ("MODEL", "__ref__"), ("SEED", "__seed__"), ] { - let kind = imported( - py, - "sqlbuild.compiler.compile.types", - "CompiledResourceType", - )? - .getattr(kind)?; + let kind = registry::class(py, RecordId::CompiledResourceType)?.getattr(kind)?; groups.push(PyTuple::new( py, [kind, extract_fixtures(scenario, prefix)?.into_any()], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index d30cd41696..23ca89d791 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -203,7 +203,7 @@ pub(crate) fn audit_is_selected( .call1((kind,))? .getattr("value")?; let resource_type = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.call1((target_value,))?; + registry::class(py, RecordId::CompiledResourceType)?.call1((target_value,))?; let key = call_with( ®istry::class(py, RecordId::CompiledObjectKey)?, &[("resource_type", resource_type), ("name", name)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 68778fbf3a..e08ffec28f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -28,7 +28,7 @@ fn object_key<'py>(py: Python<'py>, kind: &str, name: &str) -> PyResult, value: &Py, (module, class): (&str, &str)) -> PyResult> { - Ok(imported(py, module, class)? - .call1((value.bind(py),))? - .unbind()) -} - /// The native string enum `class(value)`. fn coerce_native(py: Python<'_>, value: &Py, class: RecordId) -> PyResult> { Ok(registry::class(py, class)? @@ -55,7 +45,7 @@ fn coerce_resource_type(py: Python<'_>, value: &Py) -> PyResult if value.is_none(py) { return Ok(value.clone_ref(py)); } - coerce(py, value, (COMPILE_TYPES_MODULE, "CompiledResourceType")) + coerce_native(py, value, RecordId::CompiledResourceType) } /// `CompilerDiagnostic`: enum fields coerced, and its location and path, line and column aligned. @@ -105,9 +95,7 @@ pub(crate) fn diagnostic_is_error(instance: &Bound<'_, PyAny>) -> PyResult = instance.py(); let error: Bound<'_, PyAny> = registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?; - native_field(instance, "severity")? - .eq(error)? - .into_py_any(py) + native_field(&instance, "severity")?.eq(error)?.into_py_any(py) } /// `CompileSqlReference`: its kind coerced to `SqlReferenceKind`. @@ -124,11 +112,7 @@ pub(crate) fn finish_resource_typed( py: Python<'_>, mut values: Vec>, ) -> PyResult>> { - values[0] = coerce( - py, - &values[0], - (COMPILE_TYPES_MODULE, "CompiledResourceType"), - )?; + values[0] = coerce_native(py, &values[0], RecordId::CompiledResourceType)?; Ok(values) } @@ -235,17 +219,17 @@ fn reference_rows(references: &Bound<'_, PyAny>) -> PyResult> { /// `CompileModelInput.reference_rows` and `CompiledModel.reference_rows`. pub(crate) fn record_reference_rows(instance: &Bound<'_, PyAny>) -> PyResult> { - reference_rows(&native_field(instance, "references")?) + reference_rows(&native_field(&instance, "references")?) } /// The declared key facts of model `name` with `instance`'s config and schema entry. fn model_relation_key_row(instance: &Bound<'_, PyAny>, name: String) -> PyResult> { - let values: Bound<'_, PyAny> = native_field(instance, "config")?.getattr("values")?; + let values: Bound<'_, PyAny> = native_field(&instance, "config")?.getattr("values")?; entry_relation_key_row( "ref", name, values.cast::()?, - &native_field(instance, "schema_entry")?, + &native_field(&instance, "schema_entry")?, )? .into_py_any(instance.py()) } @@ -262,13 +246,13 @@ pub(crate) fn input_relation_key_row(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let name: String = native_field(instance, "name")?.extract()?; + let name: String = native_field(&instance, "name")?.extract()?; model_relation_key_row(instance, name) } /// `CompileModelInput.attached_schema_name`: the attached schema entry's name, or `None`. pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; if entry.is_none() { return Ok(entry.unbind()); } @@ -278,8 +262,8 @@ pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResul /// `(resource type, name)` of one compiled object key. fn key_row<'py>(key: &Bound<'py, PyAny>) -> PyResult<(String, String)> { Ok(( - native_field(key, "resource_type")?.str()?.extract()?, - native_field(key, "name")?.extract()?, + native_field(&key, "resource_type")?.str()?.extract()?, + native_field(&key, "name")?.extract()?, )) } @@ -290,17 +274,17 @@ fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { /// `CompiledModel.key_row`. pub(crate) fn model_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { - key_row(&native_field(instance, "key")?)?.into_py_any(instance.py()) + key_row(&native_field(&instance, "key")?)?.into_py_any(instance.py()) } /// `CompiledModel.dep_rows`. pub(crate) fn model_dep_rows(instance: &Bound<'_, PyAny>) -> PyResult> { - key_rows(&native_field(instance, "deps")?)?.into_py_any(instance.py()) + key_rows(&native_field(&instance, "deps")?)?.into_py_any(instance.py()) } /// `CompiledModel.destination_row`: `(database, schema, name)` of the destination. pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult> { - let destination: Bound<'_, PyAny> = native_field(instance, "destination")?; + let destination: Bound<'_, PyAny> = native_field(&instance, "destination")?; ( native_field(&destination, "database")?, native_field(&destination, "schema")?, @@ -311,7 +295,7 @@ pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let columns: Bound<'_, PyAny> = native_field(instance, "inferred_columns")?; + let columns: Bound<'_, PyAny> = native_field(&instance, "inferred_columns")?; if columns.is_none() { return Ok(columns.unbind()); } @@ -329,32 +313,29 @@ pub(crate) fn model_inferred_rows(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; if entry.is_none() { return Ok(entry.unbind()); } let rows: Bound<'_, PyList> = PyList::empty(instance.py()); for column in native_field(&entry, "columns")?.try_iter()? { let column: Bound<'_, PyAny> = column?; - rows.append(( - native_field(&column, "name")?, - native_field(&column, "type")?, - ))?; + rows.append((native_field(&column, "name")?, native_field(&column, "type")?))?; } Ok(rows.into_any().unbind()) } /// `CompiledModel.has_dynamic_columns`. pub(crate) fn model_has_dynamic_columns(instance: &Bound<'_, PyAny>) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; let dynamic: bool = !entry.is_none() && native_field(&entry, "dynamic_columns")?.is_truthy()?; dynamic.into_py_any(instance.py()) } /// `CompiledModel.output_column_count`: inferred, else declared, columns, else zero. pub(crate) fn model_output_column_count(instance: &Bound<'_, PyAny>) -> PyResult> { - let inferred: Bound<'_, PyAny> = native_field(instance, "inferred_columns")?; - let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; + let inferred: Bound<'_, PyAny> = native_field(&instance, "inferred_columns")?; + let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; let count: usize = if !inferred.is_none() { inferred.len()? } else if !entry.is_none() { @@ -371,20 +352,20 @@ pub(crate) fn build_context_declaration_expansion( ) -> PyResult> { let py: Python<'_> = instance.py(); let declarations: Bound<'_, PyDict> = PyDict::new(py); - declarations.set_item("enums", native_field(instance, "public_enums")?)?; - declarations.set_item("constants", native_field(instance, "public_constants")?)?; + declarations.set_item("enums", native_field(&instance, "public_enums")?)?; + declarations.set_item("constants", native_field(&instance, "public_constants")?)?; let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item( "declarations", registry::class(py, RecordId::DeclarationResolutionContext)? .call((), Some(&declarations))?, )?; - keywords.set_item("value_renderer", native_field(instance, "value_renderer")?)?; + keywords.set_item("value_renderer", native_field(&instance, "value_renderer")?)?; keywords.set_item( "collection_rendering", - native_field(instance, "collection_rendering")?, + native_field(&instance, "collection_rendering")?, )?; - keywords.set_item("resolver", native_field(instance, "declaration_resolver")?)?; + keywords.set_item("resolver", native_field(&instance, "declaration_resolver")?)?; Ok(registry::class(py, RecordId::DeclarationExpansionContext)? .call((), Some(&keywords))? .unbind()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 9bc737c7f0..f6a44b1f4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -63,6 +63,7 @@ native_ids! { ColumnLineageConfidence => "ColumnLineageConfidence", ColumnLineageEdge => "ColumnLineageEdge", ColumnLineageMode => "ColumnLineageMode", + CompiledResourceType => "CompiledResourceType", ColumnLineageSource => "ColumnLineageSource", ColumnMigrationDecision => "ColumnMigrationDecision", ColumnMigrationEvent => "ColumnMigrationEvent", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 9e2e4c3b05..66e2939713 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; const SOURCE_RESOURCE: &str = "source"; @@ -118,9 +117,7 @@ fn column_infos<'py>( } fn resource_type<'py>(py: Python<'py>, value: &str) -> PyResult> { - py.import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? - .call1((value,)) + registry::class(py, RecordId::CompiledResourceType)?.call1((value,)) } /// The evidence a model's authored contract offers, before inspection. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 8c71f4a136..f0120f2865 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -3,14 +3,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; -use std::path::Path; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::catalogue::{ @@ -18,21 +15,21 @@ use crate::bindings::_helpers::rules::catalogue::{ verify_custom_rules, }; use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; -use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; +use crate::bindings::_helpers::rules::fact_replay::project_models; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; +use crate::bindings::_helpers::records::fields::field as native_field; -const MODEL_RESOURCE_TYPE: &str = "model"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const PATHLIB_MODULE: &str = "pathlib"; /// Build the catalogue and select the rules, project slice and settings one run evaluates. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn plan_rule_run<'py>( - options: Option<&Bound<'py, PyDict>>, -) -> PyResult> { +fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { let options = Request::new("plan_rule_run", options)?; let py: Python<'py> = options.py(); let graph: Bound<'py, PyAny> = options.get("graph")?; - let project: Bound<'py, PyAny> = field(&graph, "project")?; + let project: Bound<'py, PyAny> = native_field(&graph, "project")?; freed_memory_released(py); let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(&options.get("config")?, &project)?; @@ -56,11 +53,14 @@ pub(crate) fn plan_rule_run<'py>( )?; let mut skipped_type_proof_rules: Vec> = Vec::new(); if options.get("no_sql_analysis")?.is_truthy()? - || !field(&project_attribute(&project, "settings")?, "sql_analysis")?.is_truthy()? + || !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? { let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { - let code: Bound<'py, PyAny> = field(&rule, "code")?; + let code: Bound<'py, PyAny> = native_field(&rule, "code")?; if TYPE_PROOF_RULE_CODES.contains(&code.extract::()?.as_str()) { skipped_type_proof_rules.push(code); } else { @@ -71,7 +71,7 @@ pub(crate) fn plan_rule_run<'py>( } let mut custom_rules: Vec> = Vec::new(); for rule in &selected { - if field(rule, "custom")?.is_truthy()? { + if native_field(&rule, "custom")?.is_truthy()? { custom_rules.push(rule.clone()); } } @@ -86,16 +86,19 @@ pub(crate) fn plan_rule_run<'py>( }; let mut codes: Vec> = Vec::with_capacity(selected.len()); for rule in &selected { - codes.push(field(rule, "code")?); + codes.push(native_field(&rule, "code")?); } let selection_config: Bound<'py, PyAny> = selection_rules_config(&effective_config, model_paths.as_ref(), &selected_project)?; - let selected_config: Bound<'py, PyAny> = replaced( - &selection_config, - &[ - ("select", PyTuple::new(py, codes)?.into_any()), - ("ignore", PyTuple::empty(py).into_any()), - ], + let selected_config: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?.call( + (selection_config,), + Some(&keywords( + py, + &[ + ("select", PyTuple::new(py, codes)?.into_any()), + ("ignore", PyTuple::empty(py).into_any()), + ], + )?), )?; call_with( ®istry::class(py, RecordId::RuleRunPlan)?, @@ -128,15 +131,17 @@ fn selected_model_paths<'py>( if selected_keys.is_none() { return Ok(None); } + let model_type: Bound<'py, PyAny> = + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let names: Bound<'py, PySet> = PySet::empty(py)?; for key in selected_keys.try_iter()? { let key: Bound<'py, PyAny> = key?; - if field(&key, "resource_type")?.eq(MODEL_RESOURCE_TYPE)? { - names.add(field(&key, "name")?)?; + if native_field(&key, "resource_type")?.is(&model_type) { + names.add(native_field(&key, "name")?)?; } } let paths: Bound<'py, PySet> = PySet::empty(py)?; - for model in project_models(&field(graph, "project")?)? { + for model in project_models(&native_field(&graph, "project")?)? { if names.contains(&model.name)? { paths.add(model.posix)?; } @@ -175,38 +180,43 @@ fn selection_rules_config<'py>( let Some(model_paths) = model_paths else { return Ok(config.clone()); }; + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; let model_names: Bound<'py, PySet> = PySet::empty(py)?; for model in project_models(project)? { model_names.add(model.name)?; } let mut rule_exceptions: Vec> = Vec::new(); - for entry in field(config, "rule_exceptions")?.try_iter()? { + for entry in native_field(&config, "rule_exceptions")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; - let path: String = field(&entry, "path")?.extract()?; - let path: String = String::from_utf8_lossy(&pure_path_posix(Path::new(&path))).into_owned(); + let path: Bound<'py, PyAny> = path_class + .call1((native_field(&entry, "path")?,))? + .call_method0("as_posix")?; if model_paths.contains(path)? { rule_exceptions.push(entry); } } let mut graph_edge_exceptions: Vec> = Vec::new(); - for entry in field(config, "graph_edge_exceptions")?.try_iter()? { + for entry in native_field(&config, "graph_edge_exceptions")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; - if model_names.contains(field(&entry, "consumer")?)? { + if model_names.contains(native_field(&entry, "consumer")?)? { graph_edge_exceptions.push(entry); } } - replaced( - config, - &[ - ( - "rule_exceptions", - PyTuple::new(py, rule_exceptions)?.into_any(), - ), - ( - "graph_edge_exceptions", - PyTuple::new(py, graph_edge_exceptions)?.into_any(), - ), - ], + imported(py, DATACLASSES_MODULE, "replace")?.call( + (config,), + Some(&keywords( + py, + &[ + ( + "rule_exceptions", + PyTuple::new(py, rule_exceptions)?.into_any(), + ), + ( + "graph_edge_exceptions", + PyTuple::new(py, graph_edge_exceptions)?.into_any(), + ), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 8396178b1b..a3d84bf69b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -2,11 +2,11 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; +use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -41,7 +41,7 @@ impl<'py> StageClasses<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { let types = py.import(COMPILE_TYPES_MODULE)?; let severity = registry::class(types.py(), RecordId::DiagnosticSeverity)?; - let resource = native_field(&types, "CompiledResourceType")?; + let resource = registry::class(py, RecordId::CompiledResourceType)?; let kinds = registry::class(py, RecordId::SqlReferenceKind)?; Ok(Self { diagnostic: registry::class(py, RecordId::CompilerDiagnostic)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index d1c3d30b2c..886cfeb6e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -14,15 +14,13 @@ use crate::bindings::_helpers::analysis::schema_request::{ }; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ model_inputs, project_shapes, published_shape, }; - -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +use crate::bindings::_helpers::records::fields::field as native_field; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -133,12 +131,12 @@ fn rebound_models<'py>( ) -> PyResult>> { let py: Python<'py> = project.py(); let by_name: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in native_field(&project, "models")?.try_iter()? { let model: Object<'py> = model?; by_name.set_item(native_field(&model, "name")?, &model)?; } let graph: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in native_field(&project, "models")?.try_iter()? { let model: Object<'py> = model?; let mut upstream: Vec> = Vec::new(); for reference in native_field(&model, "references")?.try_iter()? { @@ -167,9 +165,7 @@ fn rebound_models<'py>( .getattr("config")? .getattr("values")? .call_method1("get", ("sql_analysis",))?; - if !selected_keys.contains(native_field(&model, "key")?)? - || opted.is(PyBool::new(py, false)) - { + if !selected_keys.contains(native_field(&model, "key")?)? || opted.is(PyBool::new(py, false)) { continue; } let mut reads_changed: bool = false; @@ -259,19 +255,17 @@ fn validated_diagnostics<'py>( dialect: &Object<'py>, ) -> PyResult> { let py: Python<'py> = project.py(); - let functions: Object<'py> = native_field(project, "functions")?; + let functions: Object<'py> = native_field(&project, "functions")?; let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let ignores: bool = binding_catalog::ignores_quoted_case( - &native_field(project, "effective_connection")?, - dialect, - )?; + let ignores: bool = + binding_catalog::ignores_quoted_case(&native_field(&project, "effective_connection")?, dialect)?; let mut requests: Vec> = Vec::with_capacity(models.len()); for model in models { let query_sql: String = analysis_cursor_sql( py, - &native_field(model, "query_sql")?.extract::()?, + &native_field(&model, "query_sql")?.extract::()?, &model .getattr("config")? .getattr("values")? @@ -303,15 +297,15 @@ fn validated_diagnostics<'py>( expansions.set_item(native_field(&path, "stem")?, expansion)?; } let skipped: Object<'py> = PyString::new(py, BINDING_UNKNOWN_TABLE_INTERNAL_CODE).into_any(); - let types = py.import(COMPILE_TYPES_MODULE)?; let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let severity_class: Object<'py> = registry::class(py, RecordId::DiagnosticSeverity)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model_type: Object<'py> = native_field(&types, "CompiledResourceType")?.getattr("MODEL")?; + let model_type: Object<'py> = + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut diagnostics: Vec> = Vec::new(); for ((model, request), result) in models.iter().zip(&requests).zip(results.try_iter()?) { let result: Object<'py> = result?; - let expansion: Option> = expansions.get_item(native_field(model, "name")?)?; + let expansion: Option> = expansions.get_item(native_field(&model, "name")?)?; let mut mapper: Option> = None; for diagnostic in native_field(&result, "diagnostics")?.try_iter()? { let diagnostic: Object<'py> = diagnostic?; @@ -320,7 +314,7 @@ fn validated_diagnostics<'py>( } let built: Object<'py> = match mapper.take() { Some(built) => built, - None => positions(model, &native_field(request, "sql")?, expansion.as_ref())?, + None => positions(model, &native_field(&request, "sql")?, expansion.as_ref())?, }; let position: Object<'py> = native_field(&built, "position")?; mapper = Some(built); @@ -345,8 +339,8 @@ fn validated_diagnostics<'py>( ("code", native_field(&diagnostic, "code")?), ("message", native_field(&diagnostic, "message")?), ("resource_type", model_type.clone()), - ("resource_name", native_field(model, "name")?), - ("path", native_field(model, "relative_path")?), + ("resource_name", native_field(&model, "name")?), + ("path", native_field(&model, "relative_path")?), ("line", line), ("column", column), ], @@ -363,12 +357,12 @@ fn positions<'py>( expansion: Option<&Object<'py>>, ) -> PyResult> { let py: Python<'py> = model.py(); - let authored_query: Object<'py> = native_field(model, "authored_query_sql")?; + let authored_query: Object<'py> = native_field(&model, "authored_query_sql")?; let mut passes: Vec> = Vec::new(); let expanded: Object<'py> = match expansion { None => authored_query.clone(), Some(expansion) => { - for spans in native_field(expansion, "passes")?.try_iter()? { + for spans in native_field(&expansion, "passes")?.try_iter()? { let mut rows: Vec> = Vec::new(); for span in spans?.try_iter()? { let span: Object<'py> = span?; @@ -381,11 +375,11 @@ fn positions<'py>( } passes.push(PyTuple::new(py, rows)?.into_any()); } - native_field(expansion, "expanded_sql")? + native_field(&expansion, "expanded_sql")? } }; let request: Bound<'py, PyDict> = PyDict::new(py); - request.set_item("authored", native_field(model, "authored_sql")?)?; + request.set_item("authored", native_field(&model, "authored_sql")?)?; request.set_item("query", authored_query)?; request.set_item("expanded", expanded)?; request.set_item("cleaned", cleaned)?; @@ -449,10 +443,8 @@ fn placeholders<'py>(model: &Object<'py>) -> PyResult> { } fn ignores_quoted_case<'py>(project: &Object<'py>, dialect: &Object<'py>) -> PyResult> { - let ignores: bool = binding_catalog::ignores_quoted_case( - &native_field(project, "effective_connection")?, - dialect, - )?; + let ignores: bool = + binding_catalog::ignores_quoted_case(&native_field(&project, "effective_connection")?, dialect)?; Ok(PyBool::new(project.py(), ignores).to_owned().into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index eba68527af..d392e80625 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -5,14 +5,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::assembly::{ AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, }; +use crate::bindings::_helpers::records::fields::field as native_field; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ "mode", @@ -61,10 +60,9 @@ struct Classes<'py> { impl<'py> Classes<'py> { fn new(py: Python<'py>) -> PyResult { - let types: Bound<'py, PyModule> = py.import(COMPILE_TYPES_MODULE)?; Ok(Self { direct_input: registry::class(py, RecordId::CompileDirectLogicSqlTestInputPayload)?, - resource_type: native_field(&types, "CompiledResourceType")?, + resource_type: registry::class(py, RecordId::CompiledResourceType)?, phase: registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?, severity: registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, assembly: registry::class(py, RecordId::NativeSqlTestAssembly)?, @@ -130,9 +128,9 @@ fn compiled_payload<'py>( copied(payload, &DIRECT_PAYLOAD_FIELDS)?, )?; let resource_type: Bound<'py, PyAny> = classes.model(RecordId::CompiledSqlTestResource)?; - let mode: Bound<'py, PyAny> = native_field(payload, "mode")?; + let mode: Bound<'py, PyAny> = native_field(&payload, "mode")?; let mut resources: Vec> = Vec::new(); - for name in native_field(payload, "tested_resource_names")?.try_iter()? { + for name in native_field(&payload, "tested_resource_names")?.try_iter()? { resources.push(keyword_call( &resource_type, vec![("kind", mode.clone()), ("name", name?)], @@ -164,8 +162,7 @@ fn keyed_diagnostic<'py>( (line, column, end_line, end_column, message, help): DiagnosticRow, ) -> PyResult> { let py: Python<'py> = classes.py(); - let path: Bound<'py, PyAny> = - native_field(test_input, "test_file")?.getattr("relative_path")?; + let path: Bound<'py, PyAny> = native_field(&test_input, "test_file")?.getattr("relative_path")?; let key: Bound<'py, PyTuple> = PyTuple::new( py, [ @@ -214,41 +211,31 @@ fn compiled_test<'py>( .iter() .map(|(resource_type, dep)| classes.object_key(resource_type, dep)) .collect::>()?; - let (payload, payload_fields) = - compiled_payload(classes, &native_field(test_input, "payload")?)?; - let test_file: Bound<'py, PyAny> = native_field(test_input, "test_file")?; - let test_block: Bound<'py, PyAny> = native_field(test_input, "test_block")?; + let (payload, payload_fields) = compiled_payload(classes, &native_field(&test_input, "payload")?)?; + let test_file: Bound<'py, PyAny> = native_field(&test_input, "test_file")?; + let test_block: Bound<'py, PyAny> = native_field(&test_input, "test_block")?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("key", classes.object_key("sql_test", &name)?), ("scope_deps", PyTuple::new(py, scope_deps)?.into_any()), ("name", text(py, &name)), ("test_file", test_file.clone()), ("test_block", test_block.clone()), - ("sql_body", native_field(test_input, "sql_body")?), - ("mode", native_field(test_input, "mode")?), + ("sql_body", native_field(&test_input, "sql_body")?), + ("mode", native_field(&test_input, "mode")?), ("payload", payload), ("source_path", native_field(&test_file, "relative_path")?), - ( - "ownership_root", - native_field(&test_file, "ownership_root")?, - ), + ("ownership_root", native_field(&test_file, "ownership_root")?), ("block_index", native_field(&test_block, "test_index")?), ("explicit_name", native_field(&test_block, "name")?), - ("parent_name", native_field(test_input, "parent_name")?), - ("case_name", native_field(test_input, "case_name")?), - ("case_index", native_field(test_input, "case_index")?), + ("parent_name", native_field(&test_input, "parent_name")?), + ("case_name", native_field(&test_input, "case_name")?), + ("case_index", native_field(&test_input, "case_index")?), ( "case_fingerprint", case_fingerprint.into_pyobject(py)?.into_any(), ), - ( - "parameter_schema", - native_field(test_input, "parameter_schema")?, - ), - ( - "parameter_values", - native_field(test_input, "parameter_values")?, - ), + ("parameter_schema", native_field(&test_input, "parameter_schema")?), + ("parameter_values", native_field(&test_input, "parameter_values")?), ( "target_model_names", PyTuple::new(py, target_model_names)?.into_any(), @@ -297,9 +284,9 @@ pub(crate) fn assemble_native_sql_tests<'py>( py: Python<'py>, inputs: &Bound<'py, PyAny>, ) -> PyResult> { - let test_inputs: Vec> = native_field(inputs, "test_inputs")?.extract()?; + let test_inputs: Vec> = native_field(&inputs, "test_inputs")?.extract()?; let request = AssemblyRequest { - model_inputs: native_field(inputs, "model_inputs")?.extract()?, + model_inputs: native_field(&inputs, "model_inputs")?.extract()?, test_inputs: test_inputs.clone(), lexical_syntax: inputs .getattr("sql_lexical_syntax")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index f23145f868..019c313c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -3,7 +3,6 @@ pub(super) const UDF: &str = "udf"; pub(super) const TABLE_FN: &str = "table_fn"; pub(super) const ERROR_SEVERITY: &str = "error"; -pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("model", "__ref__"), ("source", "__source__"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs index ac00c76f44..cf276ef8d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::planner_fixtures::main::{ }; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_sql::{ @@ -17,8 +16,10 @@ use crate::bindings::_helpers::planner_strategy::fixture_sql::{ }; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::constants::FIXTURE_PREFIXES; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; /// Report every statically provable fixture defect or return the completed fixture SQL groups. pub(super) fn validated_fixtures<'py>( @@ -30,7 +31,7 @@ pub(super) fn validated_fixtures<'py>( let py = test.py(); let (chain, [refs, sources, seeds, expected]) = request; let adapter = inputs.1; - let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + let resource_types = registry::class(py, RecordId::CompiledResourceType)?; let model = native_field(&resource_types, "MODEL")?; let kinds = [ model.clone(), @@ -49,7 +50,7 @@ pub(super) fn validated_fixtures<'py>( let completion = complete_fixtures(context, adapter, (&chain, groups.as_any()))?; let mut errors: BTreeSet = BTreeSet::new(); for diagnostic in &completion.diagnostics { - let key = native_field(diagnostic, "key")?; + let key = native_field(&diagnostic, "key")?; errors.insert(format!( "{}: {}", fixture_location( @@ -57,7 +58,7 @@ pub(super) fn validated_fixtures<'py>( &key.get_item(0)?, &key.get_item(1)?.extract::()? )?, - native_field(diagnostic, "message")?.extract::()? + native_field(&diagnostic, "message")?.extract::()? )); } let inferred_by_fixture = &completion.inferred_by_fixture; @@ -141,7 +142,7 @@ pub(super) fn validated_fixtures<'py>( let label = if expected_output { "expected output".to_owned() } else { - format!("mock {}", native_field(kind, "value")?.extract::()?) + format!("mock {}", native_field(&kind, "value")?.extract::()?) }; errors.insert(format!( "{}: {label} '{name}' column '{}' has incompatible type {}; resource type is {}. Use an explicit CAST, ARRAY_CONSTRUCT, or PARSE_JSON as appropriate", @@ -154,7 +155,7 @@ pub(super) fn validated_fixtures<'py>( } } if !errors.is_empty() { - let test_name: String = native_field(test, "name")?.extract()?; + let test_name: String = native_field(&test, "name")?.extract()?; return Ok(Err(errors .into_iter() .map(|error| format!("SQL test '{test_name}': {error}")) @@ -184,12 +185,12 @@ pub(super) fn fixture_location( resource_type: &Bound<'_, PyAny>, name: &str, ) -> PyResult { - let value: String = native_field(resource_type, "value")?.extract()?; + let value: String = native_field(&resource_type, "value")?.extract()?; let prefix = FIXTURE_PREFIXES .iter() .find_map(|(kind, prefix)| (*kind == value).then_some(*prefix)) .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value))?; - let test_file = native_field(test, "test_file")?; + let test_file = native_field(&test, "test_file")?; let line = fixture_line( &native_field(&test_file, "contents")?.extract::()?, &format!("{prefix}{name}"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index e2c7fb12fd..29da1d46bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,28 +3,23 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_TYPES; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; +use crate::bindings::_helpers::records::fields::field as native_field; fn prefixed_names<'py>(test: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { let py = test.py(); let mut names: Vec = if model_payload(test)? { - prefixed_ctes( - &native_field(test, "payload")?.getattr("authored_ctes")?, - prefix, - )? - .keys() - .extract()? + prefixed_ctes(&native_field(&test, "payload")?.getattr("authored_ctes")?, prefix)? + .keys() + .extract()? } else { Vec::new() }; @@ -41,7 +36,7 @@ fn function_deps<'py>( let py = test.py(); let deps = PyDict::new(py); if model_payload(test)? { - let names: Vec = native_field(plan, "model_names")?.extract()?; + let names: Vec = native_field(&plan, "model_names")?.extract()?; let mocked: std::collections::HashSet = inputs.0.extract::>()?.into_iter().collect(); let models = inputs.1; @@ -51,9 +46,7 @@ fn function_deps<'py>( }; for dep in native_field(&model, "deps")?.try_iter()? { let dep = dep?; - let resource: String = native_field(&dep, "resource_type")? - .getattr("value")? - .extract()?; + let resource: String = native_field(&dep, "resource_type")?.getattr("value")?.extract()?; if resource != UDF && resource != TABLE_FN { continue; } @@ -66,17 +59,15 @@ fn function_deps<'py>( } } } else { - let payload = native_field(test, "payload")?; - let mode: String = native_field(&payload, "mode")? - .getattr("value")? - .extract()?; + let payload = native_field(&test, "payload")?; + let mode: String = native_field(&payload, "mode")?.getattr("value")?.extract()?; if mode == UDF || mode == TABLE_FN { let names: std::collections::HashSet = payload .getattr("tested_resource_names")? .extract::>()? .into_iter() .collect(); - for function in native_field(project, "functions")?.try_iter()? { + for function in native_field(&project, "functions")?.try_iter()? { let function = function?; let key = native_field(&function, "key")?; if key @@ -133,8 +124,8 @@ pub(super) fn entry<'py>( "function_deps", function_deps(test, plan, inputs.0, (&mocks, inputs.1))?, )?; - fields.set_item("chain", native_field(plan, "chain")?)?; - fields.set_item("assertions", native_field(plan, "assertions")?)?; + fields.set_item("chain", native_field(&plan, "chain")?)?; + fields.set_item("assertions", native_field(&plan, "assertions")?)?; fields.set_item("sql_analysis_enabled", analysis)?; registry::class(py, RecordId::SqlTestPlanEntry)?.call((), Some(&fields)) } @@ -149,8 +140,8 @@ pub(super) fn expected_diagnostics( return Ok(result); }; let py = test.py(); - let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; - for step in native_field(plan, "chain")?.try_iter()? { + let resource_types = registry::class(py, RecordId::CompiledResourceType)?; + for step in native_field(&plan, "chain")?.try_iter()? { let step = step?; let name = native_field(&step, "model_name")?; let model_name: String = name.extract()?; @@ -178,7 +169,7 @@ pub(super) fn expected_diagnostics( )?; result.push(format!( "SQL test '{}': {}: {}", - native_field(test, "name")?.extract::()?, + native_field(&test, "name")?.extract::()?, location, message )); diff --git a/scripts/cold_compile_performance/_helpers/varied_statistics.py b/scripts/cold_compile_performance/_helpers/varied_statistics.py index 2068451e3d..7bb09e1fb2 100644 --- a/scripts/cold_compile_performance/_helpers/varied_statistics.py +++ b/scripts/cold_compile_performance/_helpers/varied_statistics.py @@ -8,7 +8,7 @@ VARIED_WINDOW_NODE_FRAGMENT, ) from scripts.cold_compile_performance.models import VariedProjectStatistics -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledResourceType from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _UDF_REFERENCE: re.Pattern[str] = re.compile(r'__udf\("([a-zA-Z0-9_]+)"\)') diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ca768e90bf..8cbcaa2e99 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -53,7 +53,6 @@ from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ExternalSqlReferenceResolver, @@ -10481,6 +10480,17 @@ class DiagnosticSeverity(StrEnum): WARNING = "warning" INFO = "info" +class CompiledResourceType(StrEnum): + MODEL = "model" + SOURCE = "source" + SEED = "seed" + UDF = "udf" + TABLE_FN = "table_fn" + DBT_REF = "dbt_ref" + AUDIT = "audit" + SQL_TEST = "sql_test" + SQL_SCENARIO = "sql_scenario" + class InferredNullability(StrEnum): """Conservative nullability state inferred for an output column.""" diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 1464e00c2b..b7574a6e37 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -711,7 +711,6 @@ "AuditSeverity": ("sqlbuild.compiler.auditing.types", "AuditSeverity"), "ThresholdOperator": ("sqlbuild.compiler.auditing.types", "ThresholdOperator"), "AttachedAuditTargetKind": ("sqlbuild.compiler.compile.types", "AttachedAuditTargetKind"), - "CompiledResourceType": ("sqlbuild.compiler.compile.types", "CompiledResourceType"), "FunctionLanguage": ("sqlbuild.compiler.compile.types", "FunctionLanguage"), "SqlTestMode": ("sqlbuild.compiler.compile.types", "SqlTestMode"), "CursorAlgebraError": ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"), diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index d71a52e47d..1e53f628e7 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -103,7 +103,6 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import ( macro_invocation_context as macro_invocation_context, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind as AttachedAuditTargetKind -from sqlbuild.compiler.compile.types import CompiledResourceType as CompiledResourceType from sqlbuild.compiler.compile.types import FunctionLanguage as FunctionLanguage from sqlbuild.compiler.compile.types import SqlTestMode as SqlTestMode from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index f1eac540c4..f2f1b9d938 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -379,18 +379,6 @@ class AttachedAuditTargetKind(StrEnum): SEED = "seed" -class CompiledResourceType(StrEnum): - MODEL = "model" - SOURCE = "source" - SEED = "seed" - UDF = "udf" - TABLE_FN = "table_fn" - DBT_REF = "dbt_ref" - AUDIT = "audit" - SQL_TEST = "sql_test" - SQL_SCENARIO = "sql_scenario" - - class FunctionLanguage(StrEnum): SQL = "sql" PYTHON = "python" diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index 2548cd8384..b43fa5aa2d 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,10 +4,15 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput, audit_resource_id +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledResourceType, + PlanOutput, + audit_resource_id, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 2d8fad6bcb..8eb61acfeb 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( AuditPlanEntry, CompiledObjectKey, + CompiledResourceType, CostContext, DiscoveredLoaderFunction, FunctionPlanEntry, @@ -38,7 +39,6 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index bb8df9c6ab..d185c972c0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -4,6 +4,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, RelationFixtureCompletion, RelationFixturePlanningContext, build_compiled_project, @@ -12,7 +13,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( FixtureTypeScopeTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 40739c6815..ebfacda47b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -37,7 +38,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 8a7992d288..9936f3fb86 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -14,6 +14,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileSqlReference, CursorOverridePair, @@ -32,7 +33,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 4a542a445f..418c09cced 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -10,6 +10,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, @@ -20,7 +21,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index f174224260..41d31d176e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -11,6 +11,7 @@ CompiledDirectLogicSqlTestPayload, CompiledObjectKey, CompiledProject, + CompiledResourceType, CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, @@ -20,7 +21,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 6c223c9ada..dd4fe0765c 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -10,13 +10,14 @@ CompiledAudit, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, plan_audit, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 98f64d28f0..f60d8ca0e2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, DiscoveredProjectInputs, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -18,7 +19,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index 475af48f3e..873ec2eb22 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CursorSnapshotScope, ModelCursorSnapshot, PlannerInputError, @@ -15,7 +16,7 @@ ) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, EligibleMaxFailureTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index b82f927619..5488189925 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorOverrides, CursorSnapshotScope, Fingerprint, @@ -29,7 +30,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( GatherCursorSnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 803a661c0b..3067f5ba66 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -18,6 +18,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -54,7 +55,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, FunctionLanguage, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 3426afc360..09608926ef 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -215,8 +215,7 @@ def extract_upstream_model_deps( ) -> dict[str, frozenset[str]]: """Extract upstream model dependency names for each model in the plan.""" - from sqlbuild._native import CompiledObjectKey - from sqlbuild.compiler.compile.types import CompiledResourceType + from sqlbuild._native import CompiledObjectKey, CompiledResourceType dependencies_by_resource_type: dict[object, dict[str, frozenset[str]]] = { resource_type: {} for resource_type in CompiledResourceType diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 4982244da4..bf06f2c529 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -10,6 +10,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -23,7 +24,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( MaterializationContext, diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index 58662db772..208a979a2a 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -24,7 +25,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 4a0f56e9b0..e87555169d 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -28,7 +29,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 7a19c2c803..ce1b74633e 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -13,6 +13,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -28,7 +29,6 @@ AuditAttachmentKind, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.microbatch import execute_microbatch_entry from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index 25b4a0e41c..3800e8aa8a 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -10,6 +10,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -22,7 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index d8e679f0c1..afe96ebcd5 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -20,7 +21,6 @@ ScenarioRelationMap, ScenarioRelationPlan, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import HookContext SCENARIO_NAME: str = "revenue__customer_refund" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index 1c5a05a9fc..997653f119 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, PythonHookEntry, ScenarioExecutionPlan, @@ -17,7 +18,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 6fcc3ecd2c..e896068c20 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -9,12 +9,12 @@ from sqlbuild._native import ( ChainStep, CompiledObjectKey, + CompiledResourceType, CompilePipelineResult, SqlTestPlanEntry, render_sql_test_comparison_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index e8940a0343..c79c282750 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -17,7 +18,6 @@ ) from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.types import CompiledResourceType def build_snapshot_full_refresh_entry( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 5c431c8e83..b045f5f8c8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -7,6 +7,7 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CostContext, MaterializationType, ModelPlanEntry, @@ -19,7 +20,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py index 1f72c96b35..45ce23fc8c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,7 +1,6 @@ from __future__ import annotations -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType def build_compiled_object_key( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index 3b87f9458a..e0a17839fc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -8,11 +8,12 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, PlannerScope, resolve_defer_clone_selection, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBoundaryTestCase, DeferCloneModelCountTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index e0e8a179d9..17410ca9a1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, FunctionArgument, FunctionPlanEntry, @@ -40,7 +41,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from sqlbuild.executor.build.models import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 3535e143f2..69c9c917f2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompileLineageMode, CompileModelConfig, ProjectColumnLineage, @@ -19,7 +20,7 @@ format_compile_text, project_with, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, CompileJsonLargeIntegerTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 908f900469..a212eaa5e2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -12,6 +12,7 @@ import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( ChainStep, + CompiledResourceType, CompiledSqlTest, DiagnosticPhase, PlanOutput, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 53b04a222f..6b1431bc9e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -12,11 +12,11 @@ from sqlbuild import _native as native from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, DiscoveredSqlScenarioFile, ) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index b9554f7905..6c7eb7748f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -9,6 +9,7 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -23,7 +24,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index de19758d14..fa879d11af 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -10,6 +10,7 @@ ColumnRenameHint, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredProviderUsage, @@ -32,7 +33,7 @@ TableTypePlanEntry, WarningSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import FunctionLanguage def build_model_entry( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index ba2277d229..6412cc52f2 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -11,13 +11,13 @@ BuildProgressCallbacks, ChainStep, CompiledObjectKey, + CompiledResourceType, NativeProgressProjector, PlanOutput, SqlTestPlanEntry, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 5d7f7b34ca..5d835ec7c9 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, SchemaAuditInstance, SourceEntry, parse_audit_instances, @@ -22,7 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType def build_render_adapter(adapter_name: str | None) -> BaseAdapter | None: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 5f7b1a7378..a48010ff22 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -18,6 +18,7 @@ CompiledModelSqlTestPayload, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompilerDiagnostic, CompileSqlReference, @@ -67,7 +68,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) if TYPE_CHECKING: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index cf62a2121f..7f0c985e35 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredProjectInputs, SqlHookEntry, assemble_project, @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, - CompiledResourceType, CompiledSeedView, ) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 8ce9866e21..4af195671d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, SqlLogicalType, SqlValue, @@ -21,7 +22,6 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( CompiledProjectView, - CompiledResourceType, TypedSqlValueRenderer, ) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index 7c3ffd22c2..43ee0aa579 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( CompiledLineageSourceFact, + CompiledResourceType, CompileSqlReference, SqlReferenceKind, analyze_resolved_column_reads, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( LongUnionRequiredColumnsTestCase, RequiredExternalColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 4d4345dc32..0803b86221 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -7,6 +7,7 @@ ColumnTransformKind, CompiledLineageColumnFact, CompiledLineageSourceFact, + CompiledResourceType, CompileSqlReference, InferredColumn, InferredNullability, @@ -17,7 +18,6 @@ substitute_placeholder_defaults, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( InferColumnsTestCase, PolyglotAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 3aed31fcf7..f9da29641b 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -1,6 +1,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, ContractEvidence, SchemaDynamicColumnFamily, compare_contract_evidence, @@ -8,7 +9,7 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( DynamicContractAdoptionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index 0275c794c9..2f9e482c04 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DynamicColumnContractProof, InferredColumn, @@ -20,7 +21,7 @@ SettingsConfig, SourceLocation, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView def make_contract_project( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index f6a4f748c2..2dba9df49e 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompiledSqlScenario, @@ -35,7 +36,6 @@ from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 4d55aeb1a8..11e965d1e6 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompileModelConfig, DiscoveredSchemaFile, @@ -22,7 +23,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, ) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 3337fea7b2..00b3ccd5b2 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -11,6 +11,7 @@ ColumnTransformKind, CompiledLineageColumnFact, CompiledLineageSourceFact, + CompiledResourceType, ModelColumnLineage, ProjectColumnLineage, compiled_model_record, @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.lineage._test_types import ( ColumnLineageAnalyzerTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 5d68f33411..f25daaf0b7 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -43,7 +44,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 0271fefc1c..2bdbe2429c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -10,13 +10,13 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, Fingerprint, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index 85eb362f4b..4a5832323b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -10,10 +10,11 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView class ConservativeCustomAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 6a88987264..bf24e9cfa0 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -7,6 +7,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSource, DiscoveredSourceFile, MaterializationType, @@ -19,7 +20,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView class RelationTargetTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index b8e04c4b3a..0110eb2454 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, Fingerprint, FunctionArgument, @@ -30,7 +31,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 7e8ae4ca2b..f6dd035585 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -24,6 +24,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompiledSqlScenario, @@ -77,7 +78,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, SqlTestMode, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index cd0c4d1278..3adc1d2436 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DeferralInputs, Fingerprint, @@ -25,7 +26,7 @@ plan_column_migrations, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index aa4ed448c9..84ff684bb7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, LocalConfig, PlannerRelationsContext, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 101fbd5829..2abbe1a717 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -8,12 +8,13 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ModelPlanContext, SourceEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> CompiledModelView: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 28b25f0c39..1c5a9349c5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, @@ -19,7 +20,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index cfe2db5d8b..f1d3459bcc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, @@ -42,7 +43,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index b79cb1a6ec..4f1b63bd88 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, @@ -39,7 +40,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 826bc5e16d..7f41f096e5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -9,6 +9,7 @@ from sqlbuild import _native as fixture_completion from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, CompileSqlReference, PlanWarning, @@ -25,7 +26,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index de3d39f92b..69561d341c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileInputError, CursorBounds, CursorInputRelation, @@ -22,7 +23,6 @@ apply_microbatch_limit, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, EffectiveMicrobatchLimitPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index f78652a500..d0f0444e57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,8 +2,11 @@ import pytest -from sqlbuild._native import CompiledObjectKey, build_downstream_model_name_closure -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import ( + CompiledObjectKey, + CompiledResourceType, + build_downstream_model_name_closure, +) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase SOURCE_ORDERS: CompiledObjectKey = CompiledObjectKey( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 878a03936b..2f865bed7e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -15,6 +15,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CursorBounds, CursorInputRelation, @@ -24,7 +25,7 @@ compute_plan_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index 060193c5c8..08c1098c9c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -6,12 +6,12 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, ScenarioGraphPlan, WarningSeverity, plan_scenario_graph, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, PlanScenarioGraphTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index b727f99bde..00c40992b2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -11,6 +11,7 @@ ChangeKind, CompiledObjectKey, CompiledProject, + CompiledResourceType, DirectModelVersionIdentities, DirectSourceFreshnessPlanningResult, Fingerprint, @@ -30,7 +31,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 3c1c5c141e..6069d383cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledProject, + CompiledResourceType, CursorInputRelation, PlannerInputError, PlannerRelationsContext, @@ -28,7 +29,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index fa112e1b48..c07609435c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -12,6 +12,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DiscoveredLoaderFunction, SourceEntry, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 9172b003b0..a564cb4f92 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledProject, + CompiledResourceType, CompiledSource, DiscoveredSourceFile, PlannerRelationsContext, @@ -18,7 +19,7 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView +from sqlbuild.compiler.compile.types import CompiledSourceView def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 093e7c4ddb..6a71ddcb5e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -6,13 +6,13 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, PlannedSqlTests, PlannerInputError, SqlTestPlanning, SqlTestSelection, start_sql_test_planning, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 0b5ca170fb..f0a04e69af 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -11,6 +11,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ModelPlanEntry, PlanAction, @@ -19,7 +20,7 @@ build_display_only_sqlbuild_plan, format_plan, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( DisplayFullRefreshTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 1af69db657..e68966d93b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -10,6 +10,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DeferralInputs, Fingerprint, @@ -30,7 +31,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 41b5526012..cc92e03d8a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -8,6 +8,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DirectModelVersionIdentities, FunctionArgument, @@ -23,7 +24,6 @@ ) from sqlbuild.compiler.compile.types import ( CompiledModelView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 3cc6ca8e30..71ebdbb10b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CompiledFunction, CompiledObjectKey, + CompiledResourceType, DirectModelVersionIdentities, FunctionReturnColumn, build_function_local_hashes, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( FunctionReturnContractIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index fb2b8b4c07..22a14412f2 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -15,6 +15,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSource, CompileModelConfig, DiscoveredAssetFunction, @@ -38,7 +39,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSourceView, ) from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 4b7d565d33..09196671c4 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -6,12 +6,12 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, PythonNodeGraph, PythonSqlRunLifecyclePlan, PythonSqlRunSelection, build_python_sql_run_lifecycle, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 10ff692c35..cb789cef89 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( SOURCE_FRESHNESS_TABLE_NAME, CompiledObjectKey, + CompiledResourceType, SourceEntry, SourceFreshnessConfig, SourceFreshnessIdentity, @@ -27,7 +28,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType def state_table_exists_map( diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index 9820c3f510..7db523e4b5 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( AuditPlanEntry, CompiledObjectKey, + CompiledResourceType, MeasurementThresholdBound, MeasurementThresholds, ) @@ -20,7 +21,6 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index c5144c16ce..2e2800504b 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredLoaderFunction, MaterializationType, ModelPlanEntry, @@ -29,7 +30,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index d50712544f..7c3c246bf8 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -8,13 +8,13 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredLoaderFunction, LoaderConnectionMode, PlanOutput, SourceEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py index fb682d78a3..be9bbf98d6 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, CompiledResourceType, PlanOutput from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.executor.build._helpers.indexes import build_execution_indexes from sqlbuild.executor.build.models import BuildIndexes from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index 45fa8be653..a573ae34f9 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredHookFunction, DiscoveredLoaderFunction, ModelPlanEntry, @@ -25,7 +26,6 @@ ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 669a32dc4c..1d1d63143e 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import CompiledObjectKey, PlanOutput +from sqlbuild._native import CompiledObjectKey, CompiledResourceType, PlanOutput from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py index dae1f86f97..442de8eb58 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import BuildCallbacks, SchedulerState from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index c2c9626b84..573cdf857d 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -9,8 +9,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import NodeCompletion from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py index fdd2c9506c..c1832b4d61 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.classes.build_scheduler import _test_result_authored_order from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 18a2a6aa22..bd12b224da 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -9,6 +9,7 @@ FINGERPRINT_TABLE_NAME, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, Fingerprint, FingerprintSet, MaterializationType, @@ -19,7 +20,6 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.types import CompiledResourceType class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index cf53cd8e3b..5785b253de 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -8,6 +8,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, FunctionPlanEntry, MaterializationType, @@ -22,7 +23,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index ff95e704a1..0adb02c07b 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -16,6 +16,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -36,7 +37,6 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.types import ( CompiledProjectView, - CompiledResourceType, CompiledSourceView, ) from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 1ed9f9368a..4b292bd609 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlScenario, CompilePipelineResult, DiscoveredSqlScenarioFile, @@ -34,7 +35,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 09d1b648da..5c02f6ec76 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -22,6 +22,7 @@ AuditPlanEntry, AuditProgressReporter, CompiledObjectKey, + CompiledResourceType, CostContext, PlanOutput, StatementLedgerEntry, @@ -37,7 +38,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index f3eb21e27d..f4521e26ec 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -11,11 +11,11 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, PlanOutput, SqlTestPlanEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers import testing as testing_pipeline from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline from sqlbuild.executor.pipeline.models import TestPipelineCallbacks as PipelineCallbacks diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index aa1c4745d4..e0bf2cfe63 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, FunctionPlanEntry, MaterializationType, ModelPlanEntry, @@ -19,7 +20,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py index 1118c704ba..05e84836d5 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py @@ -7,13 +7,13 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, PlanOutput, PlanReason, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers.graph_width import runnable_graph_width from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 37a09440f3..e12691760d 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -6,6 +6,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -28,7 +29,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 34da88d5e4..a597ba1243 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorGrain, CursorInputRelation, @@ -31,7 +32,6 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py index d9c4caf2ae..306e9ae570 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, HistoricalInput, MaterializationType, ModelPlanEntry, @@ -18,7 +19,6 @@ PlanReason, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index 8d88deaef8..8dbd2bc5f1 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -21,7 +22,6 @@ ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCaptureRunResult, ScenarioSnapshotColumn, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 16e0263698..a97f517c5a 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -23,7 +24,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index 1f2a173ddf..29a01a85a1 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, SeedCsvSettings, SeedPlanEntry, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType def build_seed_plan_entry(*, seed_name: str, file_path: Path) -> SeedPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index 2dc0788dde..0fb1a43388 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,10 +1,15 @@ -from sqlbuild._native import ChainStep, CompiledObjectKey, SqlTestAssertionStep, SqlTestPlanEntry +from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + CompiledResourceType, + SqlTestAssertionStep, + SqlTestPlanEntry, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType def build_comparison_test_entry(*, sql_analysis_enabled: bool = True) -> SqlTestPlanEntry: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 9d88c15421..984791a758 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -13,6 +13,7 @@ BackfillResult, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileModelInput, CompileProjectInputs, @@ -39,7 +40,7 @@ extract_sql_references, transitive_closure, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 0b72c88ed7..802272118d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -15,6 +15,7 @@ CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, @@ -37,7 +38,7 @@ ) from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 92cac5b2f2..e6852906cd 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -5,12 +5,13 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, CompiledSqlTest, DiscoveredSqlScenarioFile, project_with, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index d3ce2e7515..ffa490ec3f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -3,10 +3,11 @@ from sqlbuild._native import ( CompiledAudit, CompiledObjectKey, + CompiledResourceType, DiscoveredAuditBlock, DiscoveredAuditFile, ) -from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView def build_audit(*, name: str, attached_target_name: str | None) -> CompiledAuditView: diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 4c888d539f..b2ae63b01c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -7,6 +7,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileSqlReference, EnumDeclaration, @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration( From 30b38064f3ebbf76644a733925a1b01f519a0e90 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:18:16 +0100 Subject: [PATCH 09/11] refactor: read native record fields directly in the compile driver and merge lane updates --- .../analysis_session/stage_functions.rs | 32 ++-- .../_helpers/build_command/defer_clone.rs | 96 +++++++----- .../bindings/_helpers/check/check_command.rs | 68 ++++---- .../bindings/_helpers/clone/clone_pipeline.rs | 15 +- .../compile_attachment/declaration_scope.rs | 17 +- .../compile_attachment/model_config.rs | 22 ++- .../_helpers/compile_driver/analysis.rs | 70 +++++---- .../_helpers/compile_driver/assembly.rs | 24 +-- .../compile_driver/assembly_analysis.rs | 44 +++--- .../compile_driver/assembly_checks.rs | 56 ++++--- .../_helpers/compile_driver/assembly_facts.rs | 14 +- .../compile_driver/assembly_models.rs | 36 +++-- .../compile_driver/assembly_project.rs | 71 +++++---- .../compile_driver/assembly_sql_tests.rs | 19 ++- .../_helpers/compile_driver/command.rs | 84 +++++----- .../compile_driver/config_validation.rs | 9 +- .../_helpers/compile_driver/deferral.rs | 32 ++-- .../compile_driver/diagnostic_collector.rs | 17 +- .../bindings/_helpers/compile_driver/diff.rs | 14 +- .../_helpers/compile_driver/discovery.rs | 7 +- .../_helpers/compile_driver/inputs.rs | 69 +++++--- .../compile_driver/integration_loaders.rs | 19 ++- .../_helpers/compile_driver/phases.rs | 39 ++--- .../_helpers/compile_driver/pipeline.rs | 83 ++++++---- .../compile_driver/pipeline_entries.rs | 29 ++-- .../_helpers/compile_driver/plan_result.rs | 42 ++--- .../_helpers/compile_driver/project.rs | 23 +-- .../compile_driver/python_plan_entries.rs | 51 +++--- .../compile_driver/relation_targets.rs | 21 +-- .../compile_driver/selection_phases.rs | 75 ++++++--- .../compile_driver/target_defaults.rs | 30 ++-- .../_helpers/compile_driver/targets.rs | 61 ++++---- .../_helpers/compiled_project/source_views.rs | 2 +- .../_helpers/contract_command/command.rs | 60 +++---- .../_helpers/contract_command/write.rs | 12 +- .../bindings/_helpers/cost_command/refresh.rs | 27 ++-- .../_helpers/dbt_interop/pipeline_prologue.rs | 36 +++-- .../_helpers/discovery/imported_modules.rs | 2 +- .../_helpers/discovery/imported_providers.rs | 10 +- .../_helpers/format_command/command.rs | 9 +- .../_helpers/format_command/selection.rs | 16 +- .../_helpers/freshness/freshness_command.rs | 53 ++++--- .../bindings/_helpers/freshness/planning.rs | 23 +-- .../_helpers/freshness/publication.rs | 22 +-- .../_helpers/graph/execution_gates.rs | 8 +- .../_helpers/graph/python_node_graph.rs | 20 +-- .../_helpers/graph/python_selection.rs | 18 ++- .../_helpers/hand_built/compiled_model.rs | 44 +++--- .../_helpers/janitor/janitor_phases.rs | 30 ++-- .../_helpers/planner_changes/actions.rs | 10 +- .../_helpers/planner_changes/detect.rs | 43 ++--- .../planner_entries/execution_plan.rs | 10 +- .../_helpers/planner_identity/identities.rs | 4 +- .../_helpers/planner_phases/identities.rs | 14 +- .../planner_phases/output_assembly.rs | 30 ++-- .../_helpers/planner_scope/loaders.rs | 12 +- .../bindings/_helpers/planner_scope/scope.rs | 12 +- .../_helpers/planner_selection/selectors.rs | 6 +- .../planner_strategy/fixture_context.rs | 2 +- .../planner_strategy/fixture_reads.rs | 2 +- .../planner_warehouse/fingerprint_cache.rs | 2 +- .../_helpers/record_behaviours/compiled.rs | 51 +++--- .../_helpers/record_behaviours/discovery.rs | 4 +- .../bindings/_helpers/rules/fix_session.rs | 148 +++++++++++------- .../bindings/_helpers/rules/lint_expansion.rs | 2 +- .../src/bindings/_helpers/rules/rule_plan.rs | 90 +++++------ .../bindings/_helpers/scopes/scope_loading.rs | 2 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../_helpers/semantic_shapes/declarations.rs | 2 +- .../semantic_shapes/source_bindings.rs | 42 ++--- .../_helpers/sql_test_glue/compiled_tests.rs | 45 ++++-- .../sql_test_glue/fixture_validation.rs | 14 +- .../_helpers/sql_test_glue/plan_driver.rs | 6 +- .../_helpers/sql_test_glue/plan_entries.rs | 33 ++-- 74 files changed, 1307 insertions(+), 962 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 30dd166a28..0239b771e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -23,10 +23,10 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::stage_work::{ EXPRESSION_SHAPE_BATCHES, EXPRESSION_SHAPES_INFERRED, count_stage, }; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::models::ProjectCatalog; -use crate::bindings::_helpers::records::fields::field as native_field; const BATCH_MODE: &str = "batch"; const TABLE_FUNCTION_KIND: &str = "table_fn"; @@ -204,7 +204,7 @@ pub(crate) fn dynamic_contract_proofs<'py>( rows.push((sql, family_rows(&families)?)); } let contracts: Vec = if session.is_none() { - let by_table: Bound<'py, PyAny> = native_field(&tables, "dynamic_families_by_table")?; + let by_table: Bound<'py, PyAny> = native_field(tables, "dynamic_families_by_table")?; let mut families: Vec<(String, Vec)> = Vec::new(); for item in by_table.call_method0("items")?.try_iter()? { let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; @@ -213,10 +213,13 @@ pub(crate) fn dynamic_contract_proofs<'py>( prove_dynamic_column_contracts( py, ( - dialect_or_generic(&native_field(&tables, "dialect")?)?, - shape_rows(&native_field(&tables, "column_types_by_table")?)?, - shape_rows(&native_field(&tables, "authoritative_column_types_by_table")?)?, - shape_rows(&native_field(&tables, "column_nullability_by_table")?)?, + dialect_or_generic(&native_field(tables, "dialect")?)?, + shape_rows(&native_field(tables, "column_types_by_table")?)?, + shape_rows(&native_field( + tables, + "authoritative_column_types_by_table", + )?)?, + shape_rows(&native_field(tables, "column_nullability_by_table")?)?, families, rows, ), @@ -273,13 +276,16 @@ pub(crate) fn query_column_rows( let dialect: String = aliased_dialect(py, normalization_dialect.clone())?; let mut batch: bool = false; for query in &queries { - if native_field(&query, "mode")?.eq(BATCH_MODE)? { + if native_field(query, "mode")?.eq(BATCH_MODE)? { batch = true; break; } } let catalog: Option> = if batch { - Some(native_field(&profile_catalog(profile, &dialect)?, "native")?) + Some(native_field( + &profile_catalog(profile, &dialect)?, + "native", + )?) } else { None }; @@ -307,7 +313,7 @@ pub(crate) fn query_column_rows( } fn query_row(query: &Bound<'_, PyAny>) -> PyResult { - let placeholders: Bound<'_, PyAny> = native_field(&query, "placeholders")?; + let placeholders: Bound<'_, PyAny> = native_field(query, "placeholders")?; let placeholders: Pairs = if placeholders.is_truthy()? { placeholders .call_method0("items")? @@ -319,7 +325,7 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { }; let mut analysis_names: Vec = Vec::new(); let mut lineage: Vec<(String, String, String)> = Vec::new(); - for reference in native_field(&query, "references")?.try_iter()? { + for reference in native_field(query, "references")?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; let kind: String = native_field(&reference, "ref_kind")?.extract()?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -344,12 +350,12 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { } } Ok(( - native_field(&query, "sql")?.extract()?, + native_field(query, "sql")?.extract()?, placeholders, analysis_names, lineage, - native_field(&query, "recover_cte_facts")?.extract()?, - native_field(&query, "mode")?.extract()?, + native_field(query, "recover_cte_facts")?.extract()?, + native_field(query, "mode")?.extract()?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index ece5d008ae..147b8f8056 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -17,9 +17,9 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const VIEW_MATERIALIZATION: &str = "view"; const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; @@ -42,7 +42,9 @@ fn user_error(py: Python<'_>, message: String, code: &str) -> PyErr { } fn resource_type(key: &Bound<'_, PyAny>) -> PyResult { - native_field(&key, "resource_type")?.getattr("value")?.extract() + native_field(key, "resource_type")? + .getattr("value")? + .extract() } /// Functions and view models are recreated over cloned boundaries instead of cloned. @@ -52,7 +54,7 @@ fn recreated_over_clones(scope: &Bound<'_, PyAny>, key: &Bound<'_, PyAny>) -> Py } let model: Bound<'_, PyAny> = scope .getattr("models_by_name")? - .call_method1("get", (native_field(&key, "name")?,))?; + .call_method1("get", (native_field(key, "name")?,))?; if model.is_none() { return Ok(false); } @@ -108,8 +110,8 @@ fn traversed_names<'py>( let keys: Bound<'py, PyAny> = call_with( ®istry::native(py, function)?, &[ - ("selected", native_field(&scope, "selected_keys")?), - ("upstream", native_field(&scope, "upstream_deps")?), + ("selected", native_field(scope, "selected_keys")?), + ("upstream", native_field(scope, "upstream_deps")?), ("is_clonable", is_clonable.clone()), ("is_view", is_view.clone()), ], @@ -124,9 +126,9 @@ fn traversed_names<'py>( /// Selected models except those explicitly disabled. fn selected_executable_model_count(scope: &Bound<'_, PyAny>) -> PyResult { - let models: Bound<'_, PyAny> = native_field(&scope, "models_by_name")?; + let models: Bound<'_, PyAny> = native_field(scope, "models_by_name")?; let mut count: usize = 0; - for key in native_field(&scope, "selected_keys")?.try_iter()? { + for key in native_field(scope, "selected_keys")?.try_iter()? { let key: Bound<'_, PyAny> = key?; if resource_type(&key)? != MODEL_TYPE { continue; @@ -218,8 +220,9 @@ fn clone_input<'py>( on_item: Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = context.py(); - let project: Bound<'py, PyAny> = native_field(&pipeline, "destination_project")?; - let destination_models: Bound<'py, PyAny> = native_field(&pipeline, "destination_model_entries")?; + let project: Bound<'py, PyAny> = native_field(pipeline, "destination_project")?; + let destination_models: Bound<'py, PyAny> = + native_field(pipeline, "destination_model_entries")?; let target: Bound<'py, PyAny> = native( py, NativeAttr::ResolveTargetConfig, @@ -228,7 +231,10 @@ fn clone_input<'py>( "project_config", native_field(&prephase.discovered, "project_config")?, ), - ("local_config", native_field(&prephase.discovered, "local_config")?), + ( + "local_config", + native_field(&prephase.discovered, "local_config")?, + ), ("target_name", prephase.destination.clone()), ], )?; @@ -240,7 +246,10 @@ fn clone_input<'py>( &host::entry(py, ExecutorHost::BuildDestinationRetentionRequests)?, &[ ("project", project.clone()), - ("adapter_name", native_field(&prephase.adapter, "adapter_name")?), + ( + "adapter_name", + native_field(&prephase.adapter, "adapter_name")?, + ), ( "selected_model_names", PyFrozenSet::new(py, &selected)?.into_any(), @@ -257,10 +266,10 @@ fn clone_input<'py>( let sources: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::CloneSourceEntries)?, &[ - ("origin", native_field(&pipeline, "origin_source_entries")?), + ("origin", native_field(pipeline, "origin_source_entries")?), ( "destination", - native_field(&pipeline, "destination_source_entries")?, + native_field(pipeline, "destination_source_entries")?, ), ], )?; @@ -270,24 +279,24 @@ fn clone_input<'py>( ("source_entries", sources), ( "origin_model_entries", - native_field(&pipeline, "origin_model_entries")?, + native_field(pipeline, "origin_model_entries")?, ), ("destination_model_entries", destination_models), ( "origin_seed_entries", - native_field(&pipeline, "origin_seed_entries")?, + native_field(pipeline, "origin_seed_entries")?, ), ( "destination_seed_entries", - native_field(&pipeline, "destination_seed_entries")?, + native_field(pipeline, "destination_seed_entries")?, ), ( "destination_function_entries", - native_field(&pipeline, "destination_function_entries")?, + native_field(pipeline, "destination_function_entries")?, ), ( "execution_order", - native_field(&pipeline, "clone_plan")?.getattr("execution_order")?, + native_field(pipeline, "clone_plan")?.getattr("execution_order")?, ), ("adapter", prephase.adapter.clone()), ("destination_connection", connection.clone()), @@ -378,9 +387,12 @@ fn executed_clone<'py>( fn failed_items(result: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = result.py(); let status: Bound<'_, PyAny> = host::entry(py, ExecutorHost::CloneStatus)?; - let (failed, warning) = (native_field(&status, "FAILED")?, native_field(&status, "WARNING")?); + let (failed, warning) = ( + native_field(&status, "FAILED")?, + native_field(&status, "WARNING")?, + ); let mut items: Vec = Vec::new(); - for item in native_field(&result, "item_results")?.try_iter()? { + for item in native_field(result, "item_results")?.try_iter()? { let item: Bound<'_, PyAny> = item?; let item_status: Bound<'_, PyAny> = native_field(&item, "status")?; if !(item_status.eq(&failed)? || item_status.eq(&warning)?) { @@ -427,7 +439,10 @@ fn clone_boundaries<'py>( call_with( ®istry::class(py, RecordId::ClonePipelineConnection)?, &[ - ("config", native_field(&context.invocation, "connection_config")?), + ( + "config", + native_field(&context.invocation, "connection_config")?, + ), ("handle", connection.clone()), ], )?, @@ -437,10 +452,13 @@ fn clone_boundaries<'py>( call_with( ®istry::class(py, RecordId::ClonePipelineOptions)?, &[ - ("no_sql_validation", native_field(&request, "no_sql_validation")?), - ("no_cache", native_field(&request, "no_cache")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), + ("no_cache", native_field(request, "no_cache")?), ("select", PyTuple::new(py, &prephase.select)?.into_any()), - ("cli_vars", native_field(&request, "cli_vars")?), + ("cli_vars", native_field(request, "cli_vars")?), ], )?, ), @@ -550,13 +568,13 @@ pub(crate) fn run_defer_clone_boundary_prephase( ) -> PyResult<()> { let py: Python<'_> = context.py(); let (request, invocation) = (&context.request, &context.invocation); - let discovered: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")?; + let discovered: Bound<'_, PyAny> = native_field(invocation, "discovered_inputs")?; let split: Bound<'_, PyAny> = native( py, NativeAttr::SplitUnitTestSelectors, &[ - ("select", native_field(&request, "select")?), - ("exclude", native_field(&request, "exclude")?), + ("select", native_field(request, "select")?), + ("exclude", native_field(request, "exclude")?), ("discovered_inputs", discovered.clone()), ], )?; @@ -567,8 +585,8 @@ pub(crate) fn run_defer_clone_boundary_prephase( { return Ok(()); } - let adapter: Bound<'_, PyAny> = native_field(&invocation, "adapter")?; - let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let adapter: Bound<'_, PyAny> = native_field(invocation, "adapter")?; + let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; let project: Bound<'_, PyAny> = build_compiled_project( py, Some(keywords( @@ -576,16 +594,19 @@ pub(crate) fn run_defer_clone_boundary_prephase( &[ ("discovered_inputs", discovered.clone()), ("adapter", adapter.clone()), - ("selected_target", native_field(&request, "selected_target")?), - ("no_sql_validation", native_field(&request, "no_sql_validation")?), + ("selected_target", native_field(request, "selected_target")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), ( "analysis_selection", call_with( ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(&request, "no_cache")?)], + &[("no_cache", native_field(request, "no_cache")?)], )?, ), - ("cli_vars", native_field(&request, "cli_vars")?), + ("cli_vars", native_field(request, "cli_vars")?), ( "external_sql_reference_resolver", external_resolver(&project_dir, &discovered)?, @@ -602,7 +623,7 @@ pub(crate) fn run_defer_clone_boundary_prephase( ("exclude", native_field(&split, "exclude")?), ( "auto_load_sources", - native_field(&invocation, "should_load_sources")?, + native_field(invocation, "should_load_sources")?, ), ], )?; @@ -613,8 +634,11 @@ pub(crate) fn run_defer_clone_boundary_prephase( NativeAttr::EnforceModelExecutionLimit, &[ ("model_count", model_count.into_pyobject(py)?.into_any()), - ("target_name", native_field(&invocation, "effective_target_name")?), - ("limits", native_field(&invocation, "execution_limits")?), + ( + "target_name", + native_field(invocation, "effective_target_name")?, + ), + ("limits", native_field(invocation, "execution_limits")?), ( "refusal_note", EXECUTION_LIMIT_BUILD_NOTE.into_bound_py_any(py)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index dcfd670474..51707dada5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -43,10 +43,10 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; -use crate::bindings::_helpers::records::fields::field as native_field; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ @@ -131,8 +131,8 @@ fn selected_check_names<'py>( graph: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); - let select: Bound<'py, PyAny> = native_field(&request, "select")?; - let exclude: Bound<'py, PyAny> = native_field(&request, "exclude")?; + let select: Bound<'py, PyAny> = native_field(request, "select")?; + let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; let selectors: Vec> = select .try_iter()? .chain(exclude.try_iter()?) @@ -158,7 +158,7 @@ fn selected_check_names<'py>( PyFrozenSet::empty(py)?.into_any() }; let mut names: Vec> = Vec::new(); - for node in native_field(&graph, "nodes")?.try_iter()? { + for node in native_field(graph, "nodes")?.try_iter()? { let node: Bound<'py, PyAny> = node?; let name: Bound<'py, PyAny> = native_field(&node, "name")?; if native_field(&node, "kind")?.eq(&check_kind)? && !excluded.contains(&name)? { @@ -175,16 +175,16 @@ fn compile_pipeline<'py>( (graph, check_names): (&Bound<'py, PyAny>, &Bound<'py, PyFrozenSet>), ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = native_field(&invocation, "discovered_inputs")?; + let discovered: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")?; let required_refs: Bound<'py, PyAny> = call_with( - &native_field(&graph, "selected_sql_refs")?, + &native_field(graph, "selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; let closure: Bound<'py, PyAny> = object( py, transitive_closure_many( check_names.as_any(), - &native_field(&graph, "upstream_deps")?, + &native_field(graph, "upstream_deps")?, false, None, )?, @@ -222,19 +222,22 @@ fn compile_pipeline<'py>( } let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver( - native_field(&invocation, "effective_project_dir")?, + native_field(invocation, "effective_project_dir")?, discovered.clone(), )?; let options: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompilePipelineOptions)?, &[ - ("selected_target", native_field(&request, "selected_target")?), - ("no_sql_validation", native_field(&request, "no_sql_validation")?), + ("selected_target", native_field(request, "selected_target")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), ( "connection_config", - native_field(&invocation, "connection_config")?, + native_field(invocation, "connection_config")?, ), - ("cli_vars", native_field(&request, "cli_vars")?), + ("cli_vars", native_field(request, "cli_vars")?), ("external_sql_reference_resolver", resolver), ], )?; @@ -242,7 +245,7 @@ fn compile_pipeline<'py>( py, &[ ("discovered_inputs", discovered), - ("adapter", native_field(&invocation, "adapter")?), + ("adapter", native_field(invocation, "adapter")?), ("options", options), ("selected_keys", keys.clone()), ("relation_keys", keys), @@ -289,8 +292,8 @@ fn python_selection<'py>( let _py: Python<'py> = request.py(); let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; reject_unit_test_selectors( - &native_field(&request, "select")?, - &native_field(&request, "exclude")?, + &native_field(request, "select")?, + &native_field(request, "exclude")?, discovered, )?; let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; @@ -302,8 +305,8 @@ fn prepare<'py>( invocation: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = native_field(&invocation, "discovered_inputs")?; - let adapter: Bound<'py, PyAny> = native_field(&invocation, "adapter")?; + let discovered: Bound<'py, PyAny> = native_field(invocation, "discovered_inputs")?; + let adapter: Bound<'py, PyAny> = native_field(invocation, "adapter")?; let (planning_graph, planning_names) = python_selection(request, &discovered)?; let pipeline: Bound<'py, PyAny> = compile_pipeline(request, invocation, (&planning_graph, &planning_names))?; @@ -386,10 +389,10 @@ fn node_runtime<'py>( let py: Python<'py> = invocation.py(); let project: Bound<'py, PyAny> = native_field(&prepared.pipeline, "project")?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ - ("adapter", native_field(&invocation, "adapter")?), + ("adapter", native_field(invocation, "adapter")?), ( "connection_config", - native_field(&invocation, "connection_config")?, + native_field(invocation, "connection_config")?, ), ("connection", connection.clone()), ("run_id", native_field(&project, "run_id")?), @@ -417,7 +420,7 @@ fn record_run_state<'py>( source_map: &Bound<'py, PyAny>, ) -> PyResult<()> { record_python_run_state_results( - &native_field(&invocation, "discovered_inputs")?, + &native_field(invocation, "discovered_inputs")?, &run_state, (&python_results, load_results.as_ref()), Some(source_map), @@ -439,7 +442,7 @@ fn run_checks<'py>( let source_map: Bound<'py, PyAny> = native_field(&plan_output, "source_map")?; let callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::IngressCallbacks)?, - &[("use_color", native_field(&invocation, "use_color")?)], + &[("use_color", native_field(invocation, "use_color")?)], )?; let ingress: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecuteIngressPythonLoaderNodes)?, @@ -478,8 +481,8 @@ fn run_checks<'py>( &source_map, )?; let read_side: Bound<'py, PyAny> = run_read_side_dependencies(&ReadSideInputs { - adapter: native_field(&invocation, "adapter")?, - connection_config: native_field(&invocation, "connection_config")?, + adapter: native_field(invocation, "adapter")?, + connection_config: native_field(invocation, "connection_config")?, connection: connection.clone(), pipeline_result: prepared.pipeline.clone(), python_graph: prepared.python_graph.clone(), @@ -565,9 +568,9 @@ fn execute<'py>( providers: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = invocation.py(); - let adapter: Bound<'py, PyAny> = native_field(&invocation, "adapter")?; + let adapter: Bound<'py, PyAny> = native_field(invocation, "adapter")?; let connection: Bound<'py, PyAny> = - adapter.call_method1("connect", (native_field(&invocation, "connection_config")?,))?; + adapter.call_method1("connect", (native_field(invocation, "connection_config")?,))?; let event_writer: Bound<'py, PyAny> = Bound::new(py, ExecutionEventWriter::new(py, None)?)?.into_any(); let executed: PyResult>> = (|| { @@ -608,9 +611,9 @@ fn complete<'py>( (prepared, results): (&Prepared<'py>, &[Bound<'py, PyAny>]), ) -> PyResult { let py: Python<'py> = request.py(); - let stream: Bound<'py, PyAny> = native_field(&invocation, "progress_stream")?; + let stream: Bound<'py, PyAny> = native_field(invocation, "progress_stream")?; let style: CliStyle = CliStyle { - use_color: native_field(&invocation, "use_color")?.extract()?, + use_color: native_field(invocation, "use_color")?.extract()?, }; write_results( &stream, @@ -636,8 +639,8 @@ fn complete<'py>( write_execution_json_output( py, PyString::new(py, &results_json(results)?).into_any(), - native_field(&request, "json_output")?.is_truthy()?, - native_field(&request, "json_output_path")?, + native_field(request, "json_output")?.is_truthy()?, + native_field(request, "json_output_path")?, )?, )?; Ok(check_exit_code(&texts)) @@ -672,8 +675,11 @@ fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = (|| { - let results: Vec> = - execute(&invocation, &prepared, &native_field(&session, "providers")?)?; + let results: Vec> = execute( + &invocation, + &prepared, + &native_field(&session, "providers")?, + )?; complete(request, &invocation, (&prepared, &results)) })(); let closed: PyResult> = session.call_method0("close"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 10356cbd2d..9522e6af79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -9,9 +9,9 @@ use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; /// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. const PLANNED_FIELDS: [&str; 8] = [ @@ -53,15 +53,18 @@ fn compiled_environment<'py>( "resolved_connection", native_field(&pipeline.connection, "config")?, ), - ("no_sql_validation", native_field(&options, "no_sql_validation")?), + ( + "no_sql_validation", + native_field(options, "no_sql_validation")?, + ), ( "analysis_selection", call_with( ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(&options, "no_cache")?)], + &[("no_cache", native_field(options, "no_cache")?)], )?, ), - ("cli_vars", native_field(&options, "cli_vars")?), + ("cli_vars", native_field(options, "cli_vars")?), ("external_sql_reference_resolver", pipeline.resolver.clone()), ], )?), @@ -76,8 +79,8 @@ fn read_target_name<'py>( let py: Python<'py> = target_name.py(); let deferred: Bound<'py, PyAny> = resolve_target_config( py, - &native_field(&discovered, "project_config")?, - &native_field(&discovered, "local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, target_name, )? .getattr("defer_sources_to")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index b2bdc37e0a..227025419c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -6,6 +6,7 @@ use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_driver::stage_work::{DECLARATION_SCOPES, count_stage}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; @@ -13,7 +14,6 @@ use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; -use crate::bindings::_helpers::records::fields::field as native_field; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( @@ -59,12 +59,12 @@ pub(crate) fn build_declaration_scope<'py>( compile_cache_dir.unwrap_or_else(|| py.None().into_bound(py)), )], )?; - let test_files: Bound<'py, PyAny> = native_field(&discovered_inputs, "test_files")?; - let scenario_files: Bound<'py, PyAny> = native_field(&discovered_inputs, "scenario_files")?; + let test_files: Bound<'py, PyAny> = native_field(discovered_inputs, "test_files")?; + let scenario_files: Bound<'py, PyAny> = native_field(discovered_inputs, "scenario_files")?; registry::native(py, NativeAttr::ValidateAuthoredSqlTestCtes)?.call1(( &test_files, &scenario_files, - native_field(&sql_lexical_syntax, "native_mapping")?, + native_field(sql_lexical_syntax, "native_mapping")?, ))?; let native: Bound<'py, PyAny> = Bound::new( py, @@ -167,7 +167,8 @@ pub(crate) fn rebind_declaration_scope<'py>( ) -> PyResult> { let py: Python<'py> = scope.py(); let current: Bound<'py, PyTuple> = indexed_macro_metadata(loaded_macros)?; - let indexed: Bound<'py, PyTuple> = indexed_macro_metadata(&native_field(&scope, "loaded_macros")?)?; + let indexed: Bound<'py, PyTuple> = + indexed_macro_metadata(&native_field(scope, "loaded_macros")?)?; if current.as_any().ne(indexed.as_any())? { return Ok(py.None().into_bound(py)); } @@ -175,9 +176,9 @@ pub(crate) fn rebind_declaration_scope<'py>( discovered_inputs, loaded_macros, ( - native_field(&scope, "index")?, - native_field(&scope, "resolver")?.getattr("lookup")?, - native_field(&scope, "sql_test_scans")?, + native_field(scope, "index")?, + native_field(scope, "resolver")?.getattr("lookup")?, + native_field(scope, "sql_test_scans")?, ), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index d2e2f9c348..f8c3245c49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -9,9 +9,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; use crate::bindings::_helpers::model_config::model_config_builder::NativeModelConfigBuilder; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; /// Capture the project inputs every model's native config build and validation read. #[pyfunction] @@ -21,8 +21,8 @@ pub(crate) fn native_model_config_session<'py>( validator: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let project_config: Bound<'py, PyAny> = native_field(&inputs, "project_config")?; - let target: Bound<'py, PyAny> = native_field(&inputs, "target_config")?; + let project_config: Bound<'py, PyAny> = native_field(inputs, "project_config")?; + let target: Bound<'py, PyAny> = native_field(inputs, "target_config")?; let declared: Bound<'py, PyAny> = native_field(&project_config, "materialization_defaults")?; let materialization_defaults: Bound<'py, PyAny> = if declared.is_truthy()? { declared @@ -43,7 +43,10 @@ pub(crate) fn native_model_config_session<'py>( let target_namespace: Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = if target.is_none() { None } else { - Some((native_field(&target, "database")?, native_field(&target, "schema")?)) + Some(( + native_field(&target, "database")?, + native_field(&target, "schema")?, + )) }; let builder: Bound<'py, PyAny> = Bound::new( py, @@ -54,13 +57,13 @@ pub(crate) fn native_model_config_session<'py>( hook_entries, ), ( - native_field(&inputs, "effective_vars")?.downcast_into()?, + native_field(inputs, "effective_vars")?.downcast_into()?, unicode_environment(py)?, ), ( ( - native_field(&inputs, "effective_target_name")?, - native_field(&inputs, "run_id")?, + native_field(inputs, "effective_target_name")?, + native_field(inputs, "run_id")?, ), target_namespace, ), @@ -78,7 +81,10 @@ pub(crate) fn native_model_config_session<'py>( call_with( ®istry::class(py, RecordId::NativeModelConfigSession)?, &[ - ("path_defaults", native_field(&project_config, "path_defaults")?), + ( + "path_defaults", + native_field(&project_config, "path_defaults")?, + ), ("builder", builder), ("validator", validator.clone()), ("inherited_storage", inherited.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index faf5cce965..bafd15f163 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; @@ -59,7 +60,7 @@ fn compile_lineage<'py>( }; count_stage(py, COLUMN_LINEAGE, 1)?; Ok(build_project_column_lineage( - &graph.getattr("project")?, + &native_field(graph, "project")?, Some(dialect), &column_lineage_mode(py, member)?, None, @@ -121,7 +122,7 @@ fn seconds(milliseconds: u64) -> String { /// Whether any of `diagnostics` is an error. fn any_error(diagnostics: &Bound<'_, PyAny>) -> PyResult { for diagnostic in diagnostics.try_iter()? { - if diagnostic?.getattr("is_error")?.is_truthy()? { + if native_field(&diagnostic?, "is_error")?.is_truthy()? { return Ok(true); } } @@ -145,8 +146,8 @@ pub(crate) fn analyze<'py>( ) -> PyResult> { let options: AnalysisOptions<'py> = AnalysisOptions::read(py, options)?; let selection: &Bound<'py, PyAny> = &options.analysis_selection; - let enabled: bool = !(selection.getattr("select")?.is_truthy()? - || selection.getattr("exclude")?.is_truthy()?); + let enabled: bool = !(native_field(selection, "select")?.is_truthy()? + || native_field(selection, "exclude")?.is_truthy()?); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; let analysis: PyResult> = analyzed(py, &options, &early_lint); let _ = early_lint.borrow().stop(py); @@ -181,8 +182,8 @@ fn analyzed<'py>( options, status: Status(options.status.clone()), early_lint, - select: options.analysis_selection.getattr("select")?, - exclude: options.analysis_selection.getattr("exclude")?, + select: native_field(&options.analysis_selection, "select")?, + exclude: native_field(&options.analysis_selection, "exclude")?, }; let (discovered, discover_ms): (Bound<'py, PyAny>, u64) = driver.discover()?; let adapter: Bound<'py, PyAny> = driver.adapter(&discovered)?; @@ -194,8 +195,8 @@ fn analyzed<'py>( }; let (graph, graph_ms): (Bound<'py, PyAny>, u64) = driver.graph((&discovered, &adapter), &lint_dialect)?; - let project: Bound<'py, PyAny> = graph.getattr("project")?; - let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; + let project: Bound<'py, PyAny> = native_field(&graph, "project")?; + let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); @@ -209,7 +210,7 @@ fn analyzed<'py>( project .getattr("diagnostics")? .try_iter()? - .chain(contracts.result.getattr("diagnostics")?.try_iter()?) + .chain(native_field(&contracts.result, "diagnostics")?.try_iter()?) .collect::>>()?, )?; let rules_result: Bound<'py, PyAny> = if graph_failed { @@ -245,18 +246,27 @@ fn analyzed<'py>( "contract_cpu_ms", contracts.cpu_ms.into_pyobject(py)?.into_any(), ), - ("built_in_rules_ms", rules_result.getattr("built_in_ms")?), - ("custom_rules_ms", rules_result.getattr("custom_ms")?), + ( + "built_in_rules_ms", + native_field(&rules_result, "built_in_ms")?, + ), + ("custom_rules_ms", native_field(&rules_result, "custom_ms")?), ( "custom_rules_cpu_ms", - rules_result.getattr("custom_cpu_ms")?, + native_field(&rules_result, "custom_cpu_ms")?, ), ("early_lint_wait_ms", early_lint_wait_ms), - ("rule_cache_hits", rules_result.getattr("cache_hits")?), - ("rule_cache_misses", rules_result.getattr("cache_misses")?), + ( + "rule_cache_hits", + native_field(&rules_result, "cache_hits")?, + ), + ( + "rule_cache_misses", + native_field(&rules_result, "cache_misses")?, + ), ( "skipped_type_proof_rules", - rules_result.getattr("skipped_type_proof_rules")?, + native_field(&rules_result, "skipped_type_proof_rules")?, ), ], ) @@ -314,8 +324,8 @@ impl<'py> Driver<'_, 'py> { /// The adapter the project and local configuration select. fn adapter(&self, discovered: &Bound<'py, PyAny>) -> PyResult> { let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &discovered.getattr("project_config")?, - &discovered.getattr("local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, )?; resolve_cli_adapter(&adapter_name, &self.options.project_dir) } @@ -368,7 +378,7 @@ impl<'py> Driver<'_, 'py> { ("no_sql_validation", options.no_sql_validation.clone()), ( "skip_column_inference", - options.profile_flags.getattr("skip_column_inference")?, + native_field(&options.profile_flags, "skip_column_inference")?, ), ("column_lineage_mode", lineage_mode), ("cli_vars", options.cli_vars.clone()), @@ -457,11 +467,11 @@ impl<'py> Driver<'_, 'py> { if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; - let stage_artifacts: bool = rules_config.getattr("select")?.is_truthy()?; + let stage_artifacts: bool = native_field(&rules_config, "select")?.is_truthy()?; call_with( - &options.prepared_artifacts.getattr("start")?, + &native_field(&options.prepared_artifacts, "start")?, &[ - ("project", graph.getattr("project")?), + ("project", native_field(graph, "project")?), ("adapter", adapter.clone()), ("target_dir", target_dir), ( @@ -516,9 +526,9 @@ fn with_rule_diagnostics<'py>( registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?; let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); let mut diagnostics: Vec> = core.iter().collect(); - for fault in rules_result.getattr("findings")?.try_iter()? { + for fault in native_field(rules_result, "findings")?.try_iter()? { let fault: Bound<'py, PyAny> = fault?; - let notes: Bound<'py, PyTuple> = if fault.getattr("fixable")?.is_truthy()? { + let notes: Bound<'py, PyTuple> = if native_field(&fault, "fixable")?.is_truthy()? { PyTuple::new(py, [fix_note.clone()])? } else { PyTuple::empty(py) @@ -528,14 +538,14 @@ fn with_rule_diagnostics<'py>( &[ ("phase", phase.clone()), ("severity", severity.clone()), - ("code", fault.getattr("code")?), - ("message", fault.getattr("message")?), - ("path", fault.getattr("path")?), - ("line", fault.getattr("line")?), - ("column", fault.getattr("column")?), - ("help", fault.getattr("remediation")?), + ("code", native_field(&fault, "code")?), + ("message", native_field(&fault, "message")?), + ("path", native_field(&fault, "path")?), + ("line", native_field(&fault, "line")?), + ("column", native_field(&fault, "column")?), + ("help", native_field(&fault, "remediation")?), ("notes", notes.into_any()), - ("affected_rules", fault.getattr("affected_rules")?), + ("affected_rules", native_field(&fault, "affected_rules")?), ], )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 2e0f513cd0..fa3ba20617 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; const STAGE_REACHED: &str = "assembly stopped before the requested stage"; const EXPRESSION_SHAPES_STAGE: &str = "expression_shapes"; @@ -151,11 +152,12 @@ fn collected_project<'py>( within(&collector, |violations| { Ok((referenced_project(py, request)?, violations.clone())) })?; - let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; if !collected.is_truthy()? { return Ok(built); } - let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; + let diagnostics: Bound<'py, PyTuple> = + chained(&native_field(&built, "diagnostics")?, &collected)?; project_with( &built, Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), @@ -199,24 +201,26 @@ fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult = - with_native_project(&project, &inputs.getattr("native_project")?)?; + with_native_project(&project, &native_field(&inputs, "native_project")?)?; let references: Bound<'_, PyAny> = explicit_reference_report( py, &project, - &inputs.getattr("discovered_inputs")?, + &native_field(&inputs, "discovered_inputs")?, inputs .getattr("project_config")? .getattr("references")? .getattr("enforce_explicit")? .extract()?, )?; - let reference_diagnostics: Bound<'_, PyAny> = references.getattr("diagnostics")?; - let unmatched: Bound<'_, PyAny> = references.getattr("unmatched")?; + let reference_diagnostics: Bound<'_, PyAny> = native_field(&references, "diagnostics")?; + let unmatched: Bound<'_, PyAny> = native_field(&references, "unmatched")?; if !reference_diagnostics.is_truthy()? && !unmatched.is_truthy()? { return Ok(project); } - let diagnostics: Bound<'_, PyTuple> = - chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; + let diagnostics: Bound<'_, PyTuple> = chained( + &native_field(&project, "diagnostics")?, + &reference_diagnostics, + )?; project_with( &project, Some(keywords( @@ -238,8 +242,8 @@ fn with_native_project<'py>( if native_project.is_none() { return Ok(project.clone()); } - record_model_analyses(native_project.cast()?, &project.getattr("models")?)?; - if project.getattr("native_project")?.is(native_project) { + record_model_analyses(native_project.cast()?, &native_field(project, "models")?)?; + if native_field(project, "native_project")?.is(native_project) { return Ok(project.clone()); } project_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 0682fcd9ee..29ba31d849 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -25,6 +25,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, declared_relations, }; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; @@ -112,7 +113,7 @@ pub(crate) fn analysis_context<'py>( if context.sql_analysis_enabled.is_truthy()? { record_expression_shapes(request, &context)?; } - let catalog: Bound<'py, PyAny> = context.profile.getattr("binding_catalog")?; + let catalog: Bound<'py, PyAny> = native_field(&context.profile, "binding_catalog")?; catalog .getattr("native")? .call_method1("update_relations", (&context.complete_schemas,))?; @@ -129,7 +130,7 @@ fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = registry::class(py, RecordId::ContractPolicy)?.getattr("ENFORCED")?; let families: Bound<'py, PyDict> = PyDict::new(py); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let contract: Bound<'py, PyAny> = model_input .getattr("config")? @@ -154,7 +155,7 @@ pub(crate) fn declared_names<'py>( column_types: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = inputs.py(); - let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; + let functions: Bound<'py, PyAny> = native_field(inputs, "sql_function_inputs")?; let names: Bound<'py, PyAny> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let types: Bound<'py, PyAny> = PyTuple::new(py, declared_types(&functions, column_types)?)?.into_any(); @@ -176,10 +177,10 @@ fn profile_with_catalog<'py>( }; let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = declared_names(inputs, column_types)?; - let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(&profile, "sql_analysis_dialect")?; let ignore_case: Bound<'py, PyAny> = PyBool::new( py, - ignores_quoted_case(&inputs.getattr("effective_connection")?, &dialect)?, + ignores_quoted_case(&native_field(inputs, "effective_connection")?, &dialect)?, ) .to_owned() .into_any(); @@ -208,7 +209,7 @@ fn profile_with_catalog<'py>( ], )?), )?; - let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(&profile, "sql_analysis_dialect")?; let dialect: String = if dialect.is_truthy()? { dialect.extract()? } else { @@ -223,8 +224,8 @@ fn profile_with_catalog<'py>( profile .getattr("quoted_identifiers_ignore_case")? .extract()?, - profile.getattr("semantic_known_functions")?.extract()?, - profile.getattr("semantic_known_types")?.extract()?, + native_field(&profile, "semantic_known_functions")?.extract()?, + native_field(&profile, "semantic_known_types")?.extract()?, ), PyDict::new(py).as_any(), )?, @@ -243,19 +244,19 @@ fn function_return_types<'py>( ) -> PyResult> { let py: Python<'py> = inputs.py(); let types: Bound<'py, PyDict> = PyDict::new(py); - types.call_method1("update", (profile.getattr("function_return_types")?,))?; + types.call_method1("update", (native_field(profile, "function_return_types")?,))?; let scalar: Bound<'py, PyDict> = PyDict::new(py); - for function in inputs.getattr("sql_function_inputs")?.try_iter()? { + for function in native_field(inputs, "sql_function_inputs")?.try_iter()? { let function: Bound<'py, PyAny> = function?; - if function.getattr("return_columns")?.is_truthy()? { + if native_field(&function, "return_columns")?.is_truthy()? { continue; } - let name: Bound<'py, PyAny> = function.getattr("name")?.str()?.into_any(); + let name: Bound<'py, PyAny> = native_field(&function, "name")?.str()?.into_any(); let key: Bound<'py, PyAny> = PyString::new(py, UDF_PREFIX) .into_any() .add(name)? .call_method0("upper")?; - scalar.set_item(key, function.getattr("returns")?)?; + scalar.set_item(key, native_field(&function, "returns")?)?; } types.update(scalar.as_mapping())?; Ok(types) @@ -270,7 +271,7 @@ fn record_expression_shapes<'py>( let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let mut sources: Vec> = Vec::new(); let mut expressions: Vec> = Vec::new(); - for source_input in inputs.getattr("source_inputs")?.try_iter()? { + for source_input in native_field(inputs, "source_inputs")?.try_iter()? { let source_input: Bound<'py, PyAny> = source_input?; let expression: Bound<'py, PyAny> = source_input .getattr("source_entry")? @@ -310,7 +311,8 @@ fn record_expression_shapes<'py>( if shape.is_none() { continue; } - let name: Bound<'py, PyAny> = source_input.getattr("source_entry")?.getattr("name")?; + let name: Bound<'py, PyAny> = + native_field(source_input, "source_entry")?.getattr("name")?; let declared_types: Bound<'py, PyAny> = context .column_types .call_method1("get", (&name, PyDict::new(py)))?; @@ -350,8 +352,8 @@ fn model_analyses<'py>( let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); let lineage_mode: Bound<'py, PyAny> = registry::class(py, RecordId::ColumnLineageMode)?; - let rich: bool = mode.eq(lineage_mode.getattr("RICH")?)?; - let allow_compact: bool = mode.eq(lineage_mode.getattr("FAST")?)? || rich; + let rich: bool = mode.eq(native_field(&lineage_mode, "RICH")?)?; + let allow_compact: bool = mode.eq(native_field(&lineage_mode, "FAST")?)? || rich; let flags: (Bound<'py, PyAny>, Bound<'py, PyAny>) = ( PyBool::new(py, allow_compact).to_owned().into_any(), PyBool::new(py, rich).to_owned().into_any(), @@ -364,9 +366,9 @@ fn model_analyses<'py>( record_compile_timing("model_analysis_ms", || { record_compile_cpu_timing(py, "model_analysis_cpu_ms", || { let mut model_inputs: Vec> = Vec::new(); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; - if model_input.getattr("sql_validation_enabled")?.is_truthy()? + if native_field(&model_input, "sql_validation_enabled")?.is_truthy()? && selected_for_analysis(&model_input, names)? { model_inputs.push(model_input); @@ -403,8 +405,8 @@ fn analysis_cache<'py>( &[ ("known_functions", known_functions), ("known_types", known_types), - ("target", inputs.getattr("effective_target_name")?), - ("vars", inputs.getattr("effective_vars")?), + ("target", native_field(inputs, "effective_target_name")?), + ("vars", native_field(inputs, "effective_vars")?), ], )?; Ok(analysis_cache_context( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 2f7df00b53..621e9178cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -6,6 +6,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; @@ -37,14 +38,14 @@ pub(crate) fn test_model_query_overrides<'py>( ) -> PyResult> { let py: Python<'py> = inputs.py(); let overrides: Bound<'py, PyDict> = PyDict::new(py); - let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + let payload: Bound<'py, PyAny> = native_field(test_input, "payload")?; if !payload.is_instance(®istry::class( py, RecordId::CompileModelSqlTestInputPayload, )?)? { return Ok(overrides); } - let macro_mocks: Bound<'py, PyAny> = payload.getattr("macro_mocks")?; + let macro_mocks: Bound<'py, PyAny> = native_field(&payload, "macro_mocks")?; if !macro_mocks.is_truthy()? { return Ok(overrides); } @@ -58,28 +59,28 @@ pub(crate) fn test_model_query_overrides<'py>( )], )?), )?; - let declaration_scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let declaration_scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; let lookup: Bound<'py, PyAny> = if declaration_scope.is_none() { py.None().into_bound(py) } else { - declaration_scope.getattr("resolver")?.getattr("lookup")? + native_field(&declaration_scope, "resolver")?.getattr("lookup")? }; - let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; + let loaded_macros: Bound<'py, PyAny> = native_field(inputs, "loaded_macros")?; let declaration_resolver: Bound<'py, PyAny> = declaration_scope_resolver( - &inputs.getattr("discovered_inputs")?, - &inputs.getattr("scope_index")?, + &native_field(inputs, "discovered_inputs")?, + &native_field(inputs, "scope_index")?, (&loaded_macros, &lookup), )?; - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let file_path: Bound<'py, PyAny> = - model_input.getattr("model_file")?.getattr("file_path")?; - let model_name: Bound<'py, PyAny> = file_path.getattr("stem")?; - let macro_source_sql: Bound<'py, PyAny> = model_input.getattr("macro_source_sql")?; + native_field(&model_input, "model_file")?.getattr("file_path")?; + let model_name: Bound<'py, PyAny> = native_field(&file_path, "stem")?; + let macro_source_sql: Bound<'py, PyAny> = native_field(&model_input, "macro_source_sql")?; let macro_source_sql: Bound<'py, PyAny> = if macro_source_sql.is_truthy()? { macro_source_sql } else { - model_input.getattr("query_sql")? + native_field(&model_input, "query_sql")? }; let expanded: Bound<'py, PyAny> = expansion_result(ExpansionRequest { sql: macro_source_sql.downcast_into()?, @@ -98,7 +99,7 @@ pub(crate) fn test_model_query_overrides<'py>( &expanded.extract::()?, ( &model_name.extract::()?, - &model_input.getattr("config")?.getattr("values")?, + &native_field(&model_input, "config")?.getattr("values")?, ), )?; overrides.set_item(model_name, query)?; @@ -109,13 +110,13 @@ pub(crate) fn test_model_query_overrides<'py>( /// The compile's macro context, or one built from the effective adapter, settings and target. fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = inputs.py(); - let macro_context: Bound<'py, PyAny> = inputs.getattr("macro_context")?; + let macro_context: Bound<'py, PyAny> = native_field(inputs, "macro_context")?; if macro_context.is_truthy()? { return Ok(macro_context); } let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &inputs.getattr("project_config")?, - &inputs.getattr("local_config")?, + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, ) .and_then(|value| value.into_bound_py_any(py))?; call_with( @@ -128,8 +129,11 @@ fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_SCENARIO")?; let mut scenarios: Vec> = Vec::new(); - for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { + for scenario_input in native_field(inputs, "scenario_inputs")?.try_iter()? { let scenario_input: Bound<'py, PyAny> = scenario_input?; - let scenario_file: Bound<'py, PyAny> = scenario_input.getattr("scenario_file")?; - let name: Bound<'py, PyAny> = scenario_file.getattr("name")?; + let scenario_file: Bound<'py, PyAny> = native_field(&scenario_input, "scenario_file")?; + let name: Bound<'py, PyAny> = native_field(&scenario_file, "name")?; let key: Bound<'py, PyAny> = call_with( &key_class, &[ @@ -166,8 +170,14 @@ pub(crate) fn compiled_sql_scenarios<'py>( for field in SCENARIO_FIELDS { fields.set_item(field, scenario_input.getattr(field)?)?; } - fields.set_item("source_path", scenario_file.getattr("relative_path")?)?; - fields.set_item("ownership_root", scenario_file.getattr("ownership_root")?)?; + fields.set_item( + "source_path", + native_field(&scenario_file, "relative_path")?, + )?; + fields.set_item( + "ownership_root", + native_field(&scenario_file, "ownership_root")?, + )?; scenarios.push(scenario_class.call((), Some(&fields))?); } PyTuple::new(py, scenarios) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index fe15167773..098a62cf58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, Nati use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -26,8 +27,8 @@ pub(crate) fn native_project_facts<'py>( session: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let model_inputs: Bound<'py, PyAny> = inputs.getattr("model_inputs")?; - let dialect: Bound<'py, PyAny> = context.profile.getattr("sql_analysis_dialect")?; + let model_inputs: Bound<'py, PyAny> = native_field(inputs, "model_inputs")?; + let dialect: Bound<'py, PyAny> = native_field(&context.profile, "sql_analysis_dialect")?; let (succeeded, positions): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match &context.analyses { Some(analyses) => ( analyses.succeeded.clone().into_any(), @@ -54,8 +55,8 @@ pub(crate) fn native_project_facts<'py>( context, session, )?; - let deps: Bound<'py, PyAny> = resources.getattr("model_deps")?; - let syntax_valid: Bound<'py, PyAny> = resources.getattr("model_syntax_valid")?; + let deps: Bound<'py, PyAny> = native_field(&resources, "model_deps")?; + let syntax_valid: Bound<'py, PyAny> = native_field(&resources, "model_syntax_valid")?; let mut models: Vec> = Vec::new(); for index in 0..model_inputs.len()? { let proof: Bound<'py, PyAny> = match proofs.get_item(index)? { @@ -90,7 +91,10 @@ pub(crate) fn pivot_tables<'py>(context: &AnalysisContext<'py>) -> PyResult( let py: Python<'py> = inputs.py(); let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; let mut models: Vec> = Vec::new(); - for (index, model_input) in inputs.getattr("model_inputs")?.try_iter()?.enumerate() { + for (index, model_input) in native_field(inputs, "model_inputs")? + .try_iter()? + .enumerate() + { let model_input: Bound<'py, PyAny> = model_input?; let sql_analysis_enabled: bool = analysis_enabled - && model_input.getattr("sql_validation_enabled")?.is_truthy()? + && native_field(&model_input, "sql_validation_enabled")?.is_truthy()? && selected_for_analysis(&model_input, names)?; let request: ModelRequest<'_, 'py> = ModelRequest { model_input: &model_input, @@ -68,13 +72,15 @@ fn compiled_model<'py>( let model_input: &Bound<'py, PyAny> = request.model_input; let py: Python<'py> = model_input.py(); let model_name: Bound<'py, PyAny> = model_stem(model_input)?; - let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; - let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; + let syntax_validated: bool = + native_field(&request.native_model, "syntax_valid")?.is_truthy()?; + let config_values: Bound<'py, PyAny> = + native_field(model_input, "config")?.getattr("values")?; let analysis_query_sql: Bound<'py, PyAny> = PyString::new( py, &analysis_cursor_sql( py, - &model_input.getattr("query_sql")?.extract::()?, + &native_field(model_input, "query_sql")?.extract::()?, &config_values.call_method1("get", ("cursor_type",))?, )?, ) @@ -84,11 +90,11 @@ fn compiled_model<'py>( Some(values) => values.clone().into_any(), None => py.None().into_bound(py), }; - let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; + let input_validated: bool = native_field(model_input, "sql_validation_enabled")?.is_truthy()?; let location = || -> PyResult<(String, Bound<'py, PyAny>)> { Ok(( model_name.extract()?, - model_input.getattr("model_file")?.getattr("file_path")?, + native_field(model_input, "model_file")?.getattr("file_path")?, )) }; let syntax = || -> PyResult { @@ -97,7 +103,7 @@ fn compiled_model<'py>( .as_ref() .map(|values| values.extract()) .transpose()?, - context.profile.getattr("sql_analysis_dialect")?.extract()?, + native_field(&context.profile, "sql_analysis_dialect")?.extract()?, )) }; let validate_sql = || -> PyResult<()> { @@ -150,7 +156,7 @@ fn compiled_model<'py>( } else if request.sql_validation_enabled && validates { validate_sql()?; } - let declared_macro_deps: Bound<'py, PyAny> = model_input.getattr("macro_deps")?; + let declared_macro_deps: Bound<'py, PyAny> = native_field(model_input, "macro_deps")?; let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { declared_macro_deps } else { @@ -169,10 +175,10 @@ fn compiled_model<'py>( return model_input.call_method1( "compiled_model", (( - native_model.getattr("deps")?, + native_field(native_model, "deps")?, session_analysis, fallback, - native_model.getattr("dynamic_contract")?, + native_field(native_model, "dynamic_contract")?, macro_list, ),), ); @@ -184,13 +190,13 @@ fn compiled_model<'py>( ("fallback", fallback), ( "standalone_proof", - native_model.getattr("dynamic_contract")?, + native_field(native_model, "dynamic_contract")?, ), ], )?; hand_built_model( model_input, - &native_model.getattr("deps")?, + &native_field(native_model, "deps")?, &analysis, ¯o_deps, ) @@ -239,8 +245,8 @@ pub(crate) fn project_binding_diagnostics<'py>( }; for model in models.as_any().try_iter()? { let model: Bound<'py, PyAny> = model?; - if analyses.positions.contains(model.getattr("name")?)? { - for diagnostic in model.getattr("binding_diagnostics")?.try_iter()? { + if analyses.positions.contains(native_field(&model, "name")?)? { + for diagnostic in native_field(&model, "binding_diagnostics")?.try_iter()? { diagnostics.push(diagnostic?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index dc1e79bc0d..6e01d7a4de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::{ use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; @@ -104,12 +105,12 @@ pub(crate) fn assembled_project<'py>( ) })?; let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; - let resources: Bound<'py, PyAny> = native.getattr("resources")?; + let resources: Bound<'py, PyAny> = native_field(&native, "resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = - compiled_models(inputs, &context, (&native.getattr("models")?, names))?; + compiled_models(inputs, &context, (&native_field(&native, "models")?, names))?; let fields: Bound<'py, PyDict> = project_fields( inputs, &context, @@ -120,7 +121,7 @@ pub(crate) fn assembled_project<'py>( let binding_results: Bound<'py, PyDict> = binding_rows(py, context.analyses.as_ref())?; let resource_sql_analysis: Bound<'py, PyAny> = and_not( &context.sql_analysis_enabled, - &inputs.getattr("no_sql_validation")?, + &native_field(inputs, "no_sql_validation")?, )?; request.before_stage(AssemblyStage::SemanticCompletion, || { keywords( @@ -157,7 +158,7 @@ fn str_or_none<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// The selected target's `field`, or `None` without a selected target. fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult> { - let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + let target: Bound<'py, PyAny> = native_field(inputs, "effective_target")?; if target.is_none() { return Ok(target); } @@ -167,7 +168,7 @@ fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult(inputs: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = inputs.py(); - let connection: Bound<'py, PyAny> = inputs.getattr("effective_connection")?; + let connection: Bound<'py, PyAny> = native_field(inputs, "effective_connection")?; let database: Bound<'py, PyAny> = target_field(inputs, "database")?; let database: Bound<'py, PyAny> = if database.is_truthy()? { database @@ -182,10 +183,10 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( ) -> PyResult> { let py: Python<'py> = inputs.py(); let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &inputs.getattr("project_config")?, - &inputs.getattr("local_config")?, + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, ) .and_then(|value| value.into_bound_py_any(py))?; if adapter_name.eq(host::entry(py, AdapterHost::BuiltinAdapter)?.getattr("DUCKDB")?)? { @@ -219,25 +220,25 @@ fn project_fields<'py>( target_values: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let discovered: Bound<'py, PyAny> = inputs.getattr("discovered_inputs")?; - let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let discovered: Bound<'py, PyAny> = native_field(inputs, "discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(inputs, "project_config")?; let fields: Bound<'py, PyDict> = keywords( py, &[ ( "binding_catalog", - context.profile.getattr("binding_catalog")?, + native_field(&context.profile, "binding_catalog")?, ), - ("run_id", inputs.getattr("run_id")?), + ("run_id", native_field(inputs, "run_id")?), ( "effective_target_name", - inputs.getattr("effective_target_name")?, + native_field(inputs, "effective_target_name")?, ), ( "effective_connection", - inputs.getattr("effective_connection")?, + native_field(inputs, "effective_connection")?, ), - ("effective_vars", inputs.getattr("effective_vars")?), + ("effective_vars", native_field(inputs, "effective_vars")?), ( "effective_target_database", str_or_none(&target_values.call_method1("get", ("database",))?)?, @@ -248,16 +249,22 @@ fn project_fields<'py>( ), ( "sql_analysis_dialect", - context.profile.getattr("sql_analysis_dialect")?, + native_field(&context.profile, "sql_analysis_dialect")?, ), - ("sql_lexical_syntax", inputs.getattr("sql_lexical_syntax")?), - ("compile_cache_dir", inputs.getattr("compile_cache_dir")?), - ("settings", inputs.getattr("effective_settings")?), + ( + "sql_lexical_syntax", + native_field(inputs, "sql_lexical_syntax")?, + ), + ( + "compile_cache_dir", + native_field(inputs, "compile_cache_dir")?, + ), + ("settings", native_field(inputs, "effective_settings")?), ( "scenario", resolve_effective_scenario_config( &project_config, - &inputs.getattr("local_config")?, + &native_field(inputs, "local_config")?, )?, ), ("models", models.clone().into_any()), @@ -299,7 +306,7 @@ fn project_fields<'py>( fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; fields.set_item( "external_sql_reference_resolver", - inputs.getattr("external_sql_reference_resolver")?, + native_field(inputs, "external_sql_reference_resolver")?, )?; fields.set_item("scope_index", scope_index)?; Ok(fields) @@ -316,7 +323,7 @@ fn compiled_resources<'py>( if count == 0 { return Ok(PyTuple::empty(py)); } - let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let owner: Bound<'py, PyAny> = native_field(inputs, "native_project")?; let facts: Bound<'py, PyAny> = resources.getattr(kind.facts)?; owner.call_method1( kind.record, @@ -341,12 +348,12 @@ fn native_resource<'py>( (slot, items): (&str, &Bound<'py, PyAny>), index: usize, ) -> PyResult> { - let failure: Bound<'py, PyAny> = resources.getattr("failure")?; + let failure: Bound<'py, PyAny> = native_field(resources, "failure")?; if !failure.is_none() - && failure.getattr("slot")?.eq(slot)? - && failure.getattr("index")?.eq(index)? + && native_field(&failure, "slot")?.eq(slot)? + && native_field(&failure, "index")?.eq(index)? { - return Err(PyErr::from_value(failure.getattr("error")?)); + return Err(PyErr::from_value(native_field(&failure, "error")?)); } items.get_item(index) } @@ -358,7 +365,7 @@ fn project_view<'py>( models: &Bound<'py, PyTuple>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let owner: Bound<'py, PyAny> = native_field(inputs, "native_project")?; let mut all_native: bool = true; for model in models.as_any().try_iter()? { if !model?.is_instance_of::() { @@ -373,12 +380,12 @@ fn project_view<'py>( return owner.call_method1("project_view", (view_fields,)); } let sql_expansions: Bound<'py, PyDict> = PyDict::new(py); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; - let expansion: Bound<'py, PyAny> = model_input.getattr("sql_expansion")?; + let expansion: Bound<'py, PyAny> = native_field(&model_input, "sql_expansion")?; if !expansion.is_none() { sql_expansions.set_item( - model_input.getattr("model_file")?.getattr("file_path")?, + native_field(&model_input, "model_file")?.getattr("file_path")?, expansion, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 744d2bc143..0fcb82dafc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query use crate::bindings::_helpers::compile_driver::diagnostic_collector::report_compile_diagnostic; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::compile_driver::stage_work::{SQL_TEST_ASSEMBLY, count_stage}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::compiled_tests::assemble_native_sql_tests; @@ -38,22 +39,24 @@ pub(crate) fn assembled_sql_tests<'py>( registry::class(py, RecordId::CompileModelSqlTestInputPayload)?; let mut tests: Vec> = Vec::with_capacity(assemblies.len()); for (test_input, assembly) in test_inputs.iter().zip(assemblies) { - let failure: Bound<'py, PyAny> = assembly.getattr("failure")?; + let failure: Bound<'py, PyAny> = native_field(&assembly, "failure")?; let kind: Option = if failure.is_none() { None } else { - Some(failure.getattr("kind")?.extract()?) + Some(native_field(&failure, "kind")?.extract()?) }; - let mut test: Bound<'py, PyAny> = assembly.getattr("test")?; + let mut test: Bound<'py, PyAny> = native_field(&assembly, "test")?; if test.is_none() { return Err(assembly_error(py, &failure, kind.as_deref())?); } - for item in assembly.getattr("diagnostics")?.try_iter()? { + for item in native_field(&assembly, "diagnostics")?.try_iter()? { let (key, diagnostic): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; report_compile_diagnostic(&key, &diagnostic)?; } - let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; - if payload.is_instance(&mocked_input)? && payload.getattr("macro_mocks")?.is_truthy()? { + let payload: Bound<'py, PyAny> = native_field(test_input, "payload")?; + if payload.is_instance(&mocked_input)? + && native_field(&payload, "macro_mocks")?.is_truthy()? + { test = with_model_queries( &test, test_model_query_overrides(test_input, inputs)?.as_any(), @@ -78,7 +81,7 @@ fn assembly_error( let message: String = if failure.is_none() { String::new() } else { - failure.getattr("message")?.extract()? + native_field(failure, "message")?.extract()? }; if kind == Some(SQL_TEST_INPUT_FAILURE) { let error: Bound<'_, PyAny> = @@ -94,7 +97,7 @@ fn with_model_queries<'py>( queries: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = test.py(); - let payload: Bound<'py, PyAny> = test.getattr("payload")?; + let payload: Bound<'py, PyAny> = native_field(test, "payload")?; if !payload.is_instance(®istry::class(py, RecordId::CompiledModelSqlTestPayload)?)? { return Ok(test.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 92ffc2298a..985f4ae027 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; @@ -58,11 +59,11 @@ fn run_compile_command<'py>( reuse_attempt: Option>, ) -> PyResult { let total_start: f64 = match &reuse_attempt { - Some(attempt) => attempt.getattr("started")?.extract()?, + Some(attempt) => native_field(attempt, "started")?.extract()?, None => monotonic(py)?, }; let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let effective_request: Bound<'py, PyAny> = if request.getattr("project_dir")?.is_none() { + let effective_request: Bound<'py, PyAny> = if native_field(&request, "project_dir")?.is_none() { imported(py, "dataclasses", "replace")?.call( (request.clone(),), Some(&keywords( @@ -100,7 +101,7 @@ fn bypass_attempt<'py>( started: f64, ) -> PyResult> { let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'py, PyAny> = if project_dir.is_truthy()? { project_dir } else { @@ -120,14 +121,14 @@ fn bypass_attempt<'py>( fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); let none: Bound<'py, PyAny> = py.None().into_bound(py); - if request.getattr("json_output")?.is_truthy()? { + if native_field(request, "json_output")?.is_truthy()? { return Ok(none); } let stdout: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; if !stdout.hasattr("isatty")? || !stdout.call_method0("isatty")?.is_truthy()? { return Ok(none); } - let use_color: bool = !request.getattr("no_color")?.is_truthy()?; + let use_color: bool = !native_field(request, "no_color")?.is_truthy()?; status_reporter(&stdout, use_color) } @@ -217,7 +218,7 @@ fn recorded_compile<'py>( .into_any(), ), ("dag_artifact_path", outcome.dag_artifact_path), - ("json_output", request.getattr("json_output")?), + ("json_output", native_field(request, "json_output")?), ], )?), ); @@ -234,7 +235,7 @@ fn compiled_with_status<'py>( (total_start, reuse_timings): (f64, &Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = request.py(); - let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'py, PyAny> = if project_dir.is_none() { imported(py, "pathlib", "Path")?.call_method0("cwd")? } else { @@ -242,7 +243,7 @@ fn compiled_with_status<'py>( }; let prepared_artifacts: &Bound<'py, PyAny> = &recorders.prepared_artifacts; call_with( - &prepared_artifacts.getattr("remove_abandoned_staging")?, + &native_field(prepared_artifacts, "remove_abandoned_staging")?, &[("target_dir", target_dir(&project_dir)?)], )?; let analysis: Bound<'py, PyAny> = analyze( @@ -254,13 +255,13 @@ fn compiled_with_status<'py>( .getattr("profile_flags")? .getattr("skip_write")? .is_truthy()?; - let manifest_requested: bool = request.getattr("manifest")?.is_truthy()?; + let manifest_requested: bool = native_field(request, "manifest")?.is_truthy()?; let manifest: Bound<'py, PyAny> = manifest_payload(manifest_requested && !rules_failed, &analysis, status)?; let dag_path: Bound<'py, PyAny> = if rules_failed { py.None().into_bound(py) } else { - request.getattr("dag_path")? + native_field(request, "dag_path")? }; let recorder: Option<&Py> = Some(&recorders.artifact_writes); write_dag_artifact((&dag_path, &project_dir), (&analysis, recorder), status)?; @@ -285,28 +286,28 @@ fn compiled_with_status<'py>( analysis .getattr("diagnostics")? .try_iter()? - .chain(written.getattr("diagnostics")?.try_iter()?) + .chain(native_field(&written, "diagnostics")?.try_iter()?) .chain(withheld.try_iter()?) .collect::>>()?, )?; let mut exit_code: i64 = 0; for diagnostic in diagnostics.as_any().try_iter()? { - if diagnostic?.getattr("is_error")?.is_truthy()? { + if native_field(&diagnostic?, "is_error")?.is_truthy()? { exit_code = 1; break; } } close_status(status)?; - if let Some(skipped_note) = - format_skipped_type_proof_rules(analysis.getattr("skipped_type_proof_rules")?.extract()?) - { + if let Some(skipped_note) = format_skipped_type_proof_rules( + native_field(&analysis, "skipped_type_proof_rules")?.extract()?, + ) { output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = - if rules_failed || request.getattr("dag_path")?.is_none() { + if rules_failed || native_field(request, "dag_path")?.is_none() { py.None().into_bound(py) } else { - resolved_dag_path(&project_dir, &request.getattr("dag_path")?)? + resolved_dag_path(&project_dir, &native_field(request, "dag_path")?)? }; let outcome: CommandOutcome<'py> = CommandOutcome { exit_code, @@ -337,20 +338,23 @@ fn analysis_options<'py>( let selection: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAnalysisSelection)?, &[ - ("select", request.getattr("select")?), - ("exclude", request.getattr("exclude")?), - ("no_cache", request.getattr("no_cache")?), + ("select", native_field(request, "select")?), + ("exclude", native_field(request, "exclude")?), + ("no_cache", native_field(request, "no_cache")?), ], )?; keywords( py, &[ ("project_dir", project_dir.clone()), - ("no_sql_validation", request.getattr("no_sql_validation")?), - ("selected_target", request.getattr("selected_target")?), - ("lineage_mode", request.getattr("lineage_mode")?), - ("cli_vars", request.getattr("cli_vars")?), - ("profile_flags", request.getattr("profile_flags")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), + ("selected_target", native_field(request, "selected_target")?), + ("lineage_mode", native_field(request, "lineage_mode")?), + ("cli_vars", native_field(request, "cli_vars")?), + ("profile_flags", native_field(request, "profile_flags")?), ("analysis_selection", selection), ("status", status.0.clone()), ("prepared_artifacts", recorders.prepared_artifacts.clone()), @@ -363,10 +367,10 @@ fn rules_failed(analysis: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = analysis.py(); let rule_phase: Bound<'_, PyAny> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("RULE")?; - for diagnostic in analysis.getattr("diagnostics")?.try_iter()? { + for diagnostic in native_field(analysis, "diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; - if diagnostic.getattr("is_error")?.is_truthy()? - && diagnostic.getattr("phase")?.is(&rule_phase) + if native_field(&diagnostic, "is_error")?.is_truthy()? + && native_field(&diagnostic, "phase")?.is(&rule_phase) { return Ok(true); } @@ -417,23 +421,25 @@ fn record_output<'py>( output: &Bound<'py, PyAny>, ) -> PyResult<()> { let py: Python<'py> = request.py(); - let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; - let selected_keys: Bound<'py, PyAny> = analysis.getattr("selected_keys")?; - let validation_enabled: Bound<'py, PyAny> = - PyBool::new(py, !request.getattr("no_sql_validation")?.is_truthy()?) - .to_owned() - .into_any(); + let graph: Bound<'py, PyAny> = native_field(analysis, "graph")?; + let selected_keys: Bound<'py, PyAny> = native_field(analysis, "selected_keys")?; + let validation_enabled: Bound<'py, PyAny> = PyBool::new( + py, + !native_field(request, "no_sql_validation")?.is_truthy()?, + ) + .to_owned() + .into_any(); let shared: [(&str, Bound<'py, PyAny>); 7] = [ ("graph", graph.clone()), ("written", written.clone()), - ("manifest", request.getattr("manifest")?), - ("lineage", analysis.getattr("lineage")?), - ("lineage_mode", request.getattr("lineage_mode")?), + ("manifest", native_field(request, "manifest")?), + ("lineage", native_field(analysis, "lineage")?), + ("lineage_mode", native_field(request, "lineage_mode")?), ("diagnostics", diagnostics.clone().into_any()), ("selected_keys", selected_keys.clone()), ]; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); - let rendered: Bound<'py, PyString> = if request.getattr("json_output")?.is_truthy()? { + let rendered: Bound<'py, PyString> = if native_field(request, "json_output")?.is_truthy()? { let notice: Option> = json_semantic_notice(&graph, &selected_keys, validation_enabled.is_truthy()?)?; if let Some(notice) = notice { @@ -447,7 +453,7 @@ fn record_output<'py>( ) } else { let use_color: bool = - !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; + !native_field(request, "no_color")?.is_truthy()? && stdout_supports_color(py)?; arguments.push(( "use_color", PyBool::new(py, use_color).to_owned().into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 4b1fabf64e..641a7c4d1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::{ }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -54,7 +55,7 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { /// Reject a local SQL analysis opt-out the project forbids. fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); - let local_settings: Bound<'_, PyAny> = config.local_config.getattr("settings")?; + let local_settings: Bound<'_, PyAny> = native_field(config.local_config, "settings")?; let flags: Bound<'_, PyTuple> = PyTuple::new( py, [ @@ -75,11 +76,11 @@ fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> P /// The first target, by name, that selects a command-group warehouse. fn first_configured_group(config: &Bound<'_, PyAny>) -> PyResult> { - let targets: Bound<'_, PyAny> = config.getattr("targets")?; + let targets: Bound<'_, PyAny> = native_field(config, "targets")?; let mut names: Vec = target_names(config)?; names.sort(); for name in names { - let warehouses: Bound<'_, PyAny> = targets.get_item(&name)?.getattr("warehouses")?; + let warehouses: Bound<'_, PyAny> = native_field(&targets.get_item(&name)?, "warehouses")?; for group in WAREHOUSE_GROUPS { let value: Bound<'_, PyAny> = warehouses.getattr(group)?; if !value.is_none() { @@ -134,7 +135,7 @@ fn reject_group( fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); if let Some(group) = first_configured_group(config.project_config)? { - let adapter_name: Bound<'_, PyAny> = config.project_config.getattr("adapter")?; + let adapter_name: Bound<'_, PyAny> = native_field(config.project_config, "adapter")?; if !session_warehouse_support(py, &adapter_name, config.project_dir)? { return Err(reject_group( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 21dc42cdd8..b71ccba1b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -11,6 +11,7 @@ use sqlbuild_model_config::templates::models::DeferredSegment; use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -72,8 +73,8 @@ fn deferred_locations<'py>( for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'py, PyAny> = resource?; let location: Bound<'py, PyAny> = - deferred_location(&resource.getattr("destination")?, naming)?; - locations.set_item(resource.getattr("name")?, location)?; + deferred_location(&native_field(&resource, "destination")?, naming)?; + locations.set_item(native_field(&resource, "name")?, location)?; } } Ok(locations) @@ -85,15 +86,15 @@ fn deferred_location<'py>( naming: &DeferredNaming<'py>, ) -> PyResult> { let py: Python<'py> = target.py(); - let logical_schema: Bound<'py, PyAny> = target.getattr("logical_schema")?; - let logical_database: Bound<'py, PyAny> = target.getattr("logical_database")?; + let logical_schema: Bound<'py, PyAny> = native_field(target, "logical_schema")?; + let logical_database: Bound<'py, PyAny> = native_field(target, "logical_database")?; let mut schema: Bound<'py, PyAny> = target_field( - &naming.target_config.getattr("schema")?, + &native_field(&naming.target_config, "schema")?, &logical_schema, &naming.effective_vars, )?; let mut database: Bound<'py, PyAny> = target_field( - &naming.target_config.getattr("database")?, + &native_field(&naming.target_config, "database")?, &logical_database, &naming.effective_vars, )?; @@ -103,7 +104,7 @@ fn deferred_location<'py>( if database.is_none() { database = naming.default_database.clone(); } - let name: Bound<'py, PyAny> = target.getattr("name")?; + let name: Bound<'py, PyAny> = native_field(target, "name")?; let qualified_name: Bound<'py, PyAny> = call_with( &naming.render_qualified_name, &[ @@ -205,7 +206,8 @@ fn deferred_target_config<'py>( current_target_name: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = discovered.py(); - let targets: Bound<'py, PyAny> = discovered.getattr("project_config")?.getattr("targets")?; + let targets: Bound<'py, PyAny> = + native_field(discovered, "project_config")?.getattr("targets")?; let name: String = defer_to.str()?.to_cow()?.into_owned(); if !targets.contains(defer_to)? { return Err(planner_error( @@ -253,14 +255,14 @@ fn deferred_relations<'py>( let mut database: Bound<'py, PyAny> = py.None().into_bound(py); for location in locations.call_method0("values")?.try_iter()? { let location: Bound<'py, PyAny> = location?; - let schema: Bound<'py, PyAny> = location.getattr("schema")?; + let schema: Bound<'py, PyAny> = native_field(&location, "schema")?; if !schema.is_none() { let schema: String = schema.extract()?; if !schemas.contains(&schema) { schemas.push(schema); } } - let location_database: Bound<'py, PyAny> = location.getattr("database")?; + let location_database: Bound<'py, PyAny> = native_field(&location, "database")?; if !location_database.is_none() && database.is_none() { database = location_database; } @@ -271,7 +273,7 @@ fn deferred_relations<'py>( } schemas.sort(); let relations: Bound<'py, PyAny> = call_with( - &adapter.getattr("list_relations")?, + &native_field(adapter, "list_relations")?, &[ ("connection", connection.clone()), ("database", database), @@ -280,7 +282,7 @@ fn deferred_relations<'py>( )?; for relation in relations.try_iter()? { let relation: Bound<'py, PyAny> = relation?; - relations_by_name.set_item(relation.getattr("name")?, relation)?; + relations_by_name.set_item(native_field(&relation, "name")?, relation)?; } Ok(relations_by_name) } @@ -317,16 +319,16 @@ fn resolve_model_deferral_inputs<'py>( let target_config: Bound<'py, PyAny> = deferred_target_config( &arguments.value("discovered_inputs")?, &defer_to, - &project.getattr("effective_target_name")?, + &native_field(&project, "effective_target_name")?, )?; let locations: Bound<'py, PyDict> = deferred_locations( &project, &DeferredNaming { target_config, - effective_vars: project.getattr("effective_vars")?, + effective_vars: native_field(&project, "effective_vars")?, default_schema: adapter.call_method0("default_schema")?, default_database: adapter.call_method0("default_database")?, - render_qualified_name: adapter.getattr("render_qualified_name")?, + render_qualified_name: native_field(&adapter, "render_qualified_name")?, }, )?; let relations: Bound<'py, PyDict> = deferred_relations( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs index b5fbf09e5b..a4ce5df811 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; @@ -105,13 +106,15 @@ pub(crate) fn report_compile_diagnostic( if let Ok(collector) = collector.cast::() { return collector.get().add(py, key, diagnostic); } - let location: Bound<'_, PyAny> = diagnostic.getattr("location")?; - let path: Bound<'_, PyAny> = diagnostic.getattr("path")?; + let location: Bound<'_, PyAny> = native_field(diagnostic, "location")?; + let path: Bound<'_, PyAny> = native_field(diagnostic, "path")?; let pointer: String = if !location.is_none() { format!( "\n --> {}:{}", - location.getattr("path")?.call_method0("as_posix")?.str()?, - location.getattr("line")?.str()? + native_field(&location, "path")? + .call_method0("as_posix")? + .str()?, + native_field(&location, "line")?.str()? ) } else if !path.is_none() { format!("\n --> {}", path.call_method0("as_posix")?.str()?) @@ -119,9 +122,9 @@ pub(crate) fn report_compile_diagnostic( String::new() }; let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("code", diagnostic.getattr("code")?)?; - keywords.set_item("help", diagnostic.getattr("help")?)?; - let message: String = format!("{}{pointer}", diagnostic.getattr("message")?.str()?); + keywords.set_item("code", native_field(diagnostic, "code")?)?; + keywords.set_item("help", native_field(diagnostic, "help")?)?; + let message: String = format!("{}{pointer}", native_field(diagnostic, "message")?.str()?); Err(PyErr::from_value( registry::error(py, ErrorId::CompileInput)?.call((message,), Some(&keywords))?, )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index 1ef39a7861..533040b9f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::graph::views::project_lineage_views; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -64,15 +65,16 @@ fn diff_graph<'py>(project: &Bound<'py, PyAny>, keys: &KeyIndexes<'py>) -> PyRes for attribute in ["models", "sources", "seeds"] { for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'py, PyAny> = resource?; - let name: String = resource.getattr("name")?.extract()?; + let name: String = native_field(&resource, "name")?.extract()?; graph .names - .push((name, keys.index(&resource.getattr("key")?)?)); + .push((name, keys.index(&native_field(&resource, "key")?)?)); } } - let upstream: Vec<(usize, Vec)> = adjacency(&lineage.getattr("upstream_deps")?, keys)?; + let upstream: Vec<(usize, Vec)> = + adjacency(&native_field(&lineage, "upstream_deps")?, keys)?; let downstream: Vec<(usize, Vec)> = - adjacency(&lineage.getattr("downstream_deps")?, keys)?; + adjacency(&native_field(&lineage, "downstream_deps")?, keys)?; for item in lineage .getattr("tag_index")? .call_method0("items")? @@ -162,14 +164,14 @@ fn diff_model_names<'py>( let model_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut names: Vec> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; - if chosen && key.getattr("resource_type")?.eq(&model_type)? { + if chosen && native_field(&key, "resource_type")?.eq(&model_type)? { names.push(model_reads::name_object(&model)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 81a1634eb1..92e7aeb24d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -43,6 +43,7 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -242,7 +243,7 @@ fn assembled_inputs<'py>( .getattr("setting_overrides")? .contains(SQL_ANALYSIS_SETTING_KEY)? { - local_config.getattr("settings")?.getattr("sql_analysis")? + native_field(&local_config, "settings")?.getattr("sql_analysis")? } else { project_config .getattr("settings")? @@ -343,7 +344,7 @@ fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { &[("item_count", item_count.into_pyobject(py)?.into_any())], )?; call_with( - &lifecycle.getattr("completed")?, + &native_field(lifecycle, "completed")?, &[("metadata", metadata.into_any())], )?; Ok(()) @@ -409,7 +410,7 @@ fn imported_python_paths(python: &Bound<'_, PyDict>) -> PyResult { } for group in groups { for item in group.try_iter()? { - paths.add(item?.getattr("relative_path")?)?; + paths.add(native_field(&item?, "relative_path")?)?; } } Ok(paths.len()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 2ae7d99e06..562e243f7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::model_config::template_data::run_id; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_indexes::{ @@ -123,7 +124,7 @@ pub(crate) fn attached_inputs<'py>( within(&collector, |violations| { Ok((compile_inputs(py, request)?, violations.clone())) })?; - let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; if !collected.is_truthy()? { return Ok(built); } @@ -148,15 +149,15 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult = target_context(request, &discovered, &adapter_context)?; let run_id: Bound<'py, PyAny> = run_id(&request.run_id.bind(py).clone())?; let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? - .call1((discovered.getattr("macro_files")?,))?; - let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; + .call1((native_field(&discovered, "macro_files")?,))?; + let syntax: Bound<'py, PyAny> = native_field(&adapter_context, "sql_lexical_syntax")?; let declaration_scope: Bound<'py, PyAny> = build_declaration_scope( &discovered, &loaded_macros, &syntax, Some(target.compile_cache_dir.clone()), )?; - let value_renderer: Bound<'py, PyAny> = adapter_context.getattr("value_renderer")?; + let value_renderer: Bound<'py, PyAny> = native_field(&adapter_context, "value_renderer")?; let model_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::ModelInputBuildContext)?, &[ @@ -166,16 +167,19 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult(py: Python<'py>, request: &InputsRequest) -> PyResult = keywords( py, &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), + ("local_config", native_field(&discovered, "local_config")?), ("discovered_inputs", discovered.clone()), ("sql_lexical_syntax", syntax), ("run_id", run_id), @@ -208,7 +215,10 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult(py: Python<'py>, request: &InputsRequest) -> PyResult( )?; let target_name: Bound<'py, PyAny> = selected.get_item(0)?; let target: Bound<'py, PyAny> = selected.get_item(1)?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; let cli_vars: Bound<'py, PyAny> = request.cli_vars.bind(py).clone(); let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { PyDict::new(py).into_any() @@ -269,16 +279,19 @@ fn target_context<'py>( ®istry::class(py, RecordId::MacroContext)?, &[ ("adapter_name", adapter_name), - ("sql_analysis_enabled", settings.getattr("sql_analysis")?), + ( + "sql_analysis_enabled", + native_field(&settings, "sql_analysis")?, + ), ("target_name", target_name.clone()), ("vars", vars.clone()), ( "_value_renderer", - adapter_context.getattr("value_renderer")?, + native_field(adapter_context, "value_renderer")?, ), ( "_collection_rendering", - adapter_context.getattr("collection_rendering")?, + native_field(adapter_context, "collection_rendering")?, ), ( "_enforce_explicit_references", @@ -375,7 +388,7 @@ fn attached_resources<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let context: &Bound<'py, PyAny> = &models.context; - let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let settings: Bound<'py, PyAny> = native_field(context, "effective_settings")?; let resources: Bound<'py, PyAny> = attach_resources( py, Some(keywords( @@ -389,8 +402,8 @@ fn attached_resources<'py>( py, [ settings.clone(), - context.getattr("target_config")?, - context.getattr("macro_context")?.getattr("adapter_name")?, + native_field(context, "target_config")?, + native_field(context, "macro_context")?.getattr("adapter_name")?, ], )? .into_any(), @@ -401,7 +414,10 @@ fn attached_resources<'py>( ), ( "python_functions_inherit_default_namespace", - adapter_context.getattr("python_functions_inherit_default_namespace")?, + native_field( + adapter_context, + "python_functions_inherit_default_namespace", + )?, ), ], )?), @@ -421,13 +437,16 @@ fn attached_resources<'py>( py, [ settings, - adapter_context.getattr("value_renderer")?, - context.getattr("sql_lexical_syntax")?, + native_field(adapter_context, "value_renderer")?, + native_field(context, "sql_lexical_syntax")?, ], )? .into_any(), ), - ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ( + "scan_cache", + native_field(declaration_scope, "sql_test_scans")?, + ), ("source_inputs", sources.clone()), ], )?), @@ -452,8 +471,8 @@ fn effective_connection<'py>( return Ok(resolved); } build_effective_connection( - &discovered.getattr("project_config")?, - &discovered.getattr("local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, &target.target.clone(), &target.vars.clone(), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 77adbbe5b2..07eb7e45d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -8,6 +8,7 @@ use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::third_party::ThirdPartyHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -43,15 +44,18 @@ pub(crate) fn integration_loader_functions<'py>( let factory: Bound<'py, PyAny> = host::entry(py, integration.loader_factory)?; for source_file in source_files.try_iter()? { let source_file: Bound<'py, PyAny> = source_file?; - for source_entry in source_file.getattr("source_entries")?.try_iter()? { + for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { let source_entry: Bound<'py, PyAny> = source_entry?; - let declared: Bound<'py, PyAny> = source_entry.getattr("integration_loader")?; - if declared.is_none() || declared.getattr("kind")?.extract::()? != kind { + let declared: Bound<'py, PyAny> = + native_field(&source_entry, "integration_loader")?; + if declared.is_none() + || native_field(&declared, "kind")?.extract::()? != kind + { continue; } let name: String = integration_loader_name( &kind, - &source_entry.getattr("name")?.extract::()?, + &native_field(&source_entry, "name")?.extract::()?, ); let function: Bound<'py, PyAny> = call_with( &factory, @@ -63,8 +67,11 @@ pub(crate) fn integration_loader_functions<'py>( loaders.push(call_with( &record_class, &[ - ("file_path", source_file.getattr("file_path")?), - ("relative_path", source_file.getattr("relative_path")?), + ("file_path", native_field(&source_file, "file_path")?), + ( + "relative_path", + native_field(&source_file, "relative_path")?, + ), ("name", name.into_pyobject(py)?.into_any()), ("function", function), ("connection_mode", external.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 1af7553338..0c8d7bb8db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_ada use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::manifest::builder::build_manifest; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -55,19 +56,21 @@ pub(crate) fn manifest_payload<'py>( } let started: f64 = monotonic(py)?; status.start("Building manifest...")?; - let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; - let adapter_type: Bound<'py, PyAny> = - resolve_effective_adapter_name(&project_config, &discovered.getattr("local_config")?)?; + let discovered: Bound<'py, PyAny> = native_field(analysis, "discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(&discovered, "project_config")?; + let graph: Bound<'py, PyAny> = native_field(analysis, "graph")?; + let adapter_type: Bound<'py, PyAny> = resolve_effective_adapter_name( + &project_config, + &native_field(&discovered, "local_config")?, + )?; let payload: Bound<'py, PyAny> = build_manifest(Some(&keywords( py, &[ - ("project", graph.getattr("project")?), + ("project", native_field(&graph, "project")?), ("plan_output", py.None().into_bound(py)), - ("project_name", project_config.getattr("name")?), + ("project_name", native_field(&project_config, "name")?), ("adapter_type", adapter_type), - ("graph", graph.getattr("native")?), + ("graph", native_field(&graph, "native")?), ], )?))? .into_bound(py) @@ -107,7 +110,7 @@ pub(crate) fn write_dag_artifact<'py>( let py: Python<'py> = analysis.py(); let started: f64 = monotonic(py)?; status.start("Writing DAG artifact...")?; - let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; + let discovered: Bound<'py, PyAny> = native_field(analysis, "discovered_inputs")?; let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; let project_name: String = discovered @@ -115,7 +118,7 @@ pub(crate) fn write_dag_artifact<'py>( .getattr("name")? .extract()?; let dag_json: String = build_dag_json( - &analysis.getattr("graph")?, + &native_field(analysis, "graph")?, &project_name, Some(&python_graph), )?; @@ -146,8 +149,8 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( Ok(planning_diagnostics( &target_dir(project_dir)?, ( - &analysis.getattr("adapter")?, - &analysis.getattr("graph")?.getattr("project")?, + &native_field(analysis, "adapter")?, + &native_field(analysis, "graph")?.getattr("project")?, ), Prepared { planned_tests: None, @@ -168,7 +171,7 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( } let reference_error: Bound<'py, PyAny> = registry::error(py, ErrorId::SqlTestReference)?; let location: Bound<'py, PyAny> = if value.is_instance(&reference_error)? { - value.getattr("location")? + native_field(&value, "location")? } else { py.None().into_bound(py) }; @@ -183,9 +186,9 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( "severity", registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, ), - ("code", value.getattr("code")?), + ("code", native_field(&value, "code")?), ("message", value.str()?.into_any()), - ("help", value.getattr("help")?), + ("help", native_field(&value, "help")?), ("location", location), ], )?; @@ -221,7 +224,7 @@ pub(crate) fn write_artifacts<'py>( py.None().into_bound(py) } else { call_with( - &prepared_artifacts.getattr("publish")?, + &native_field(prepared_artifacts, "publish")?, &[ ("target_dir", target.clone()), ("manifest", manifest.clone()), @@ -232,8 +235,8 @@ pub(crate) fn write_artifacts<'py>( write_static_target( &target, ( - &analysis.getattr("adapter")?, - &analysis.getattr("graph")?.getattr("project")?, + &native_field(analysis, "adapter")?, + &native_field(analysis, "graph")?.getattr("project")?, ), manifest, Prepared { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 924e59d7b7..b5de764392 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::diagnostics::trackers::current_tracker; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; @@ -82,12 +83,12 @@ fn compiled_phase<'py>( } else { registry::class(py, RecordId::CompilePipelineOptions)?.call0()? }; - let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; + let configured: Bound<'py, PyAny> = native_field(&options, "connection_config")?; let connection_config: Bound<'py, PyAny> = if configured.is_none() { build_effective_connection_config( discovered, - Some(options.getattr("selected_target")?), - Some(options.getattr("cli_vars")?), + Some(native_field(&options, "selected_target")?), + Some(native_field(&options, "cli_vars")?), )? } else { configured @@ -103,12 +104,18 @@ fn compiled_phase<'py>( &[ ("discovered_inputs", discovered.clone()), ("adapter", adapter.clone()), - ("selected_target", options.getattr("selected_target")?), - ("no_sql_validation", options.getattr("no_sql_validation")?), - ("cli_vars", options.getattr("cli_vars")?), + ( + "selected_target", + native_field(&options, "selected_target")?, + ), + ( + "no_sql_validation", + native_field(&options, "no_sql_validation")?, + ), + ("cli_vars", native_field(&options, "cli_vars")?), ( "external_sql_reference_resolver", - options.getattr("external_sql_reference_resolver")?, + native_field(&options, "external_sql_reference_resolver")?, ), ("resolved_connection", connection_config.clone()), ("analysis_selection", selection), @@ -147,14 +154,14 @@ fn reject_compile_errors( on_progress: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = project.py(); - let plan_sql_tests: bool = options.getattr("plan_sql_tests")?.is_truthy()?; + let plan_sql_tests: bool = native_field(options, "plan_sql_tests")?.is_truthy()?; let sql_test: Bound<'_, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_TEST")?; let mut errors: Vec> = Vec::new(); - for diagnostic in project.getattr("diagnostics")?.try_iter()? { + for diagnostic in native_field(project, "diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; - if diagnostic.getattr("is_error")?.is_truthy()? - && (plan_sql_tests || diagnostic.getattr("resource_type")?.ne(&sql_test)?) + if native_field(&diagnostic, "is_error")?.is_truthy()? + && (plan_sql_tests || native_field(&diagnostic, "resource_type")?.ne(&sql_test)?) { errors.push(diagnostic); } @@ -176,13 +183,13 @@ fn reject_compile_errors( }; lines.push(format!( "[{}] {}:{}:{}: {}", - diagnostic.getattr("code")?.str()?, - diagnostic.getattr("path")?.str()?, + native_field(diagnostic, "code")?.str()?, + native_field(diagnostic, "path")?.str()?, or_one("line")?, or_one("column")?, - diagnostic.getattr("message")?.str()?, + native_field(diagnostic, "message")?.str()?, )); - helps.add(diagnostic.getattr("help")?)?; + helps.add(native_field(diagnostic, "help")?)?; } let help: Bound<'_, PyAny> = if helps.len() == 1 { helps.pop().unwrap_or_else(|| py.None().into_bound(py)) @@ -193,7 +200,7 @@ fn reject_compile_errors( (lines.join("\n"),), Some(&keywords( py, - &[("code", first.getattr("code")?), ("help", help)], + &[("code", native_field(first, "code")?), ("help", help)], )?), )?; Err(PyErr::from_value(error)) @@ -222,10 +229,10 @@ fn run_compile_pipeline<'py>( } else { hooks }; - let on_progress: Bound<'py, PyAny> = hooks.getattr("on_progress")?; + let on_progress: Bound<'py, PyAny> = native_field(&hooks, "on_progress")?; let compiled: Bound<'py, PyAny> = compiled_phase((&discovered, &adapter), &options, &on_progress)?; - let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; let graph: Bound<'py, PyAny> = project_graph(&project)?; run_rules_gate( (&discovered, &adapter), @@ -243,7 +250,7 @@ fn run_rules_gate<'py>( ) -> PyResult<()> { let py: Python<'py> = discovered.py(); let rules_error: Bound<'py, PyAny> = registry::error(py, ErrorId::Rules)?; - let project_dir: Bound<'py, PyAny> = discovered.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(discovered, "project_dir")?; if project_dir.is_none() { return Err(PyErr::from_value( rules_error.call1(("compiler rules require a resolved project directory",))?, @@ -251,7 +258,7 @@ fn run_rules_gate<'py>( } let mut config: Bound<'py, PyAny> = load_rules_config(py, project_dir.extract()?)?.into_bound(py); - if options.getattr("no_cache")?.is_truthy()? { + if native_field(options, "no_cache")?.is_truthy()? { let cache: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::RulesCacheConfig)?, &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], @@ -280,7 +287,10 @@ fn run_rules_gate<'py>( ("project_dir", project_dir), ("dialect", dialect), ("selected_keys", selected_keys), - ("no_sql_analysis", options.getattr("no_sql_validation")?), + ( + "no_sql_analysis", + native_field(options, "no_sql_validation")?, + ), ], )?), )?; @@ -289,7 +299,7 @@ fn run_rules_gate<'py>( let message: String = rules_progress(elapsed, &result, true)?; on_progress.call1((message,))?; } - let findings: Bound<'py, PyAny> = result.getattr("findings")?; + let findings: Bound<'py, PyAny> = native_field(&result, "findings")?; if !findings.is_truthy()? { return Ok(()); } @@ -297,13 +307,16 @@ fn run_rules_gate<'py>( ®istry::class(py, RecordId::RulesResult)?, &[ ("findings", findings), - ("evaluated_models", result.getattr("evaluated_models")?), + ( + "evaluated_models", + native_field(&result, "evaluated_models")?, + ), ( "unevaluated_resources", - result.getattr("unevaluated_resources")?, + native_field(&result, "unevaluated_resources")?, ), - ("cache_hits", result.getattr("cache_hits")?), - ("cache_misses", result.getattr("cache_misses")?), + ("cache_hits", native_field(&result, "cache_hits")?), + ("cache_misses", native_field(&result, "cache_misses")?), ], )?; let rendered: String = format_rules_result(&summary, false)?; @@ -323,8 +336,11 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = - connection_with_hooks(adapter, &compiled.getattr("connection_config")?, hooks)?; + let opened: Bound<'py, PyAny> = connection_with_hooks( + adapter, + &native_field(compiled, "connection_config")?, + hooks, + )?; connection = Some(opened.clone()); pipeline_result( &PlanningInputs { @@ -332,9 +348,9 @@ fn planned<'py>( adapter, connection: &opened, }, - &compiled.getattr("project")?, + &native_field(compiled, "project")?, options, - &hooks.getattr("on_progress")?, + &native_field(hooks, "on_progress")?, ) })(); if let Some(connection) = connection.filter(|connection| !connection.is_none()) { @@ -345,13 +361,16 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; - write_plan_capture(&result.getattr("plan_output")?)?; + write_plan_capture(&native_field(&result, "plan_output")?)?; imported(py, "dataclasses", "replace")?.call( (result,), Some(&keywords( py, &[ - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(compiled, "compile_seconds")?, + ), ( "planning_seconds", planning_seconds.into_pyobject(py)?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 30e0a2fbd6..4a3a3da6c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -9,6 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, }; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -62,7 +63,7 @@ pub(crate) fn compile_analysis_selection<'py>( } } let selection: Bound<'py, PyAny> = registry::class(py, RecordId::CompileAnalysisSelection)?; - let no_cache: Bound<'py, PyAny> = options.getattr("no_cache")?; + let no_cache: Bound<'py, PyAny> = native_field(options, "no_cache")?; if (options .getattr("resolve_python_run_selectors")? .is_truthy()? @@ -77,9 +78,12 @@ pub(crate) fn compile_analysis_selection<'py>( call_with( &selection, &[ - ("select", options.getattr("select")?), - ("exclude", options.getattr("exclude")?), - ("auto_load_sources", options.getattr("auto_load_sources")?), + ("select", native_field(options, "select")?), + ("exclude", native_field(options, "exclude")?), + ( + "auto_load_sources", + native_field(options, "auto_load_sources")?, + ), ("no_cache", no_cache), ], ) @@ -103,7 +107,8 @@ pub(crate) fn build_project_graph_from_compiled_project<'py>( /// The project's natively held graph; dict views are built only when a boundary reads them. pub(crate) fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let native: Bound<'py, NativeProjectGraph> = project.getattr("lineage_graph")?.cast_into()?; + let native: Bound<'py, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; Ok(Bound::new( project.py(), CompiledProjectGraph::new(project.clone().unbind(), native.unbind()), @@ -153,9 +158,12 @@ pub(crate) fn build_static_pipeline_result<'py>( call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ - ("project", context.getattr("project")?), + ("project", native_field(&context, "project")?), ("plan_output", arguments.value("plan_output")?), - ("compile_seconds", context.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(&context, "compile_seconds")?, + ), ("planning_seconds", PyFloat::new(py, 0.0).into_any()), ], ) @@ -192,12 +200,15 @@ fn run_compile_only_pipeline<'py>( call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ - ("project", compiled.getattr("project")?), + ("project", native_field(&compiled, "project")?), ( "plan_output", registry::class(py, RecordId::PlanOutput)?.call0()?, ), - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(&compiled, "compile_seconds")?, + ), ("planning_seconds", PyFloat::new(py, 0.0).into_any()), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 9951b30ba3..12ced32ae3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -44,17 +45,17 @@ pub(crate) fn pipeline_result<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let discovered: &Bound<'py, PyAny> = inputs.discovered_inputs; - let select: Bound<'py, PyAny> = options.getattr("select")?; - let exclude: Bound<'py, PyAny> = options.getattr("exclude")?; + let select: Bound<'py, PyAny> = native_field(options, "select")?; + let exclude: Bound<'py, PyAny> = native_field(options, "exclude")?; let (deferred_locations, deferred_relations): Deferral<'py> = - deferral(inputs, project, &options.getattr("defer_to")?)?; + deferral(inputs, project, &native_field(options, "defer_to")?)?; let empty = || -> PyResult> { Ok(PyFrozenSet::empty(py)?.into_any()) }; let mut selected_sql_keys: Bound<'py, PyAny> = py.None().into_bound(py); let mut selected_python_node_names: Bound<'py, PyAny> = empty()?; let mut python_read_source_names: Bound<'py, PyAny> = empty()?; let mut run_selection: Bound<'py, PyAny> = py.None().into_bound(py); - let sql_test_selection: Bound<'py, PyAny> = options.getattr("sql_test_selection")?; - if !sql_test_selection.getattr("models_selected")?.is_truthy()? { + let sql_test_selection: Bound<'py, PyAny> = native_field(options, "sql_test_selection")?; + if !native_field(&sql_test_selection, "models_selected")?.is_truthy()? { selected_sql_keys = empty()?; } else if options .getattr("resolve_python_run_selectors")? @@ -66,8 +67,8 @@ pub(crate) fn pipeline_result<'py>( )?; run_selection = run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; - selected_sql_keys = run_selection.getattr("sql_keys")?; - selected_python_node_names = run_selection.getattr("python_node_names")?; + selected_sql_keys = native_field(&run_selection, "sql_keys")?; + selected_python_node_names = native_field(&run_selection, "python_node_names")?; python_read_source_names = python_read_source_names_for( py, Some(keywords( @@ -138,14 +139,17 @@ pub(crate) fn pipeline_result<'py>( ("deferral", deferral), ("policies", policies), ("on_progress", on_progress.clone()), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), + ( + "project_config", + native_field(discovered, "project_config")?, + ), + ("local_config", native_field(discovered, "local_config")?), ], )?), )?; let custom_materializations: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadCustomMaterializations)? - .call1((discovered.getattr("materialization_files")?,))?; + .call1((native_field(discovered, "materialization_files")?,))?; let python_outputs: Bound<'py, PyAny> = build_python_run_plan_outputs( py, Some(keywords( @@ -162,15 +166,15 @@ pub(crate) fn pipeline_result<'py>( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ ("project", project.clone()), - ("plan_output", python_outputs.getattr("plan_output")?), + ("plan_output", native_field(&python_outputs, "plan_output")?), ("custom_materializations", custom_materializations), ( "python_node_names", - python_outputs.getattr("selected_python_node_names")?, + native_field(&python_outputs, "selected_python_node_names")?, ), ( "python_plan_entries", - python_outputs.getattr("python_plan_entries")?, + native_field(&python_outputs, "python_plan_entries")?, ), ], ) @@ -207,7 +211,7 @@ fn deferral<'py>( ("defer_to", defer_to.clone()), ( "current_target_name", - project.getattr("effective_target_name")?, + native_field(project, "effective_target_name")?, ), ], )?), @@ -221,7 +225,7 @@ fn deferral<'py>( &[ ("project", project.clone()), ("deferred_target_config", target_config), - ("effective_vars", project.getattr("effective_vars")?), + ("effective_vars", native_field(project, "effective_vars")?), ("default_schema", adapter.call_method0("default_schema")?), ( "default_database", @@ -229,7 +233,7 @@ fn deferral<'py>( ), ( "render_qualified_name", - adapter.getattr("render_qualified_name")?, + native_field(adapter, "render_qualified_name")?, ), ], )?), @@ -259,7 +263,7 @@ pub(crate) fn connection_with_hooks<'py>( hooks: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let on_start: Bound<'py, PyAny> = hooks.getattr("on_connection_start")?; + let on_start: Bound<'py, PyAny> = native_field(hooks, "on_connection_start")?; if !on_start.is_none() { on_start.call1((SINGLE_CONNECTION,))?; } @@ -273,14 +277,14 @@ pub(crate) fn connection_with_hooks<'py>( { Ok(connection) => connection, Err(error) => { - let on_error: Bound<'py, PyAny> = hooks.getattr("on_connection_error")?; + let on_error: Bound<'py, PyAny> = native_field(hooks, "on_connection_error")?; if error.is_instance_of::(hooks.py()) && !on_error.is_none() { on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } return Err(error); } }; - let on_complete: Bound<'py, PyAny> = hooks.getattr("on_connection_complete")?; + let on_complete: Bound<'py, PyAny> = native_field(hooks, "on_connection_complete")?; if !on_complete.is_none() { on_complete.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 850ff27a5b..60a2019f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -137,12 +138,13 @@ fn python_compiled_project<'py>( named_target_schema_strategy(discovered, adapter, request.selected_target.bind(py))?; managed_loader_target_isolation(discovered, adapter, request.resolved_connection.bind(py))?; let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); - let no_cache: bool = !selection.is_none() && selection.getattr("no_cache")?.is_truthy()?; + let no_cache: bool = + !selection.is_none() && native_field(selection, "no_cache")?.is_truthy()?; let compile_inputs: Bound<'py, PyAny> = record_compile_timing("attachment_ms", || { frontier_compile_inputs(py, request, no_cache) })?; if !selection.is_none() { - let on_inputs_ready: Bound<'py, PyAny> = selection.getattr("on_inputs_ready")?; + let on_inputs_ready: Bound<'py, PyAny> = native_field(selection, "on_inputs_ready")?; if !on_inputs_ready.is_none() { call_with(&on_inputs_ready, &[("inputs", compile_inputs.clone())])?; } @@ -168,7 +170,7 @@ fn python_compiled_project<'py>( ), ( "analysis_cache_dir", - compile_inputs.getattr("compile_cache_dir")?, + native_field(&compile_inputs, "compile_cache_dir")?, ), ("analysis_model_names", analysis_model_names), ], @@ -187,7 +189,7 @@ fn targeted_project<'py>( let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); managed_write_schemas(adapter, project)?; - let effective_schema: Bound<'py, PyAny> = project.getattr("effective_target_schema")?; + let effective_schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?; let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { effective_schema } else { @@ -196,13 +198,13 @@ fn targeted_project<'py>( let defaults: TargetDefaults<'py> = TargetDefaults { default_schema, default_database: adapter.call_method0("default_database")?, - render_qualified_name: adapter.getattr("render_qualified_name")?, + render_qualified_name: native_field(adapter, "render_qualified_name")?, python_functions_inherit_default_namespace: adapter .call_method0("python_functions_inherit_default_namespace")?, }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name(&project_config, &local_config.clone()) .and_then(|value| value.into_bound_py_any(py))?; @@ -248,7 +250,7 @@ fn frontier_compile_inputs<'py>( let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( - &discovered.getattr("project_config")?, + &native_field(discovered, "project_config")?, &py.None().into_bound(py), ) .and_then(|value| value.into_bound_py_any(py))?; @@ -261,7 +263,10 @@ fn frontier_compile_inputs<'py>( "python_functions_inherit_default_namespace", adapter.call_method0("python_functions_inherit_default_namespace")?, ), - ("sql_lexical_syntax", adapter.getattr("sql_lexical_syntax")?), + ( + "sql_lexical_syntax", + native_field(adapter, "sql_lexical_syntax")?, + ), ], )?; let inputs: InputsRequest = InputsRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 625a1a4cac..dd794cb26c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -31,8 +32,8 @@ impl<'py> NodeKinds<'py> { fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?; Ok(Self { - task: kinds.getattr("TASK")?, - asset: kinds.getattr("ASSET")?, + task: native_field(&kinds, "TASK")?, + asset: native_field(&kinds, "ASSET")?, }) } @@ -89,14 +90,14 @@ pub(crate) fn python_read_source_names_for<'py>( host::entry(py, UserCodeHost::SqlResourceRefKind)?.getattr("SOURCE")?; let mut names: Vec> = Vec::new(); for reference in call_with( - &graph.getattr("selected_sql_refs")?, + &native_field(&graph, "selected_sql_refs")?, &[("selected_names", selected)], )? .try_iter()? { let reference: Bound<'py, PyAny> = reference?; - if reference.getattr("kind")?.eq(&source)? { - names.push(reference.getattr("name")?); + if native_field(&reference, "kind")?.eq(&source)? { + names.push(native_field(&reference, "name")?); } } frozenset.call1((PyTuple::new(py, names)?,)) @@ -158,7 +159,7 @@ pub(crate) fn build_python_run_plan_outputs<'py>( let selection: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::PythonSqlRunSelection)?, &[ - ("sql_keys", plan_output.getattr("selected_keys")?), + ("sql_keys", native_field(&plan_output, "selected_keys")?), ("python_node_names", selected.clone()), ], )?; @@ -166,7 +167,7 @@ pub(crate) fn build_python_run_plan_outputs<'py>( let entries: Bound<'py, PyTuple> = plan_entries( &lifecycle, (&graph, &kinds), - &plan_output.getattr("python_identity_fingerprints")?, + &native_field(&plan_output, "python_identity_fingerprints")?, )?; call_with( &outputs, @@ -217,7 +218,7 @@ fn plan_entries<'py>( let phases: Bound<'py, PyAny> = registry::class(py, RecordId::PythonRunPhase)?; let entry_class: Bound<'py, PyAny> = registry::class(py, RecordId::PythonPlanEntry)?; let upstream: HashMap> = upstream_names(graph)?; - let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; + let nodes: Bound<'py, PyAny> = native_field(graph, "nodes_by_name")?; let mut entries: Vec> = Vec::new(); for (attribute, phase) in [ ("ingress_python_node_names", "PRE_SQL_INGRESS"), @@ -230,11 +231,11 @@ fn plan_entries<'py>( .collect::>()?; for name in ordered_python_names(&selected, &upstream) { let node: Bound<'py, PyAny> = nodes.get_item(name.as_str())?; - let kind: Bound<'py, PyAny> = node.getattr("kind")?; + let kind: Bound<'py, PyAny> = native_field(&node, "kind")?; if !kinds.runs(&kind)? { continue; } - let identity: Bound<'py, PyAny> = node.getattr("identity")?; + let identity: Bound<'py, PyAny> = native_field(&node, "identity")?; let previous: Bound<'py, PyAny> = previous_identity(&identity, previous_identities)?; let attribute_or_none = |value: &Bound<'py, PyAny>, name: &str| { if value.is_none() { @@ -246,7 +247,7 @@ fn plan_entries<'py>( entries.push(call_with( &entry_class, &[ - ("name", node.getattr("name")?), + ("name", native_field(&node, "name")?), ("kind", kind), ("phase", phases.getattr(phase)?), ( @@ -269,7 +270,7 @@ fn plan_entries<'py>( "previous_metadata_json", attribute_or_none(&previous, "metadata_json")?, ), - ("provider_usages", node.getattr("provider_usages")?), + ("provider_usages", native_field(&node, "provider_usages")?), ], )?); } @@ -288,8 +289,8 @@ fn previous_identity<'py>( let key: Bound<'py, PyTuple> = PyTuple::new( identity.py(), [ - identity.getattr("node_type")?, - identity.getattr("node_name")?, + native_field(identity, "node_type")?, + native_field(identity, "node_name")?, ], )?; previous_identities.call_method1("get", (key,)) @@ -303,19 +304,19 @@ fn identity_status<'py>( let statuses: Bound<'py, PyAny> = registry::class(identity.py(), RecordId::PythonIdentityStatus)?; if identity.is_none() { - return statuses.getattr("UNKNOWN"); + return native_field(&statuses, "UNKNOWN"); } let previous: Bound<'py, PyAny> = previous_identity(identity, previous_identities)?; if previous.is_none() { - return statuses.getattr("NEW"); + return native_field(&statuses, "NEW"); } if previous .getattr("version_hash")? - .eq(identity.getattr("version_hash")?)? + .eq(native_field(identity, "version_hash")?)? { - return statuses.getattr("UNCHANGED"); + return native_field(&statuses, "UNCHANGED"); } - statuses.getattr("CHANGED") + native_field(&statuses, "CHANGED") } /// Warnings for skipped task/asset deps of planned source loaders. @@ -326,15 +327,15 @@ fn skipped_ingress_warnings<'py>( ) -> PyResult>> { let py: Python<'py> = plan_output.py(); let mut loaders: Vec = Vec::new(); - for entry in plan_output.getattr("source_load_entries")?.try_iter()? { - let loader: String = entry?.getattr("loader")?.extract()?; + for entry in native_field(plan_output, "source_load_entries")?.try_iter()? { + let loader: String = native_field(&entry?, "loader")?.extract()?; if !loaders.contains(&loader) { loaders.push(loader); } } loaders.sort(); - let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; - let selected: Bound<'py, PyAny> = run_selection.getattr("python_node_names")?; + let nodes: Bound<'py, PyAny> = native_field(graph, "nodes_by_name")?; + let selected: Bound<'py, PyAny> = native_field(run_selection, "python_node_names")?; let upstream: HashMap> = upstream_names(graph)?; let warning_class: Bound<'py, PyAny> = registry::class(py, RecordId::PlanWarning)?; let severity: Bound<'py, PyAny> = @@ -349,7 +350,7 @@ fn skipped_ingress_warnings<'py>( continue; } let kind: Bound<'py, PyAny> = - nodes.get_item(upstream_name.as_str())?.getattr("kind")?; + native_field(&nodes.get_item(upstream_name.as_str())?, "kind")?; if !kinds.runs(&kind)? { continue; } @@ -357,7 +358,7 @@ fn skipped_ingress_warnings<'py>( "Source loader '{loader}' has unselected upstream {} '{upstream_name}'. SQLBuild \ will not run it and cannot verify side effects or payload availability; use \ +source:{loader} to refresh upstream ingress dependencies.", - kind.getattr("value")?.str()? + native_field(&kind, "value")?.str()? ); warnings.push(call_with( &warning_class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index 03045bde39..92cadbfb21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::references::source_relations::render_source_relation; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; @@ -27,9 +28,9 @@ impl<'py> RefKinds<'py> { fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host::entry(py, UserCodeHost::SqlResourceRefKind)?; Ok(Self { - model: kinds.getattr("MODEL")?, - seed: kinds.getattr("SEED")?, - source: kinds.getattr("SOURCE")?, + model: native_field(&kinds, "MODEL")?, + seed: native_field(&kinds, "SEED")?, + source: native_field(&kinds, "SOURCE")?, }) } } @@ -145,8 +146,8 @@ fn resolved_relation<'py>( if !planned.is_none() { return Ok(planned); } - let kind: Bound<'py, PyAny> = reference.getattr("kind")?; - let name: Bound<'py, PyAny> = reference.getattr("name")?; + let kind: Bound<'py, PyAny> = native_field(reference, "kind")?; + let name: Bound<'py, PyAny> = native_field(reference, "name")?; if kind.eq(&kinds.source)? { return Err(shared_error( py, @@ -164,8 +165,8 @@ fn resolved_relation<'py>( let mut location: Option> = None; for resource in resources.try_iter()? { let resource: Bound<'py, PyAny> = resource?; - if resource.getattr("name")?.eq(&name)? { - location = Some(resource.getattr("destination")?); + if native_field(&resource, "name")?.eq(&name)? { + location = Some(native_field(&resource, "destination")?); break; } } @@ -174,7 +175,7 @@ fn resolved_relation<'py>( py, format!( "Python node references unknown {} '{}'", - kind.getattr("value")?.str()?, + native_field(&kind, "value")?.str()?, name.str()? ), )); @@ -204,8 +205,8 @@ fn planned_relation<'py>( kinds: &RefKinds<'py>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let kind: Bound<'py, PyAny> = reference.getattr("kind")?; - let name: Bound<'py, PyAny> = reference.getattr("name")?; + let kind: Bound<'py, PyAny> = native_field(reference, "kind")?; + let name: Bound<'py, PyAny> = native_field(reference, "name")?; if kind.eq(&kinds.source)? { let source: Bound<'py, PyAny> = plan_output .getattr("python_source_entries")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index f94c859c10..edde609702 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::planner_entries::command_plans::{ }; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -24,7 +25,8 @@ const DATACLASSES_MODULE: &str = "dataclasses"; /// Whether a selection names nothing to select or exclude. fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { - Ok(!selection.getattr("select")?.is_truthy()? && !selection.getattr("exclude")?.is_truthy()?) + Ok(!native_field(selection, "select")?.is_truthy()? + && !native_field(selection, "exclude")?.is_truthy()?) } /// Options whose selectors name only resources, with unit tests selected apart. @@ -42,19 +44,22 @@ fn unit_test_selectors_applied<'py>( return Ok(resolved); } let split: Bound<'py, PyAny> = split_project_unit_test_selectors( - &resolved.getattr("select")?, - &resolved.getattr("exclude")?, + &native_field(&resolved, "select")?, + &native_field(&resolved, "exclude")?, discovered, - resolved.getattr("accepts_unit_test_selectors")?.extract()?, + native_field(&resolved, "accepts_unit_test_selectors")?.extract()?, )?; imported(py, DATACLASSES_MODULE, "replace")?.call( (resolved,), Some(&keywords( py, &[ - ("select", split.getattr("select")?), - ("exclude", split.getattr("exclude")?), - ("sql_test_selection", split.getattr("sql_test_selection")?), + ("select", native_field(&split, "select")?), + ("exclude", native_field(&split, "exclude")?), + ( + "sql_test_selection", + native_field(&split, "sql_test_selection")?, + ), ], )?), ) @@ -71,13 +76,16 @@ fn planner_scopes<'py>( &call_with( ®istry::class(py, RecordId::PlannerSelection)?, &[ - ("select", selection.getattr("select")?), - ("exclude", selection.getattr("exclude")?), + ("select", native_field(selection, "select")?), + ("exclude", native_field(selection, "exclude")?), ], )?, &call_with( ®istry::class(py, RecordId::PlannerPolicies)?, - &[("auto_load_sources", selection.getattr("auto_load_sources")?)], + &[( + "auto_load_sources", + native_field(selection, "auto_load_sources")?, + )], )?, ) } @@ -123,7 +131,7 @@ pub(crate) fn resolve_configured_rule_selection<'py>( let keys: Vec> = models .call_method0("values")? .try_iter()? - .map(|model| model?.getattr("key")) + .map(|model| native_field(&model?, "key")) .collect::>()?; Ok(PyFrozenSet::new(py, keys)?.into_any()) } @@ -167,7 +175,7 @@ fn deferred_locations<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); - let defer_to: Bound<'py, PyAny> = options.getattr("defer_to")?; + let defer_to: Bound<'py, PyAny> = native_field(options, "defer_to")?; if defer_to.is_none() { return Ok(py.None().into_bound(py)); } @@ -180,7 +188,7 @@ fn deferred_locations<'py>( ("defer_to", defer_to), ( "current_target_name", - project.getattr("effective_target_name")?, + native_field(project, "effective_target_name")?, ), ], )?), @@ -192,7 +200,7 @@ fn deferred_locations<'py>( &[ ("project", project.clone()), ("deferred_target_config", deferred_target), - ("effective_vars", project.getattr("effective_vars")?), + ("effective_vars", native_field(project, "effective_vars")?), ("default_schema", adapter.call_method0("default_schema")?), ( "default_database", @@ -200,7 +208,7 @@ fn deferred_locations<'py>( ), ( "render_qualified_name", - adapter.getattr("render_qualified_name")?, + native_field(adapter, "render_qualified_name")?, ), ], )?), @@ -246,7 +254,7 @@ pub(crate) fn compile_static_command_context<'py>( ], )?), )?; - let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; let deferred: Bound<'py, PyAny> = deferred_locations((&discovered, &adapter), &options, &project)?; let scope: Bound<'py, PyAny> = resolve_static_command_scope( @@ -254,13 +262,16 @@ pub(crate) fn compile_static_command_context<'py>( &call_with( ®istry::class(py, RecordId::PlannerSelection)?, &[ - ("select", options.getattr("select")?), - ("exclude", options.getattr("exclude")?), + ("select", native_field(&options, "select")?), + ("exclude", native_field(&options, "exclude")?), ("selected_keys", selected_keys), - ("sql_test_selection", options.getattr("sql_test_selection")?), + ( + "sql_test_selection", + native_field(&options, "sql_test_selection")?, + ), ], )?, - options.getattr("auto_load_sources")?.extract()?, + native_field(&options, "auto_load_sources")?.extract()?, )?; let relations: Bound<'py, PyAny> = resolve_static_relation_context( py, @@ -276,16 +287,22 @@ pub(crate) fn compile_static_command_context<'py>( ®istry::class(py, RecordId::DeferralInputs)?, &[ ("deferred_locations", deferred), - ("defer_sources_to", options.getattr("defer_sources_to")?), + ( + "defer_sources_to", + native_field(&options, "defer_sources_to")?, + ), ( "source_deferral_enabled", - options.getattr("source_deferral_enabled")?, + native_field(&options, "source_deferral_enabled")?, ), ], )?, ), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), + ("local_config", native_field(&discovered, "local_config")?), ( "relation_keys", request.optional("relation_keys")?.unwrap_or_else(none), @@ -299,8 +316,14 @@ pub(crate) fn compile_static_command_context<'py>( ("project", project), ("scope", scope), ("relations", relations), - ("connection_config", compiled.getattr("connection_config")?), - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "connection_config", + native_field(&compiled, "connection_config")?, + ), + ( + "compile_seconds", + native_field(&compiled, "compile_seconds")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 18cd253746..e15fb7b9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeComp use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -46,7 +47,7 @@ pub(crate) fn apply_target_defaults<'py>( let py: Python<'py> = project.py(); let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; let mut models: Vec> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; models.push(if let Ok(native) = model.cast::() { Bound::new( @@ -65,14 +66,14 @@ pub(crate) fn apply_target_defaults<'py>( py, &[( "destination", - resolved_target(&model.getattr("destination")?, defaults)?, + resolved_target(&native_field(&model, "destination")?, defaults)?, )], )?), )? }); } let mut seeds: Vec> = Vec::new(); - for seed in project.getattr("seeds")?.try_iter()? { + for seed in native_field(project, "seeds")?.try_iter()? { let seed: Bound<'py, PyAny> = seed?; seeds.push(if let Ok(native) = seed.cast::() { Bound::new( @@ -91,14 +92,14 @@ pub(crate) fn apply_target_defaults<'py>( py, &[( "destination", - resolved_target(&seed.getattr("destination")?, defaults)?, + resolved_target(&native_field(&seed, "destination")?, defaults)?, )], )?), )? }); } let mut functions: Vec> = Vec::new(); - for function in project.getattr("functions")?.try_iter()? { + for function in native_field(project, "functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; let native: &Bound<'py, NativeCompiledFunction> = function.cast()?; functions.push( @@ -124,14 +125,14 @@ pub(crate) fn apply_target_defaults<'py>( ( "effective_target_database", or_else( - project.getattr("effective_target_database")?, + native_field(project, "effective_target_database")?, &defaults.default_database, )?, ), ( "effective_target_schema", or_else( - project.getattr("effective_target_schema")?, + native_field(project, "effective_target_schema")?, &defaults.default_schema, )?, ), @@ -149,8 +150,8 @@ fn resolved_target<'py>( defaults: &TargetDefaults<'py>, ) -> PyResult> { let py: Python<'py> = target.py(); - let target_schema: Bound<'py, PyAny> = target.getattr("schema")?; - let target_database: Bound<'py, PyAny> = target.getattr("database")?; + let target_schema: Bound<'py, PyAny> = native_field(target, "schema")?; + let target_database: Bound<'py, PyAny> = native_field(target, "database")?; let schema: Bound<'py, PyAny> = if target_schema.is_none() { defaults.default_schema.clone() } else { @@ -161,7 +162,7 @@ fn resolved_target<'py>( } else { target_database.clone() }; - let name: Bound<'py, PyAny> = target.getattr("name")?; + let name: Bound<'py, PyAny> = native_field(target, "name")?; let qualified_name: Bound<'py, PyAny> = call_with( &defaults.render_qualified_name, &[ @@ -172,7 +173,7 @@ fn resolved_target<'py>( )?; if schema.eq(&target_schema)? && database.eq(&target_database)? - && qualified_name.eq(target.getattr("qualified_name")?)? + && qualified_name.eq(native_field(target, "qualified_name")?)? { return Ok(target.clone()); } @@ -183,8 +184,11 @@ fn resolved_target<'py>( ("schema", schema), ("name", name), ("qualified_name", qualified_name), - ("logical_schema", target.getattr("logical_schema")?), - ("logical_database", target.getattr("logical_database")?), + ("logical_schema", native_field(target, "logical_schema")?), + ( + "logical_database", + native_field(target, "logical_database")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 9e1e0ecd0d..26249342fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -42,6 +42,7 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::discovery::guidance::setting_help; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -128,8 +129,8 @@ pub(crate) fn named_target_schema_strategy( { return Ok(()); } - let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let target_name: Bound<'_, PyAny> = resolve_target_name( py, &project_config.clone(), @@ -143,7 +144,7 @@ pub(crate) fn named_target_schema_strategy( let target_config: Bound<'_, PyAny> = resolve_target_config(py, &project_config, &local_config, &target_name.clone()) .and_then(|value| value.into_bound_py_any(py))?; - let schema: Option = text_or_none(&target_config.getattr("schema")?)?; + let schema: Option = text_or_none(&native_field(&target_config, "schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); } @@ -185,10 +186,10 @@ pub(crate) fn managed_loader_target_isolation( ) -> PyResult<()> { let py: Python<'_> = discovered.py(); let mut managed_sources: Vec> = Vec::new(); - for source_file in discovered.getattr("source_files")?.try_iter()? { - for entry in source_file?.getattr("source_entries")?.try_iter()? { + for source_file in native_field(discovered, "source_files")?.try_iter()? { + for entry in native_field(&source_file?, "source_entries")?.try_iter()? { let entry: Bound<'_, PyAny> = entry?; - if entry.getattr("managed")?.is_truthy()? { + if native_field(&entry, "managed")?.is_truthy()? { managed_sources.push(entry); } } @@ -196,11 +197,11 @@ pub(crate) fn managed_loader_target_isolation( if managed_sources.is_empty() { return Ok(()); } - let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let mut target_names: Vec = Vec::new(); for config in [&project_config, &local_config] { - for name in config.getattr("targets")?.try_iter()? { + for name in native_field(config, "targets")?.try_iter()? { let name: String = name?.extract()?; if !target_names.contains(&name) { target_names.push(name); @@ -261,8 +262,8 @@ fn target_namespaces( let name: Bound<'_, PyAny> = PyString::new(py, target_name).into_any(); let target_config: Bound<'_, PyAny> = resolve_target_config( py, - &discovered.getattr("project_config")?, - &discovered.getattr("local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, &name.clone(), ) .and_then(|value| value.into_bound_py_any(py))?; @@ -329,17 +330,17 @@ fn managed_source_namespace( let py: Python<'_> = source_entry.py(); let database: Bound<'_, PyAny> = first_truthy( &[ - source_entry.getattr("database")?, - target_config.getattr("database")?, + native_field(source_entry, "database")?, + native_field(target_config, "database")?, connection.call_method1("get", ("database",))?, ], || adapter.call_method0("default_database"), )?; let schema: Bound<'_, PyAny> = first_truthy( &[ - source_entry.getattr("schema")?, - target_config.getattr("loader_schema")?, - target_config.getattr("schema")?, + native_field(source_entry, "schema")?, + native_field(target_config, "loader_schema")?, + native_field(target_config, "schema")?, connection.call_method1("get", ("schema",))?, ], || adapter.call_method0("default_schema"), @@ -405,8 +406,11 @@ pub(crate) fn project_targets( let mut targets: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'_, PyAny> = resource?; - let resource_name: String = resource.getattr("name")?.str()?.to_cow()?.into_owned(); - let destination: Bound<'_, PyAny> = resource.getattr("destination")?; + let resource_name: String = native_field(&resource, "name")? + .str()? + .to_cow()? + .into_owned(); + let destination: Bound<'_, PyAny> = native_field(&resource, "destination")?; match targets .iter_mut() .find(|(existing, _)| *existing == resource_name) @@ -429,8 +433,8 @@ fn required_target_parts( target: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = target.py(); - let database: Option = optional_text(&target.getattr("database")?)?; - let schema: Option = optional_text(&target.getattr("schema")?)?; + let database: Option = optional_text(&native_field(target, "database")?)?; + let schema: Option = optional_text(&native_field(target, "schema")?)?; let missing: Vec<&str> = missing_target_parts(database.as_deref(), schema.as_deref()); let Some(first) = missing.first() else { return Ok(()); @@ -516,24 +520,27 @@ pub(crate) fn managed_write_schemas( for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'_, PyAny> = resource?; managed_write_schema( - &format!("{label} '{}'", resource.getattr("name")?.str()?.to_cow()?), - &resource.getattr("destination")?.getattr("schema")?, + &format!( + "{label} '{}'", + native_field(&resource, "name")?.str()?.to_cow()? + ), + &native_field(&resource, "destination")?.getattr("schema")?, connection_schema.as_deref(), )?; } } - for source in project.getattr("sources")?.try_iter()? { + for source in native_field(project, "sources")?.try_iter()? { let source: Bound<'_, PyAny> = source?; - let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; - if !entry.getattr("managed")?.is_truthy()? { + let entry: Bound<'_, PyAny> = native_field(&source, "source_entry")?; + if !native_field(&entry, "managed")?.is_truthy()? { continue; } managed_write_schema( &format!( "Managed source '{}'", - source.getattr("name")?.str()?.to_cow()? + native_field(&source, "name")?.str()?.to_cow()? ), - &entry.getattr("schema")?, + &native_field(&entry, "schema")?, connection_schema.as_deref(), )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 6915e80186..62c118242b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -185,7 +185,7 @@ impl NativeSourceInput { let attached: &AttachedSource = &source.attached; Ok(( attached.name.clone(), - !native_field(&attached.entry.bind(py), "loader")?.is_none(), + !native_field(attached.entry.bind(py), "loader")?.is_none(), attached.database.clone(), attached.schema.clone(), )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 30d45446e9..b6599f754c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -26,9 +26,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const GENERATE_ACTION: &str = "generate"; @@ -89,10 +89,13 @@ fn select_resources<'py>( py, &[ ("discovered_inputs", discovered.clone()), - ("adapter", native_field(&context, "adapter")?), - ("selected_target", native_field(&request, "from_target")?), - ("cli_vars", native_field(&request, "cli_vars")?), - ("resolved_connection", native_field(&context, "connection_config")?), + ("adapter", native_field(context, "adapter")?), + ("selected_target", native_field(request, "from_target")?), + ("cli_vars", native_field(request, "cli_vars")?), + ( + "resolved_connection", + native_field(context, "connection_config")?, + ), ("analysis_selection", analysis), ], )?), @@ -105,8 +108,8 @@ fn select_resources<'py>( ®istry::native(py, NativeAttr::SelectProjectGraph)?, &[ ("graph", graph.clone()), - ("select", native_field(&request, "select")?), - ("exclude", native_field(&request, "exclude")?), + ("select", native_field(request, "select")?), + ("exclude", native_field(request, "exclude")?), ], )?; let compiled: Bound<'py, PyAny> = native_field(&graph, "project")?; @@ -165,11 +168,11 @@ fn compare<'py>( progress: &Progress<'py>, ) -> PyResult> { let py: Python<'py> = request.py(); - let adapter: Bound<'py, PyAny> = native_field(&context, "adapter")?; + let adapter: Bound<'py, PyAny> = native_field(context, "adapter")?; let reporter: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ConnectionProgressReporter)?, &[ - ("adapter_name", native_field(&context, "adapter_name")?), + ("adapter_name", native_field(context, "adapter_name")?), ("stream", progress.stream.clone()), ( "use_color", @@ -180,7 +183,7 @@ fn compare<'py>( let connection_started: Instant = Instant::now(); reporter.call_method1("on_connection_start", (1,))?; let connection: Bound<'py, PyAny> = - match adapter.call_method1("connect", (native_field(&context, "connection_config")?,)) { + match adapter.call_method1("connect", (native_field(context, "connection_config")?,)) { Ok(connection) => connection, Err(error) => { if exception(py, &error) { @@ -192,7 +195,7 @@ fn compare<'py>( let compared: PyResult> = (|| { connection_timing(&reporter, "on_connection_complete", connection_started)?; let count: usize = selection.models.len() + selection.sources.len(); - let target: String = native_field(&request, "from_target")?.str()?.to_string(); + let target: String = native_field(request, "from_target")?.str()?.to_string(); let status: Bound<'py, PyAny> = progress.status()?; let started: Instant = Instant::now(); status.call_method1( @@ -259,10 +262,10 @@ fn write_updates<'py>( status.call_method1("start", ("Updating repository contracts...",))?; let write: ContractWrite<'py> = ContractWrite { project_dir: project_dir.clone(), - overwrite: native_field(&request, "overwrite")?, - adapter: native_field(&context, "adapter")?, - resolved_connection: native_field(&context, "connection_config")?, - cli_vars: native_field(&request, "cli_vars")?, + overwrite: native_field(request, "overwrite")?, + adapter: native_field(context, "adapter")?, + resolved_connection: native_field(context, "connection_config")?, + cli_vars: native_field(request, "cli_vars")?, }; let updated: Bound<'py, PyAny> = match write_contracts(&write, graph, &result) { Ok(updated) => updated, @@ -323,7 +326,7 @@ fn resource_text(item: Bound<'_, PyAny>) -> PyResult { fn report(result: &Bound<'_, PyAny>) -> PyResult { Ok(ContractReport { - from_target: native_field(&result, "from_target")?.str()?.to_string(), + from_target: native_field(result, "from_target")?.str()?.to_string(), resources: result .getattr("evidence")? .try_iter()? @@ -339,12 +342,13 @@ fn report(result: &Bound<'_, PyAny>) -> PyResult { fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); - let json_output: bool = native_field(&request, "json_output")?.is_truthy()?; + let json_output: bool = native_field(request, "json_output")?.is_truthy()?; let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let use_color: bool = - !native_field(&request, "no_color")?.is_truthy()? && !json_output && stdout_supports_color(py)?; + let use_color: bool = !native_field(request, "no_color")?.is_truthy()? + && !json_output + && stdout_supports_color(py)?; Ok(Progress { stream, use_color }) } @@ -352,17 +356,17 @@ fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { #[pyfunction] #[pyo3(signature = (*, request))] fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let action: String = native_field(&request, "action")?.str()?.to_string(); + let action: String = native_field(request, "action")?.str()?.to_string(); let generate_write: bool = - action == GENERATE_ACTION && native_field(&request, "write")?.is_truthy()?; - if native_field(&request, "overwrite")?.is_truthy()? && !generate_write { + action == GENERATE_ACTION && native_field(request, "write")?.is_truthy()?; + if native_field(request, "overwrite")?.is_truthy()? && !generate_write { return Err(user_error( py, "contract --overwrite requires generate --write", "C471", )); } - let requested_dir: Bound<'_, PyAny> = native_field(&request, "project_dir")?; + let requested_dir: Bound<'_, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'_, PyAny> = if requested_dir.is_truthy()? { requested_dir } else { @@ -375,12 +379,12 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< call_with( ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, &[ - ("select", native_field(&request, "select")?), - ("exclude", native_field(&request, "exclude")?), + ("select", native_field(request, "select")?), + ("exclude", native_field(request, "exclude")?), ("discovered_inputs", discovered.clone()), ], )?; - let from_target: Bound<'_, PyAny> = native_field(&request, "from_target")?; + let from_target: Bound<'_, PyAny> = native_field(request, "from_target")?; if !discovered .getattr("project_config")? .getattr("targets")? @@ -395,7 +399,7 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< let context: Bound<'_, PyAny> = resolve_adapter_connection_context( &discovered, &project_dir, - (py.None().into_bound(py), native_field(&request, "cli_vars")?), + (py.None().into_bound(py), native_field(request, "cli_vars")?), true, )?; let selection: Selection<'_> = select_resources(request, &discovered, &context)?; @@ -414,7 +418,7 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< )?; } let report: ContractReport = report(&result)?; - let rendered: String = if native_field(&request, "json_output")?.is_truthy()? { + let rendered: String = if native_field(request, "json_output")?.is_truthy()? { render_contract_json(&report) .map_err(|error| PyValueError::new_err(format!("{error:?}")))? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index e298080c11..5bc9f1632c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -8,9 +8,9 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const UTF8: &str = "utf-8"; @@ -30,14 +30,14 @@ pub(crate) fn write_contracts<'py>( result: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); - let project: Bound<'py, PyAny> = native_field(&graph, "project")?; + let project: Bound<'py, PyAny> = native_field(graph, "project")?; let models: Bound<'py, PyDict> = by_name(&native_field(&project, "models")?)?; let sources: Bound<'py, PyDict> = by_name(&native_field(&project, "sources")?)?; let model_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let updates: Bound<'py, PyDict> = PyDict::new(py); let updated_evidence: Bound<'py, PyList> = PyList::empty(py); - for evidence in native_field(&result, "evidence")?.try_iter()? { + for evidence in native_field(result, "evidence")?.try_iter()? { let evidence: Bound<'py, PyAny> = evidence?; let physical_columns: Bound<'py, PyAny> = native_field(&evidence, "physical_columns")?; let findings: Bound<'py, PyAny> = native_field(&evidence, "findings")?; @@ -58,7 +58,9 @@ pub(crate) fn write_contracts<'py>( ], )?; ( - write.project_dir.div(native_field(&model, "relative_path")?)?, + write + .project_dir + .div(native_field(&model, "relative_path")?)?, edited, ) } else { @@ -148,7 +150,7 @@ fn applied<'py>( &[ ("discovered_inputs", discovered), ("adapter", write.adapter.clone()), - ("selected_target", native_field(&result, "from_target")?), + ("selected_target", native_field(result, "from_target")?), ("resolved_connection", write.resolved_connection.clone()), ("cli_vars", write.cli_vars.clone()), ("analysis_selection", selection), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 0b885bfd28..6911b73225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -16,16 +16,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { Ok(CostCoverage { - status: native_field(&summary, "status")?.getattr("value")?.extract()?, - expected_statement_count: native_field(&summary, "expected_statement_count")?.extract()?, - observed_statement_count: native_field(&summary, "observed_statement_count")?.extract()?, - query_count: native_field(&summary, "query_count")?.extract()?, + status: native_field(summary, "status")? + .getattr("value")? + .extract()?, + expected_statement_count: native_field(summary, "expected_statement_count")?.extract()?, + observed_statement_count: native_field(summary, "observed_statement_count")?.extract()?, + query_count: native_field(summary, "query_count")?.extract()?, limitations: summary .getattr("limitations")? .try_iter()? @@ -72,7 +74,7 @@ fn target_database<'py>( ) -> PyResult> { let py: Python<'py> = target_config.py(); if !target_config.is_none() { - let database: Bound<'py, PyAny> = native_field(&target_config, "database")?; + let database: Bound<'py, PyAny> = native_field(target_config, "database")?; if !database.is_none() { return Ok(database); } @@ -85,11 +87,11 @@ fn target_database<'py>( } fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { - let selector: Bound<'_, PyAny> = native_field(&request, "selector")?; + let selector: Bound<'_, PyAny> = native_field(request, "selector")?; if selector.extract::()? == HISTORY_SELECTOR { return Ok(()); } - let requested: Bound<'_, PyAny> = native_field(&request, "project_dir")?; + let requested: Bound<'_, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'_, PyAny> = if requested.is_none() { imported(py, "pathlib", "Path")?.call_method0("cwd")? } else { @@ -100,7 +102,9 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let current: Bound<'_, PyAny> = native_field(&record, "cost")?; - let status: String = native_field(¤t, "status")?.getattr("value")?.extract()?; + let status: String = native_field(¤t, "status")? + .getattr("value")? + .extract()?; if !REFRESHABLE_STATUSES.contains(&status.as_str()) { return Ok(()); } @@ -119,7 +123,10 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { let adapter_name: Bound<'_, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ("project_config", native_field(&discovered, "project_config")?), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), ("local_config", native_field(&discovered, "local_config")?), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index c587de26bf..207cf71e76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -13,11 +13,11 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::DbtCompileRefere use crate::bindings::_helpers::dbt_interop::runtime_config::{ interop_adapter, manifest_path, parse_dbt_execution_args, plan_options, vars_mapping, }; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; -use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_OPTIONS: (&[&str], &[&str]) = ( &[ @@ -66,21 +66,21 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( request: &Bound<'py, PyAny>, ) -> PyResult> { let py = request.py(); - let mut executable = native_field(&request, "dbt_executable")?; + let mut executable = native_field(request, "dbt_executable")?; if !executable.is_truthy()? { executable = registry::native(py, NativeAttr::ResolveDbtExecutable)?.call0()?; } - let mut stream = native_field(&request, "progress_stream")?; + let mut stream = native_field(request, "progress_stream")?; if !stream.is_truthy()? { stream = py.import("sys")?.getattr("stdout")?; } - let command = native_field(&request, "command")?; - let parsed = parse_dbt_execution_args(&command, &native_field(&request, "args")?)?; + let command = native_field(request, "command")?; + let parsed = parse_dbt_execution_args(&command, &native_field(request, "args")?)?; let routed = call_with( ®istry::native(py, NativeAttr::RouteDbtInteropArgs)?, &[("command", command), ("parsed", parsed)], )?; - let project_dir = native_field(&request, "project_dir")?; + let project_dir = native_field(request, "project_dir")?; let discovered = discover_project_inputs( py, Some(keywords(py, &[("project_dir", project_dir.clone())])?), @@ -97,14 +97,14 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( &dbt_args, )? .into_any(); - let mut runner = native_field(&request, "dbt_runner")?; + let mut runner = native_field(request, "dbt_runner")?; if !runner.is_truthy()? { runner = call_with( ®istry::native(py, NativeAttr::DbtRunner)?, &[("dbt_executable", executable.clone())], )?; } - let dbt_stream = native_field(&request, "dbt_stdout_stream")?; + let dbt_stream = native_field(request, "dbt_stdout_stream")?; call_with( ®istry::class(py, RecordId::DbtInteropInvocation)?, &[ @@ -118,7 +118,10 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( stream }, ), - ("effective_sqlbuild_args", native_field(&routed, "sqlbuild_args")?), + ( + "effective_sqlbuild_args", + native_field(&routed, "sqlbuild_args")?, + ), ("routed", routed), ("discovered_inputs", discovered), ("dbt_options", options), @@ -181,7 +184,10 @@ fn compile_dbt_interop_project<'py>( let name = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ("project_config", native_field(&discovered, "project_config")?), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), ("local_config", native_field(&discovered, "local_config")?), ], )?; @@ -244,7 +250,10 @@ fn resolve_dbt_interop_plan<'py>( ®istry::class(py, RecordId::DbtInteropCommandArgs)?, &[ ("select", tuple.call1((native_field(&routed, "select")?,))?), - ("exclude", tuple.call1((native_field(&routed, "exclude")?,))?), + ( + "exclude", + tuple.call1((native_field(&routed, "exclude")?,))?, + ), ( "dbt_command_args", tuple.call1((native_field(&routed, "dbt_args")?,))?, @@ -253,7 +262,10 @@ fn resolve_dbt_interop_plan<'py>( "sqlbuild_command_args", tuple.call1((native_field(&invocation, "effective_sqlbuild_args")?,))?, ), - ("dbt_executable", native_field(&invocation, "dbt_executable")?), + ( + "dbt_executable", + native_field(&invocation, "dbt_executable")?, + ), ("sqlbuild_executable", args.value("sqlbuild_executable")?), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 94a67268c0..ba8affd828 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -10,10 +10,10 @@ use crate::bindings::_helpers::discovery::python_module_files::discover_python_m use crate::bindings::_helpers::event_exporting::scope::current_scope; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; -use crate::bindings::_helpers::records::fields::field as native_field; /// One project directory under discovery. pub(crate) struct ProjectDir<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index aebb1f2581..4653475d06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const PROVIDER_ERROR: ErrorId = ErrorId::ProviderDiscovery; const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; @@ -41,7 +41,7 @@ fn provider_name<'py>( relative_path: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = provider_class.py(); - let explicit: Bound<'py, PyAny> = native_field(&provider_class, "provider_name")?; + let explicit: Bound<'py, PyAny> = native_field(provider_class, "provider_name")?; if !explicit.is_none() { reject_invalid_identity(&explicit, relative_path)?; } @@ -273,10 +273,10 @@ fn exporter_provider_usages<'py>( (project, providers): (&ProjectDir<'py>, &Bound<'py, PyDict>), ) -> PyResult> { let py: Python<'py> = project.py(); - let function: Bound<'py, PyAny> = native_field(&declaration, "function")?; - let name: String = native_field(&declaration, "name")?.extract()?; + let function: Bound<'py, PyAny> = native_field(declaration, "function")?; + let name: String = native_field(declaration, "name")?.extract()?; let relative_path: String = project - .relative(&native_field(&declaration, "file_path")?)? + .relative(&native_field(declaration, "file_path")?)? .str()? .extract()?; let empty: Bound<'py, PyAny> = imported(py, "inspect", "Parameter")?.getattr("empty")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index 65a22d98f7..67f7efd85d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -97,7 +97,8 @@ fn resolve_inputs<'py>( selected, discovered, }) => { - resolved.has_fixture_candidates = native_field(&discovered, "test_files")?.is_truthy()?; + resolved.has_fixture_candidates = + native_field(&discovered, "test_files")?.is_truthy()?; let adapter_dialect: Bound<'py, PyAny> = native_field(&adapter, "sql_analysis_dialect_name")?; resolved.dialect = if adapter_dialect.is_truthy()? { @@ -132,14 +133,14 @@ fn fixture_inputs<'py>(resolved: &Resolved<'py>) -> PyResult = native_field(&discovered, "local_config")?; + let local: Bound<'py, PyAny> = native_field(discovered, "local_config")?; let settings_owner: Bound<'py, PyAny> = if local .getattr("setting_overrides")? .contains(SQL_ANALYSIS_SETTING_KEY)? { local } else { - native_field(&discovered, "project_config")? + native_field(discovered, "project_config")? }; let enabled: bool = settings_owner .getattr("settings")? @@ -195,7 +196,7 @@ fn complete_status(status: &Bound<'_, PyAny>, report: &LintReport, elapsed: f64) status.call_method1("error", (message,))?; } else { call_with( - &native_field(&status, "complete")?, + &native_field(status, "complete")?, &[("message", PyString::new(status.py(), &message).into_any())], )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 542e0cf62d..631ccf7ebc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -18,9 +18,9 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const SELECTION_CODE: &str = "S007"; @@ -77,8 +77,11 @@ fn discovered_adapter<'py>( let adapter_name: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, &[ - ("project_config", native_field(&discovered, "project_config")?), - ("local_config", native_field(&discovered, "local_config")?), + ( + "project_config", + native_field(discovered, "project_config")?, + ), + ("local_config", native_field(discovered, "local_config")?), ], )?; resolve_cli_adapter(&adapter_name, project_dir) @@ -91,7 +94,7 @@ fn selectors<'py>(py: Python<'py>, values: &[String]) -> PyResult) -> PyResult> { let mut pairs: Vec<(String, PathBuf)> = Vec::new(); - for model_file in native_field(&scope.discovered, "model_files")?.try_iter()? { + for model_file in native_field(scope.discovered, "model_files")?.try_iter()? { let model_file: Bound<'_, PyAny> = model_file?; let relative_path: Bound<'_, PyAny> = native_field(&model_file, "relative_path")?; let configured: Bound<'_, PyAny> = model_file @@ -178,7 +181,10 @@ fn model_paths( } } let mut paths: BTreeSet = BTreeSet::new(); - for model in native_field(&graph, "project")?.getattr("models")?.try_iter()? { + for model in native_field(&graph, "project")? + .getattr("models")? + .try_iter()? + { let model: Bound<'_, PyAny> = model?; if names.contains(&native_field(&model, "name")?.extract::()?) { let relative: PathBuf = native_field(&model, "relative_path")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 44b0a6b18d..3a493440b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -32,10 +32,10 @@ use crate::bindings::_helpers::result_output::documents::write_execution_json_ou use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; -use crate::bindings::_helpers::records::fields::field as native_field; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; @@ -82,7 +82,7 @@ fn source_names_for_keys( let py: Python<'_> = graph.py(); let source_type: Bound<'_, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; - let upstream_deps: Bound<'_, PyAny> = native_field(&graph, "upstream_deps")?; + let upstream_deps: Bound<'_, PyAny> = native_field(graph, "upstream_deps")?; for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; let upstream: Bound<'_, PyAny> = @@ -114,7 +114,7 @@ fn selected_source_names( let selected_keys: Bound<'_, PyAny> = if select.is_truthy()? { graph_keys(graph, select)? } else { - native_field(&graph, "all_keys")?.call_method0("values")? + native_field(graph, "all_keys")?.call_method0("values")? }; let selected: BTreeSet = source_names_for_keys(graph, &selected_keys)?; let excluded: BTreeSet = if exclude.is_truthy()? { @@ -174,9 +174,9 @@ fn previous_records<'py>( )?; let read: StateRead<'py> = StateRead { connection: connection.clone(), - execute: native_field(&adapter, "execute")?, - render_qualified_name: native_field(&adapter, "render_qualified_name")?, - render_read_latest_sql: native_field(&adapter, "render_read_latest_source_freshness_sql")?, + execute: native_field(adapter, "execute")?, + render_qualified_name: native_field(adapter, "render_qualified_name")?, + render_read_latest_sql: native_field(adapter, "render_read_latest_source_freshness_sql")?, }; let records: Bound<'py, PyDict> = PyDict::new(py); for schema in &schemas { @@ -211,16 +211,19 @@ fn observe( &[ ("discovered_inputs", discovered), ("project_dir", project_dir), - ("selected_target", &native_field(&request, "selected_target")?), - ("cli_vars", &native_field(&request, "cli_vars")?), + ( + "selected_target", + &native_field(request, "selected_target")?, + ), + ("cli_vars", &native_field(request, "cli_vars")?), ], )?), )?; let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; let observed = (|| { - let project: Bound<'_, PyAny> = native_field(&graph, "project")?; + let project: Bound<'_, PyAny> = native_field(graph, "project")?; let by_identity: Option> = - if native_field(&request, "compare_state")?.extract::()? { + if native_field(request, "compare_state")?.extract::()? { Some(previous_records(adapter, &connection, &project)?.into_any()) } else { None @@ -267,22 +270,22 @@ fn write_json_output( #[pyfunction] #[pyo3(signature = (*, request))] fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let fail_on_stale: bool = native_field(&request, "fail_on_stale")?.extract()?; - let fail_on_error: bool = native_field(&request, "fail_on_error")?.extract()?; - if fail_on_stale && !native_field(&request, "compare_state")?.extract::()? { + let fail_on_stale: bool = native_field(request, "fail_on_stale")?.extract()?; + let fail_on_error: bool = native_field(request, "fail_on_error")?.extract()?; + if fail_on_stale && !native_field(request, "compare_state")?.extract::()? { return Err(user_error( py, (STALE_REQUIRES_STATE_MESSAGE, STALE_REQUIRES_STATE_CODE), )); } - let project_dir: Bound<'_, PyAny> = match native_field(&request, "project_dir")? { + let project_dir: Bound<'_, PyAny> = match native_field(request, "project_dir")? { directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; let discovered: Bound<'_, PyAny> = discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; - let select: Bound<'_, PyAny> = native_field(&request, "select")?; - let exclude: Bound<'_, PyAny> = native_field(&request, "exclude")?; + let select: Bound<'_, PyAny> = native_field(request, "select")?; + let exclude: Bound<'_, PyAny> = native_field(request, "exclude")?; reject_unit_test_selectors(&select, &exclude, &discovered)?; let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, @@ -296,9 +299,15 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult &[ ("discovered_inputs", &discovered), ("adapter", &adapter), - ("selected_target", &native_field(&request, "selected_target")?), - ("no_sql_validation", &native_field(&request, "no_sql_validation")?), - ("cli_vars", &native_field(&request, "cli_vars")?), + ( + "selected_target", + &native_field(request, "selected_target")?, + ), + ( + "no_sql_validation", + &native_field(request, "no_sql_validation")?, + ), + ("cli_vars", &native_field(request, "cli_vars")?), ], )?), )?; @@ -315,15 +324,15 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult }; let payload: String = freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}")))?; - let json_output_path: Bound<'_, PyAny> = native_field(&request, "json_output_path")?; - if native_field(&request, "json_output")?.extract::()? { + let json_output_path: Bound<'_, PyAny> = native_field(request, "json_output_path")?; + if native_field(request, "json_output")?.extract::()? { write_json_output(py, &payload, (true, &json_output_path))?; } else { if !json_output_path.is_none() { write_json_output(py, &payload, (false, &json_output_path))?; } let use_color: bool = - !native_field(&request, "no_color")?.extract::()? && stdout_supports_color(py)?; + !native_field(request, "no_color")?.extract::()? && stdout_supports_color(py)?; write(py, &render_freshness_text(&results, CliStyle { use_color }))?; } Ok(freshness_exit_code( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 9319287001..7f3fb613a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::freshness::support::text; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; -use crate::bindings::_helpers::records::fields::field as native_field; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; @@ -51,9 +51,9 @@ fn previous_records<'py>( let py: Python<'py> = adapter.py(); let read: StateRead<'py> = StateRead { connection: connection.clone(), - execute: native_field(&adapter, "execute")?, + execute: native_field(adapter, "execute")?, render_qualified_name: arguments.value("render_qualified_name")?, - render_read_latest_sql: native_field(&adapter, "render_read_latest_source_freshness_sql")?, + render_read_latest_sql: native_field(adapter, "render_read_latest_source_freshness_sql")?, }; let database: Bound<'py, PyAny> = arguments.value("state_database")?; let exists_by_schema: Bound<'py, PyAny> = arguments.value("state_table_exists_by_schema")?; @@ -65,7 +65,10 @@ fn previous_records<'py>( .is_truthy()?; let previous: Bound<'py, PyAny> = read_latest(&read, (&database, &schema), exists, source_names)?; - for (identity, candidate) in native_field(&previous, "records")?.downcast::()?.iter() { + for (identity, candidate) in native_field(&previous, "records")? + .downcast::()? + .iter() + { let replace: bool = match merged.get_item(&identity)? { None => true, Some(current) => candidate @@ -119,7 +122,7 @@ fn adapter_batch<'py>( }; let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); for source in sources { - let name: Bound<'py, PyAny> = native_field(&source, "name")?; + let name: Bound<'py, PyAny> = native_field(source, "name")?; unknown_sources.set_item(&name, unknown(&name, ("ERROR", message.clone()))?)?; } call_with( @@ -168,7 +171,7 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( if arguments.value("filter_previous_to_sources")?.is_truthy()? { let mut names: Vec = sources .iter() - .map(|source| native_field(&source, "name")?.extract()) + .map(|source| native_field(source, "name")?.extract()) .collect::>()?; names.sort(); Some(names) @@ -184,10 +187,10 @@ pub(crate) fn build_direct_source_freshness_planning_result<'py>( let observed_sources: Bound<'py, PyDict> = PyDict::new(py); let mut adapter_sources: Vec> = Vec::new(); for source in &sources { - if native_field(&source, "managed")?.is_truthy()? { + if native_field(source, "managed")?.is_truthy()? { continue; } - let name: Bound<'py, PyAny> = native_field(&source, "name")?; + let name: Bound<'py, PyAny> = native_field(source, "name")?; let Some(observed) = observation_source(&observer.adapter, source)? else { unknown_sources.set_item( &name, @@ -319,13 +322,13 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( scope: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = scope.py(); - let downstream_deps: Bound<'py, PyAny> = native_field(&scope, "downstream_deps")?; + let downstream_deps: Bound<'py, PyAny> = native_field(scope, "downstream_deps")?; let changed: Bound<'py, PyDict> = PyDict::new(py); let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); let errors: Bound<'py, PyDict> = PyDict::new(py); let stale: Bound<'py, PySet> = PySet::empty(py)?; let blocked: Bound<'py, PySet> = PySet::empty(py)?; - for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { + for identity in native_field(source_freshness, "changed_identities")?.try_iter()? { let identity: Bound<'py, PyAny> = identity?; let models: Bound<'py, PyFrozenSet> = downstream_models(&native_field(&identity, "source_name")?, &downstream_deps)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index ab2baf3104..5606688024 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -12,9 +12,9 @@ use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; /// Observed records grouped by `(database, schema)` in first-seen order. type RecordsByLocation<'py> = Vec<( @@ -28,7 +28,7 @@ fn successful_model_names<'py>(result: &Bound<'py, PyAny>) -> PyResult = host::entry(py, ExecutorHost::ExecutionStatus)?.getattr("SUCCESS")?; let mut names: Vec> = Vec::new(); - for model_result in native_field(&result, "model_results")?.try_iter()? { + for model_result in native_field(result, "model_results")?.try_iter()? { let model_result: Bound<'py, PyAny> = model_result?; let status: Bound<'py, PyAny> = native_field(&model_result, "status")?; let value: Bound<'py, PyAny> = status.getattr_opt("value")?.unwrap_or(status); @@ -106,12 +106,12 @@ fn eligible_records<'py>( run_id: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = plan.py(); - let freshness: Bound<'py, PyAny> = native_field(&plan, "source_freshness")?; + let freshness: Bound<'py, PyAny> = native_field(plan, "source_freshness")?; if freshness.is_none() || native_field(&freshness, "propagation")?.is_none() { return Ok(Vec::new()); } let entries_by_name: Bound<'py, PyDict> = PyDict::new(py); - for entry in native_field(&plan, "model_entries")?.try_iter()? { + for entry in native_field(plan, "model_entries")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; entries_by_name.set_item(native_field(&entry, "name")?, entry)?; } @@ -159,23 +159,23 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { &[ ( "render_qualified_name", - native_field(&adapter, "render_qualified_name")?, + native_field(adapter, "render_qualified_name")?, ), ( "render_framework_type", - native_field(&adapter, "render_framework_type")?, + native_field(adapter, "render_framework_type")?, ), ( "render_insert_records_sql", - native_field(&adapter, "render_insert_source_freshness_records_sql")?, + native_field(adapter, "render_insert_source_freshness_records_sql")?, ), ( "render_create_table_sql", - native_field(&adapter, "render_create_source_freshness_table_sql")?, + native_field(adapter, "render_create_source_freshness_table_sql")?, ), ( "render_create_index_sqls", - native_field(&adapter, "render_create_source_freshness_index_sqls")?, + native_field(adapter, "render_create_source_freshness_index_sqls")?, ), ], ) @@ -191,7 +191,7 @@ fn write_records<'py>( for ((database, schema), records) in grouped { let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); for record in records { - let source_name: Bound<'py, PyAny> = native_field(&record, "source_name")?; + let source_name: Bound<'py, PyAny> = native_field(record, "source_name")?; let mut position: Option = None; for (index, (known, _)) in by_source.iter().enumerate() { if known.eq(&source_name)? { @@ -230,7 +230,7 @@ fn write_records<'py>( py, &[ ("connection", connection.clone()), - ("execute", native_field(&adapter, "execute")?), + ("execute", native_field(adapter, "execute")?), ("database", database.clone()), ("schema", schema.clone()), ("records", PyTuple::new(py, &source_records)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 44ec984b2a..319aeb6f8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -8,9 +8,9 @@ use crate::bindings::_helpers::analysis::explicit_references::{ GateRow, HookRow, native_execution_gate_edges, }; use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. @@ -92,7 +92,7 @@ fn model_python_hook_names<'py>( model: &Bound<'py, PyAny>, ) -> PyResult> { let entry_type: Bound<'_, PyAny> = registry::class(py, RecordId::PythonHookEntry)?; - let values: Bound<'_, PyAny> = native_field(&model, "config")?.getattr("values")?; + let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; let mut names: Vec = Vec::new(); for hook_key in MODEL_HOOK_KEYS { let entries: Bound<'_, PyAny> = values.call_method1("get", (hook_key,))?; @@ -123,12 +123,12 @@ pub(crate) fn sql_ref_key<'py>( reference: &Bound<'py, PyAny>, ) -> PyResult> { let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)? - .call1((native_field(&reference, "kind")?.getattr("value")?,))?; + .call1((native_field(reference, "kind")?.getattr("value")?,))?; keyword_call( ®istry::class(py, RecordId::CompiledObjectKey)?, &[ ("resource_type", resource_type), - ("name", native_field(&reference, "name")?), + ("name", native_field(reference, "name")?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index 91f1c34ff1..0eb247bf0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -13,9 +13,9 @@ use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_s use crate::bindings::_helpers::graph::views::compiled_graph_keys; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const NAME_KEY: &str = "name"; @@ -43,7 +43,7 @@ fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> if value.is_none() { return Ok(value.clone()); } - native_field(&value, "value") + native_field(value, "value") } /// The decorator configuration the identity of one node of `kind` hashes, read through `get`. @@ -203,10 +203,10 @@ fn upstream_names( ) -> PyResult>> { let by_key: Bound<'_, PyDict> = PyDict::new(py); for node in nodes { - by_key.set_item(native_field(&node, "function")?, native_field(&node, "name")?)?; + by_key.set_item(native_field(node, "function")?, native_field(node, "name")?)?; } for node in nodes { - let name: Bound<'_, PyAny> = native_field(&node, "name")?; + let name: Bound<'_, PyAny> = native_field(node, "name")?; by_key.set_item( PyTuple::new(py, [PyString::new(py, NAME_KEY).into_any(), name.clone()])?, name, @@ -217,7 +217,7 @@ fn upstream_names( .iter() .map(|node| { let mut names: Vec = Vec::new(); - for dependency in native_field(&node, "depends_on")?.try_iter()? { + for dependency in native_field(node, "depends_on")?.try_iter()? { let dependency: Bound<'_, PyAny> = dependency?; if dependency.is_instance(sql_ref)? { continue; @@ -234,8 +234,8 @@ fn upstream_names( fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { Ok(InventoryNode { - name: native_field(&node, "name")?.extract()?, - tags: native_field(&node, "tags")?.extract()?, + name: native_field(node, "name")?.extract()?, + tags: native_field(node, "tags")?.extract()?, relative_path: node .getattr("relative_path")? .call_method0("as_posix")? @@ -250,7 +250,7 @@ pub(crate) fn python_node_graph<'py>( ) -> PyResult> { let py: Python<'py> = discovered_inputs.py(); let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(&discovered_inputs, "providers")?.try_iter()? { + for provider in native_field(discovered_inputs, "providers")?.try_iter()? { let provider: Bound<'py, PyAny> = provider?; providers.set_item(native_field(&provider, "name")?, provider)?; } @@ -325,8 +325,8 @@ fn graph_record<'py>( let by_name: Bound<'py, PyDict> = PyDict::new(py); let by_selector: Bound<'py, PyDict> = PyDict::new(py); for node in nodes { - let name: Bound<'py, PyAny> = native_field(&node, "name")?; - let kind: String = native_field(&node, "kind")?.getattr("value")?.extract()?; + let name: Bound<'py, PyAny> = native_field(node, "name")?; + let kind: String = native_field(node, "kind")?.getattr("value")?.extract()?; by_name.set_item(&name, node)?; by_selector.set_item(format!("{kind}:{}", name.str()?), node)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 84d62c541c..ac2aef966f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -18,9 +18,9 @@ use sqlbuild_analysis::python_selection::models::{ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; type NameRows = Vec<(String, Vec)>; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. @@ -64,16 +64,18 @@ fn name_rows(mapping: &Bound<'_, PyAny>) -> PyResult { fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { let mut sql_deps: Vec = Vec::new(); - for reference in native_field(&node, "sql_deps")?.try_iter()? { + for reference in native_field(node, "sql_deps")?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; sql_deps.push(( - native_field(&reference, "kind")?.getattr("value")?.extract()?, + native_field(&reference, "kind")? + .getattr("value")? + .extract()?, native_field(&reference, "name")?.extract()?, )); } Ok(PythonSelectionNode { - name: native_field(&node, "name")?.extract()?, - kind: native_field(&node, "kind")?.getattr("value")?.extract()?, + name: native_field(node, "name")?.extract()?, + kind: native_field(node, "kind")?.getattr("value")?.extract()?, sql_deps, }) } @@ -86,9 +88,9 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { .try_iter()? .map(|node| selection_node(&node?)) .collect::>()?, - upstream: name_rows(&native_field(&graph, "upstream_deps")?)?, - downstream: name_rows(&native_field(&graph, "downstream_deps")?)?, - tags: name_rows(&native_field(&graph, "tag_index")?)?, + upstream: name_rows(&native_field(graph, "upstream_deps")?)?, + downstream: name_rows(&native_field(graph, "downstream_deps")?)?, + tags: name_rows(&native_field(graph, "tag_index")?)?, paths: graph .getattr("path_index")? .call_method0("items")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index 3a377d56f9..f1a5f48c84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,10 +1,10 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use crate::bindings::_helpers::records::fields::field as native_field; type Object<'py> = Bound<'py, PyAny>; @@ -16,7 +16,7 @@ pub(crate) fn compiled_model<'py>( macro_deps: &Object<'py>, ) -> PyResult> { let py: Python<'py> = model_input.py(); - let model_file: Object<'py> = native_field(&model_input, "model_file")?; + let model_file: Object<'py> = native_field(model_input, "model_file")?; let model_name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let none: Object<'py> = py.None().into_bound(py); let mut inferred: Object<'py> = none.clone(); @@ -26,8 +26,8 @@ pub(crate) fn compiled_model<'py>( let mut diagnostics: Object<'py> = PyTuple::empty(py).into_any(); let mut validated: Object<'py> = false_object(py); let mut proof: Object<'py> = registry::native(py, NativeAttr::DynamicContractProof)? - .call1((native_field(&analysis, "standalone_proof")?,))?; - let session: Object<'py> = native_field(&analysis, "session")?; + .call1((native_field(analysis, "standalone_proof")?,))?; + let session: Object<'py> = native_field(analysis, "session")?; if !session.is_none() { let (session, position): (Object<'py>, Object<'py>) = session.extract()?; let arguments: Bound<'py, PyDict> = PyDict::new(py); @@ -43,7 +43,7 @@ pub(crate) fn compiled_model<'py>( has_star = native_field(&result, "has_star")?; star_resolved = native_field(&result, "star_resolved")?; } else { - inferred = fallback_columns(&native_field(&analysis, "fallback")?)?; + inferred = fallback_columns(&native_field(analysis, "fallback")?)?; } let session_proof: Object<'py> = native_field(&outcome, "dynamic_column_contract")?; if session_proof.is_truthy()? { @@ -79,7 +79,7 @@ pub(crate) fn compiled_model<'py>( ], )?); } - let config: Object<'py> = native_field(&model_input, "config")?; + let config: Object<'py> = native_field(model_input, "config")?; let key: Object<'py> = call( &key_class, &[ @@ -94,17 +94,20 @@ pub(crate) fn compiled_model<'py>( ("deps", PyTuple::new(py, dependency_keys)?.into_any()), ("name", model_name.clone()), ("relative_path", native_field(&model_file, "relative_path")?), - ("query_sql", native_field(&model_input, "query_sql")?), + ("query_sql", native_field(model_input, "query_sql")?), ("config", config.clone()), ("destination", destination(&config, &model_name)?), - ("references", native_field(&model_input, "references")?), - ("schema_entry", native_field(&model_input, "schema_entry")?), + ("references", native_field(model_input, "references")?), + ("schema_entry", native_field(model_input, "schema_entry")?), ("inferred_columns", inferred), ("fast_lineage_columns", lineage), ("fast_lineage_has_star", has_star), ("fast_lineage_star_resolved", star_resolved), ("authored_sql", native_field(&model_file, "contents")?), - ("authored_query_sql", native_field(&model_file, "query_sql")?), + ( + "authored_query_sql", + native_field(&model_file, "query_sql")?, + ), ( "output_column_locations", native_field(&model_file, "output_column_locations")?, @@ -116,19 +119,19 @@ pub(crate) fn compiled_model<'py>( ("macro_deps", macro_deps.clone()), ( "enum_declarations", - native_field(&model_input, "enum_declarations")?, + native_field(model_input, "enum_declarations")?, ), ( "constant_declarations", - native_field(&model_input, "constant_declarations")?, + native_field(model_input, "constant_declarations")?, ), - ("enum_columns", native_field(&model_input, "enum_columns")?), + ("enum_columns", native_field(model_input, "enum_columns")?), ("binding_diagnostics", diagnostics), ("binding_validated", validated), ("dynamic_column_contract", proof), ( "rejected_sql_analysis_opt_out", - native_field(&model_input, "rejected_sql_analysis_opt_out")?, + native_field(model_input, "rejected_sql_analysis_opt_out")?, ), ], ) @@ -145,7 +148,7 @@ fn true_object(py: Python<'_>) -> Object<'_> { /// A model's configured physical and logical destination. fn destination<'py>(config: &Object<'py>, model_name: &Object<'py>) -> PyResult> { let py: Python<'py> = config.py(); - let values: Object<'py> = native_field(&config, "values")?; + let values: Object<'py> = native_field(config, "values")?; let text = |key: &str| -> PyResult>> { let value: Object<'py> = values.call_method1("get", (key,))?; Ok(value.is_instance_of::().then_some(value)) @@ -161,8 +164,11 @@ fn destination<'py>(config: &Object<'py>, model_name: &Object<'py>) -> PyResult< ("schema", text("schema")?.unwrap_or_else(|| none.clone())), ("name", text("alias")?.unwrap_or_else(|| model_name.clone())), ("qualified_name", none.clone()), - ("logical_schema", native_field(&config, "logical_schema")?), - ("logical_database", native_field(&config, "logical_database")?), + ("logical_schema", native_field(config, "logical_schema")?), + ( + "logical_database", + native_field(config, "logical_database")?, + ), ], ) } @@ -200,7 +206,7 @@ fn located<'py>( if !diagnostics.is_truthy()? { return Ok(PyTuple::empty(py).into_any()); } - let model_file: Object<'py> = native_field(&model_input, "model_file")?; + let model_file: Object<'py> = native_field(model_input, "model_file")?; let path: Object<'py> = native_field(&model_file, "relative_path")?; let mut rows: Vec> = Vec::new(); for item in diagnostics.try_iter()? { @@ -222,7 +228,7 @@ fn located<'py>( ], )? .into_any(); - let expansion: Object<'py> = native_field(&model_input, "sql_expansion")?; + let expansion: Object<'py> = native_field(model_input, "sql_expansion")?; let expansion_row: Object<'py> = if expansion.is_none() { expansion } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index fe6cf37fa3..a16c115d78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -19,9 +19,9 @@ use crate::bindings::_helpers::janitor::janitor_planning::{ }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = @@ -46,7 +46,7 @@ fn stdout(py: Python<'_>) -> PyResult> { } fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { - native_field(&invocation, "use_color")?.extract() + native_field(invocation, "use_color")?.extract() } /// `OperationLifecycle(operation_kind=..., operation_name=...)`. @@ -63,10 +63,10 @@ pub(crate) fn compile_janitor_project<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); let context: Bound<'py, PyAny> = resolve_adapter_connection_context( - &native_field(&invocation, "discovered_inputs")?, - &native_field(&invocation, "effective_project_dir")?, + &native_field(invocation, "discovered_inputs")?, + &native_field(invocation, "effective_project_dir")?, ( - native_field(&invocation, "selected_target")?, + native_field(invocation, "selected_target")?, py.None().into_bound(py), ), true, @@ -77,9 +77,9 @@ pub(crate) fn compile_janitor_project<'py>( PlanningProgressReporter::new(py, &stdout(py)?, use_color(invocation)?)?, )? .into_any(); - let as_target: Bound<'py, PyAny> = native_field(&invocation, "as_target")?; + let as_target: Bound<'py, PyAny> = native_field(invocation, "as_target")?; let target: Bound<'py, PyAny> = if as_target.is_none() { - native_field(&invocation, "selected_target")? + native_field(invocation, "selected_target")? } else { as_target }; @@ -93,11 +93,14 @@ pub(crate) fn compile_janitor_project<'py>( &[ ( "discovered_inputs", - native_field(&invocation, "discovered_inputs")?, + native_field(invocation, "discovered_inputs")?, ), ("adapter", native_field(&context, "adapter")?), ("selected_target", target), - ("resolved_connection", native_field(&context, "connection_config")?), + ( + "resolved_connection", + native_field(&context, "connection_config")?, + ), ], )?), )?; @@ -184,7 +187,9 @@ pub(crate) fn close_janitor_warehouse( } fn candidate_count(plan: &Bound<'_, PyAny>) -> PyResult { - let mut count: usize = native_field(&plan, "old_name_views")?.getattr("drops")?.len()?; + let mut count: usize = native_field(plan, "old_name_views")? + .getattr("drops")? + .len()?; for field in CANDIDATE_FIELDS { count += plan.getattr(field)?.len()?; } @@ -232,8 +237,9 @@ pub(crate) fn build_janitor_execution_plan<'py>( }, ), &JanitorSettings { - retention_days: native_field(&settings, "retention_days")?.extract()?, - delete_tracked_only: native_field(&janitor, "delete_tracked_only")?.is_truthy()?, + retention_days: native_field(settings, "retention_days")?.extract()?, + delete_tracked_only: native_field(&janitor, "delete_tracked_only")? + .is_truthy()?, exclude_patterns: native_field(&janitor, "exclude_patterns")?.extract()?, direct_mode: true, state_history_versions: settings diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index dd05716525..5435b9a3fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -5,9 +5,9 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CREATE_TABLE_ACTION: &str = "create_table"; @@ -23,11 +23,11 @@ pub(crate) fn resolve_model_actions<'py>( let model_kind: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let action_class: Bound<'py, PyAny> = registry::class(py, RecordId::ResolvedModelAction)?; - let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; - let model_changes: Bound<'py, PyAny> = native_field(&changes, "models")?; + let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; + let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + let model_changes: Bound<'py, PyAny> = native_field(changes, "models")?; let resolved: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(&scope, "execution_order")?.try_iter()? { + for key in native_field(scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index bf8ccf1507..60bf12cb77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -43,10 +43,10 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; -use crate::bindings::_helpers::records::fields::field as native_field; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; @@ -101,14 +101,15 @@ pub(crate) fn detect_changes<'py>( scope_model_changes((project, scope, snapshot), None, expected)?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = registry::class(py, RecordId::FunctionChangeResult)?; - let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; - let recorded: Bound<'py, PyAny> = native_field(&snapshot, "fingerprints")?.getattr("functions")?; + let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; + let recorded: Bound<'py, PyAny> = + native_field(snapshot, "fingerprints")?.getattr("functions")?; let query_change_tracking: bool = project .getattr("settings")? .getattr("query_change_tracking")? .is_truthy()?; - let dialect: Option = native_field(&snapshot, "column_dialect")?.extract()?; - for function in native_field(&project, "functions")?.try_iter()? { + let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; + for function in native_field(project, "functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = native_field(&function, "name")?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; @@ -129,7 +130,9 @@ pub(crate) fn detect_changes<'py>( &name.extract::()?, ( &fingerprint_sql, - &native_field(&function, "language")?.str()?.extract::()?, + &native_field(&function, "language")? + .str()? + .extract::()?, dialect.as_deref(), ), )? != fingerprint @@ -246,7 +249,7 @@ pub(crate) fn scope_model_changes<'py>( .extract()?, }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &native_field(&project, "functions")?, + &native_field(project, "functions")?, project .getattr("sql_analysis_dialect")? .extract::>()? @@ -255,7 +258,7 @@ pub(crate) fn scope_model_changes<'py>( .into_any(); let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = host::entry(py, UserCodeHost::BuildHookIdentities)? - .call1((native_field(&project, "hook_functions")?,))?; + .call1((native_field(project, "hook_functions")?,))?; for item in identities.call_method0("items")?.try_iter()? { let item: Bound<'py, PyAny> = item?; hook_version_hashes.set_item( @@ -272,10 +275,10 @@ pub(crate) fn scope_model_changes<'py>( }; let model_kind: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; - let selected: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; + let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; + let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; let changes: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(&scope, "execution_order")?.try_iter()? { + for key in native_field(scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { continue; @@ -343,7 +346,7 @@ pub(crate) fn model_change<'py>( .getattr("models")? .call_method1("get", (&model_name,))?; let recorded: Option = recorded_fingerprint(&fingerprint)?; - let dialect: Bound<'py, PyAny> = native_field(&snapshot, "column_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(snapshot, "column_dialect")?; let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { Some(model_query_hash( py, @@ -363,7 +366,7 @@ pub(crate) fn model_change<'py>( (Some(hash), Some(recorded)) => *hash != recorded.definition_hash, _ => false, }; - if !relation_exists && native_field(&snapshot, "renamed_models")?.contains(&model_name)? { + if !relation_exists && native_field(snapshot, "renamed_models")?.contains(&model_name)? { result.query_changed = definition_changed; let backfill: Backfill = if definition_changed { replay_policy(model)? @@ -390,7 +393,7 @@ pub(crate) fn model_change<'py>( .as_ref() .map(|recorded| recorded.metadata_json.as_str()) .filter(|text| *text != EMPTY_FINGERPRINT_METADATA_JSON); - let renames: Bound<'py, PyAny> = native_field(&snapshot, "reference_renames")?; + let renames: Bound<'py, PyAny> = native_field(snapshot, "reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( (model_reads::query_sql(model)?.extract()?, &fingerprint), @@ -441,7 +444,7 @@ pub(crate) fn model_change<'py>( fn replay_policy(model: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = model.py(); let raw: Option = authored_config_str( - &native_field(&model, "config")?.getattr("values")?, + &native_field(model, "config")?.getattr("values")?, REPLAY_CONFIG_KEY, )? .map(|value| value.extract()) @@ -461,9 +464,9 @@ fn recorded_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult( if warehouse.is_none() { return Ok(Vec::new()); } - let schema_entry: Bound<'py, PyAny> = native_field(&model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 4bf2b5d6d0..940c305f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -31,13 +31,13 @@ use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; -use crate::bindings::_helpers::records::fields::field as native_field; const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; /// Selected scopes, and the overrides with cursor overrides typed for the selected models. @@ -58,7 +58,9 @@ fn scopes_and_cursor_overrides<'py>( for key in native_field(&selected, "selected_keys")?.try_iter()? { let key = key?; let name = native_field(&key, "name")?; - if native_field(&key, "resource_type")?.eq(&model_type)? && models_by_name.contains(&name)? { + if native_field(&key, "resource_type")?.eq(&model_type)? + && models_by_name.contains(&name)? + { models.push(models_by_name.get_item(&name)?); } } @@ -87,13 +89,13 @@ fn change_results<'py>( &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, snapshot, ), - native_field(&overrides, "full_refresh")?.is_truthy()?, + native_field(overrides, "full_refresh")?.is_truthy()?, ( native_field(&versions, "model_version_hashes")?, native_field(&versions, "model_metadata_jsons")?, ), )?; - if !native_field(&policies, "selection_diagnostics")?.is_truthy()? { + if !native_field(policies, "selection_diagnostics")?.is_truthy()? { let empty = native( py, RecordId::PlannerChangeResults, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 6ba727a16c..2bfa79d714 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -150,8 +150,8 @@ pub(crate) fn direct_model_version_identities<'py>( } let model_metadata: Bound<'py, PyDict> = PyDict::new(py); let model_local: Bound<'py, PyDict> = PyDict::new(py); - let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; - for key in native_field(&scope, "execution_order")?.try_iter()? { + let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + for key in native_field(scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; let order_key: GraphKey = graph_key(&key)?; graph.execution_order.push(order_key.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index b1d045fa3c..be23229cb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -16,7 +17,6 @@ use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{Py, PyRefMut, Python}; use std::collections::{BTreeMap, HashSet}; -use crate::bindings::_helpers::records::fields::field as native_field; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); @@ -48,12 +48,12 @@ fn version_identities<'py>( ) -> PyResult> { identity_work(project.py())?.identity_builds += 1; direct_model_version_identities( - (&native_field(&project, "functions")?, &scope), - native_field(&project, "sql_analysis_dialect")?.extract()?, + (&native_field(project, "functions")?, &scope), + native_field(project, "sql_analysis_dialect")?.extract()?, ( - &native_field(&project, "seeds")?, + &native_field(project, "seeds")?, &project.py().None().into_bound(project.py()), - &native_field(&project, "hook_functions")?, + &native_field(project, "hook_functions")?, ), ) } @@ -102,7 +102,7 @@ fn reusable_model_names( let stale = class_field(identities, |i: &PlannerIdentityContext| { &i.stale_warning_identities })?; - for name in native_field(&execution_changes, "models")?.try_iter()? { + for name in native_field(execution_changes, "models")?.try_iter()? { let name = name?; let same = |field: &str| -> PyResult { execution @@ -185,7 +185,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( for key in &remaining { *work .stale_detections - .entry(native_field(&key, "name")?.extract()?) + .entry(native_field(key, "name")?.extract()?) .or_default() += 1; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 5335281b81..1b3478de46 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -21,12 +21,12 @@ use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_ use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::records::fields::field as native_field; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -112,13 +112,16 @@ pub(crate) fn base_plan_output<'py>( changes: reconciliation.changes.clone(), model_entries: entries.entries, model_warnings: entries.warnings, - reload_sources: native_field(&overrides, "reload_sources")?, + reload_sources: native_field(overrides, "reload_sources")?, extras, })?, metadata: Vec::new(), }; output.set("source_freshness", source_freshness.clone()); - output.set("cursor_overrides", native_field(&overrides, "cursor_overrides")?); + output.set( + "cursor_overrides", + native_field(overrides, "cursor_overrides")?, + ); Ok(output) } @@ -255,7 +258,7 @@ pub(crate) fn add_plan_warnings<'py>( let warnings = concat(py, &[output.field("warnings")?, freshness])?; output.set("warnings", warnings); } - if !native_field(&policies, "selection_diagnostics")?.is_truthy()? { + if !native_field(policies, "selection_diagnostics")?.is_truthy()? { return Ok(output); } let stale = object( @@ -325,7 +328,7 @@ fn age_names<'py>(statuses: &Bound<'py, PyAny>, wanted: &str) -> PyResult(source_freshness: &Bound<'py, PyAny>) -> PyResult> { let py = source_freshness.py(); - let propagation = native_field(&source_freshness, "propagation")?; + let propagation = native_field(source_freshness, "propagation")?; let propagated = |field: &str| -> PyResult> { if propagation.is_none() { return tuple(py, Vec::new()); @@ -341,29 +344,32 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( let py = source_freshness.py(); let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; - diagnostics.set_item("enabled", native_field(&policies, "selection_diagnostics")?)?; + diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; output.metadata.push(( "direct_source_freshness", freshness_metadata(source_freshness)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs index 54ff7f9661..41cc14bbb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -6,6 +6,7 @@ use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_scope::values::{ dependency_names, edges, keys, loader_graph, loader_names, project_graph, }; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; @@ -14,7 +15,6 @@ use sqlbuild_analysis::planner_scope::main::{ expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, }; use std::collections::HashSet; -use crate::bindings::_helpers::records::fields::field as native_field; #[pyfunction] #[pyo3(signature = (*, loader_function, loader_functions))] @@ -35,8 +35,8 @@ pub(crate) fn loader_to_source_entry<'py>( loader: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let name = native_field(&loader, "name")?; - let mut destination = native_field(&loader, "destination")?; + let name = native_field(loader, "name")?; + let mut destination = native_field(loader, "destination")?; if destination.is_none() { destination = format!("__loader__{}", name.extract::()?) .into_pyobject(py)? @@ -46,8 +46,8 @@ pub(crate) fn loader_to_source_entry<'py>( py, &destination.extract::()?, ( - Some(native_field(&project, "effective_target_database")?), - Some(native_field(&project, "effective_target_schema")?), + Some(native_field(project, "effective_target_database")?), + Some(native_field(project, "effective_target_schema")?), ), )?; let kwargs = PyDict::new(py); @@ -79,7 +79,7 @@ pub(crate) fn intermediate_map<'py>( let graph = project_graph(project)?; let names = intermediate_sources(&graph, &keys(selected)?, upstream); let loaders = PyDict::new(project.py()); - for loader in native_field(&project, "loader_functions")?.try_iter()? { + for loader in native_field(project, "loader_functions")?.try_iter()? { let loader = loader?; loaders.set_item(native_field(&loader, "name")?, &loader)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index 0ca96ebe10..72ac0202d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_de use crate::bindings::_helpers::planner_scope::values::{edges, keys}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; @@ -22,7 +23,6 @@ use sqlbuild_analysis::planner_scope::main::{ managed_sources::managed_sources, source_selectors::source_selectors, }; use std::collections::HashSet; -use crate::bindings::_helpers::records::fields::field as native_field; #[pyfunction] #[pyo3(signature = (*, selected_keys, upstream_deps, project))] @@ -32,9 +32,12 @@ pub(crate) fn managed_source_upstream_keys<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let mut managed: HashSet = HashSet::new(); - for source in native_field(&project, "sources")?.try_iter()? { + for source in native_field(project, "sources")?.try_iter()? { let source = source?; - if !native_field(&source, "source_entry")?.getattr("loader")?.is_none() { + if !native_field(&source, "source_entry")? + .getattr("loader")? + .is_none() + { managed.insert(native_field(&source, "name")?.extract()?); } } @@ -73,7 +76,8 @@ pub(crate) fn build_planner_scope<'py>( let auto_load = args.value("auto_load_sources")?.is_truthy()?; let mut upstream = build_execution_upstream_deps(&project)?.into_any(); let mut downstream = build_downstream_deps(&upstream)?.into_any(); - let lineage: Bound<'py, NativeProjectGraph> = native_field(&project, "lineage_graph")?.cast_into()?; + let lineage: Bound<'py, NativeProjectGraph> = + native_field(&project, "lineage_graph")?.cast_into()?; let all_keys = lineage_graph_names(py, lineage.borrow())?.into_any(); let mut selected = args.value("selected_keys")?; if selected.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 55c6d00b5e..2a77feee21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -20,9 +20,9 @@ use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const VIEW_OPTIONS: &[&str] = &[ "all_keys", @@ -225,11 +225,11 @@ fn match_project_selector<'py>( tag_index, path_index, })?; - let value: Bound<'py, PyString> = native_field(&parsed, "value")?.downcast_into()?; + let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { Ok(graph_names(graph.graph())) })?; - let kind: String = native_field(&parsed, "kind")?.str()?.extract()?; + let kind: String = native_field(parsed, "kind")?.str()?.extract()?; selected( &carrier, match_selector(graph.graph(), &kind, &carrier.send(&value)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index f828abad66..81d2f1d76c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -53,7 +53,7 @@ pub(crate) fn planning_context( Some(models) => models, None => { let models = PyDict::new(py); - for model in native_field(&project, "models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, &model)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 8935deb2a9..ec5d29ca61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -21,8 +21,8 @@ use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -use crate::bindings::types::CompilerDetach; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::types::CompilerDetach; /// The columns each fixture must supply for the chain's models, and the fixtures read through a star. pub(crate) fn required_fixture_columns( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 9ea14abd7a..d1c4b85a8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; +use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::PyRefMut; use pyo3::exceptions::{PyOSError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -21,7 +22,6 @@ use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest: use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; use std::collections::{BTreeMap, HashMap}; use std::hash::{DefaultHasher, Hash, Hasher}; -use crate::bindings::_helpers::records::fields::field as native_field; const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs index f14f80fee7..5597535659 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -4,10 +4,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::relation_keys::entry_relation_key_row; -use crate::bindings::_helpers::records::fields::field as native_field; /// `CompilerDiagnostic` positions. const DIAGNOSTIC_PHASE_FIELD: usize = 0; @@ -95,7 +95,9 @@ pub(crate) fn diagnostic_is_error(instance: &Bound<'_, PyAny>) -> PyResult = instance.py(); let error: Bound<'_, PyAny> = registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?; - native_field(&instance, "severity")?.eq(error)?.into_py_any(py) + native_field(instance, "severity")? + .eq(error)? + .into_py_any(py) } /// `CompileSqlReference`: its kind coerced to `SqlReferenceKind`. @@ -219,17 +221,17 @@ fn reference_rows(references: &Bound<'_, PyAny>) -> PyResult> { /// `CompileModelInput.reference_rows` and `CompiledModel.reference_rows`. pub(crate) fn record_reference_rows(instance: &Bound<'_, PyAny>) -> PyResult> { - reference_rows(&native_field(&instance, "references")?) + reference_rows(&native_field(instance, "references")?) } /// The declared key facts of model `name` with `instance`'s config and schema entry. fn model_relation_key_row(instance: &Bound<'_, PyAny>, name: String) -> PyResult> { - let values: Bound<'_, PyAny> = native_field(&instance, "config")?.getattr("values")?; + let values: Bound<'_, PyAny> = native_field(instance, "config")?.getattr("values")?; entry_relation_key_row( "ref", name, values.cast::()?, - &native_field(&instance, "schema_entry")?, + &native_field(instance, "schema_entry")?, )? .into_py_any(instance.py()) } @@ -246,13 +248,13 @@ pub(crate) fn input_relation_key_row(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let name: String = native_field(&instance, "name")?.extract()?; + let name: String = native_field(instance, "name")?.extract()?; model_relation_key_row(instance, name) } /// `CompileModelInput.attached_schema_name`: the attached schema entry's name, or `None`. pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; if entry.is_none() { return Ok(entry.unbind()); } @@ -262,8 +264,8 @@ pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResul /// `(resource type, name)` of one compiled object key. fn key_row<'py>(key: &Bound<'py, PyAny>) -> PyResult<(String, String)> { Ok(( - native_field(&key, "resource_type")?.str()?.extract()?, - native_field(&key, "name")?.extract()?, + native_field(key, "resource_type")?.str()?.extract()?, + native_field(key, "name")?.extract()?, )) } @@ -274,17 +276,17 @@ fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { /// `CompiledModel.key_row`. pub(crate) fn model_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { - key_row(&native_field(&instance, "key")?)?.into_py_any(instance.py()) + key_row(&native_field(instance, "key")?)?.into_py_any(instance.py()) } /// `CompiledModel.dep_rows`. pub(crate) fn model_dep_rows(instance: &Bound<'_, PyAny>) -> PyResult> { - key_rows(&native_field(&instance, "deps")?)?.into_py_any(instance.py()) + key_rows(&native_field(instance, "deps")?)?.into_py_any(instance.py()) } /// `CompiledModel.destination_row`: `(database, schema, name)` of the destination. pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult> { - let destination: Bound<'_, PyAny> = native_field(&instance, "destination")?; + let destination: Bound<'_, PyAny> = native_field(instance, "destination")?; ( native_field(&destination, "database")?, native_field(&destination, "schema")?, @@ -295,7 +297,7 @@ pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let columns: Bound<'_, PyAny> = native_field(&instance, "inferred_columns")?; + let columns: Bound<'_, PyAny> = native_field(instance, "inferred_columns")?; if columns.is_none() { return Ok(columns.unbind()); } @@ -313,29 +315,32 @@ pub(crate) fn model_inferred_rows(instance: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; if entry.is_none() { return Ok(entry.unbind()); } let rows: Bound<'_, PyList> = PyList::empty(instance.py()); for column in native_field(&entry, "columns")?.try_iter()? { let column: Bound<'_, PyAny> = column?; - rows.append((native_field(&column, "name")?, native_field(&column, "type")?))?; + rows.append(( + native_field(&column, "name")?, + native_field(&column, "type")?, + ))?; } Ok(rows.into_any().unbind()) } /// `CompiledModel.has_dynamic_columns`. pub(crate) fn model_has_dynamic_columns(instance: &Bound<'_, PyAny>) -> PyResult> { - let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; + let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; let dynamic: bool = !entry.is_none() && native_field(&entry, "dynamic_columns")?.is_truthy()?; dynamic.into_py_any(instance.py()) } /// `CompiledModel.output_column_count`: inferred, else declared, columns, else zero. pub(crate) fn model_output_column_count(instance: &Bound<'_, PyAny>) -> PyResult> { - let inferred: Bound<'_, PyAny> = native_field(&instance, "inferred_columns")?; - let entry: Bound<'_, PyAny> = native_field(&instance, "schema_entry")?; + let inferred: Bound<'_, PyAny> = native_field(instance, "inferred_columns")?; + let entry: Bound<'_, PyAny> = native_field(instance, "schema_entry")?; let count: usize = if !inferred.is_none() { inferred.len()? } else if !entry.is_none() { @@ -352,20 +357,20 @@ pub(crate) fn build_context_declaration_expansion( ) -> PyResult> { let py: Python<'_> = instance.py(); let declarations: Bound<'_, PyDict> = PyDict::new(py); - declarations.set_item("enums", native_field(&instance, "public_enums")?)?; - declarations.set_item("constants", native_field(&instance, "public_constants")?)?; + declarations.set_item("enums", native_field(instance, "public_enums")?)?; + declarations.set_item("constants", native_field(instance, "public_constants")?)?; let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item( "declarations", registry::class(py, RecordId::DeclarationResolutionContext)? .call((), Some(&declarations))?, )?; - keywords.set_item("value_renderer", native_field(&instance, "value_renderer")?)?; + keywords.set_item("value_renderer", native_field(instance, "value_renderer")?)?; keywords.set_item( "collection_rendering", - native_field(&instance, "collection_rendering")?, + native_field(instance, "collection_rendering")?, )?; - keywords.set_item("resolver", native_field(&instance, "declaration_resolver")?)?; + keywords.set_item("resolver", native_field(instance, "declaration_resolver")?)?; Ok(registry::class(py, RecordId::DeclarationExpansionContext)? .call((), Some(&keywords))? .unbind()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs index 9066e08360..bf2008d88c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs @@ -1,8 +1,8 @@ //! Computed attributes of the natively hosted discovery and pipeline records. +use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; -use crate::bindings::_helpers::records::fields::field as native_field; /// SQL scalar type labels by logical value kind. const SCALAR_TYPE_LABELS: [(&str, &str); 6] = [ @@ -24,7 +24,7 @@ pub(crate) fn constant_logical_type(declaration: &Bound<'_, PyAny>) -> PyResult< /// `ConstantDeclaration.scalar_type`: the SQL scalar type label, or the upper-cased logical type. pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult> { - let value: Bound<'_, PyAny> = native_field(&declaration, "value")?; + let value: Bound<'_, PyAny> = native_field(declaration, "value")?; let kind: String = native_field(&value, "kind")?.str()?.extract()?; if let Some((_, label)) = SCALAR_TYPE_LABELS.iter().find(|(name, _)| *name == kind) { return label.into_py_any(declaration.py()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 592e5e280c..f5a82effb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -6,17 +6,24 @@ use std::path::PathBuf; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyTuple}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, +}; +use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_rules::fix_verification::main::refusal::refusal; use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; @@ -30,7 +37,6 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ }; use crate::bindings::_helpers::rules::relation_keys::{RelationKeys, project_relation_keys}; use crate::bindings::_helpers::rules::run_lint::given_files; -use crate::bindings::_helpers::records::fields::field as native_field; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; @@ -112,11 +118,9 @@ impl<'py> FixSession<'py> { fn run(&mut self) -> PyResult> { let py = self.original.py(); let mut current = self.original.copy()?; - let paths: Vec> = self - .inputs - .getattr("model_files")? + let paths: Vec> = field(&self.inputs, "model_files")? .try_iter()? - .map(|model| native_field(&model?, "file_path")) + .map(|model| model_file_path(&model?)) .collect::>()?; let model_paths = PyFrozenSet::new(py, paths)?; let mut seen: BTreeSet = BTreeSet::new(); @@ -207,15 +211,14 @@ impl<'py> FixSession<'py> { let mut pending: Vec> = Vec::new(); let mut remaining: Vec> = Vec::new(); for violation in violations { - if !violation - .getattr("code")? + if !field(&violation, "code")? .extract::()? .starts_with("SQBRSQL") { continue; } let reason = self.refusal(&violation, model_paths)?; - let edit = native_field(&violation, "fix")?; + let edit = field(&violation, "fix")?; if reason.is_some() || edit.is_none() { remaining.push(report( &violation, @@ -230,7 +233,7 @@ impl<'py> FixSession<'py> { )?); continue; } - let path = native_field(&edit, "file_path")?; + let path = field(&edit, "file_path")?; let chosen = if let Some(chosen) = edits.get_item(&path)? { chosen.cast_into::()? } else { @@ -238,12 +241,12 @@ impl<'py> FixSession<'py> { edits.set_item(&path, &chosen)?; chosen }; - let start: usize = native_field(&edit, "start")?.extract()?; - let end: usize = native_field(&edit, "end")?.extract()?; + let start: usize = field(&edit, "start")?.extract()?; + let end: usize = field(&edit, "end")?.extract()?; let mut overlaps = false; for other in chosen.iter() { - if start < native_field(&other, "end")?.extract::()? - && native_field(&other, "start")?.extract::()? < end + if start < field(&other, "end")?.extract::()? + && field(&other, "start")?.extract::()? < end { overlaps = true; break; @@ -262,20 +265,20 @@ impl<'py> FixSession<'py> { violation: &Bound<'py, PyAny>, model_paths: &Bound<'py, PyAny>, ) -> PyResult> { - let code = native_field(&violation, "code")?; - let path = native_field(&violation, "file_path")?; - let has_fix = !native_field(&violation, "fix")?.is_none(); + let code = field(violation, "code")?; + let path = field(violation, "file_path")?; + let has_fix = !field(violation, "fix")?.is_none(); let facts = FixEligibility { model_body: model_paths.contains(&path)?, has_fix, safe_code: SAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), conditionless_join: code.eq("SQBRSQL003")?, - snowflake: self.config.getattr("dialect")?.eq("snowflake")?, + snowflake: field(&self.config, "dialect")?.eq("snowflake")?, }; if let Some(reason) = refusal(&facts) { return Ok(Some(reason.to_owned())); } - let unavailable = native_field(&violation, "fix_unavailable_reason")?; + let unavailable = field(violation, "fix_unavailable_reason")?; if !unavailable.is_none() || !has_fix { return unavailable.extract(); } @@ -320,7 +323,7 @@ impl<'py> FixSession<'py> { let mut reports: Vec> = Vec::new(); for report in self.reports.iter().chain(pending) { reports.push( - if let Some(reason) = self.blocked.get_item(native_field(&report, "file_path")?)? { + if let Some(reason) = self.blocked.get_item(field(report, "file_path")?)? { refused(report, &reason.extract::()?)? } else { report.clone() @@ -475,8 +478,8 @@ impl<'py> FixSession<'py> { let affected = self.blocked.copy()?; let mut reports: Vec> = Vec::new(); for report in &self.reports { - if native_field(&report, "status")?.eq("applied")? { - affected.set_item(native_field(&report, "file_path")?, py.None())?; + if field(report, "status")?.eq("applied")? { + affected.set_item(field(report, "file_path")?, py.None())?; } reports.push(refused(report, reason)?); } @@ -493,12 +496,61 @@ fn replace<'py>( object: &Bound<'py, PyAny>, changes: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - let kwargs = keywords(object.py(), changes)?; - object - .py() - .import("dataclasses")? - .getattr("replace")? - .call((object,), Some(&kwargs)) + replaced(object, changes) +} + +/// A discovered model file's contents, read in place from a native model file. +fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + match model.cast::() { + Ok(native) => Ok(native.get().texts(model.py()).1.clone().into_any()), + Err(_) => native_field(model, "contents"), + } +} + +/// A discovered model file's path, read in place from a native model file. +fn model_file_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + match model.cast::() { + Ok(native) => Ok(native.get().paths(model.py()).0.clone()), + Err(_) => native_field(model, "file_path"), + } +} + +/// `model` with its edited text and the query `text` holds where the old query started. +fn edited_model<'py>( + model: &Bound<'py, PyAny>, + text: Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let Ok(native) = model.cast::() else { + let current = native_field(model, "contents")?; + let start = current.call_method1("index", (native_field(model, "query_sql")?,))?; + let query = text + .get_item(pyo3::types::PySlice::new( + py, + start.extract()?, + isize::MAX, + 1, + ))? + .call_method0("strip")?; + return model.call_method1("with_text", (text, query)); + }; + let native: &NativeModelFile = native.get(); + let (query_sql, contents) = native.texts(py); + let contents: String = contents.to_str()?.to_owned(); + let Some(offset) = contents.find(query_sql.to_str()?) else { + return Err(pyo3::exceptions::PyValueError::new_err( + "substring not found", + )); + }; + let start: usize = contents[..offset].chars().count(); + let text: Bound<'py, PyString> = text.cast_into::()?; + let edited: String = text.to_str()?.to_owned(); + let tail: &str = edited + .char_indices() + .nth(start) + .map_or("", |(index, _)| &edited[index..]); + let query: Bound<'py, PyString> = PyString::new(py, python_strip(tail)); + Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind()))?.into_any()) } fn updated_inputs<'py>( @@ -507,22 +559,13 @@ fn updated_inputs<'py>( ) -> PyResult> { let py = inputs.py(); let mut models: Vec> = Vec::new(); - for model in native_field(&inputs, "model_files")?.try_iter()? { + for model in field(inputs, "model_files")?.try_iter()? { let model = model?; - let current = native_field(&model, "contents")?; - if let Some(text) = files.get_item(native_field(&model, "file_path")?)? + let current = model_contents(&model)?; + if let Some(text) = files.get_item(model_file_path(&model)?)? && !text.eq(¤t)? { - let start = current.call_method1("index", (native_field(&model, "query_sql")?,))?; - let query = text - .get_item(pyo3::types::PySlice::new( - py, - start.extract()?, - isize::MAX, - 1, - ))? - .call_method0("strip")?; - models.push(model.call_method1("with_text", (text, query))?); + models.push(edited_model(&model, text)?); } else { models.push(model); } @@ -547,9 +590,9 @@ fn edited_files<'py>( .map(|edit| { let edit = edit?; Ok(FixEdit { - start: native_field(&edit, "start")?.extract()?, - end: native_field(&edit, "end")?.extract()?, - replacement: native_field(&edit, "replacement")?.extract()?, + start: field(&edit, "start")?.extract()?, + end: field(&edit, "end")?.extract()?, + replacement: field(&edit, "replacement")?.extract()?, }) }) .collect::>()?; @@ -574,7 +617,7 @@ fn failing_edits<'py>( for (path, _) in edits.iter() { resolved.set_item(path.call_method0("resolve")?, path)?; } - for (path, reason) in native_field(&verdict, "failing")?.cast_into::()?.iter() { + for (path, reason) in field(verdict, "failing")?.cast_into::()?.iter() { if let Some(original) = resolved.get_item(path)? { result.set_item(original, reason)?; } @@ -617,9 +660,9 @@ fn report<'py>( call_with( ®istry::class(py, RecordId::RuleFixResult)?, &[ - ("file_path", native_field(&source, "file_path")?), - ("code", native_field(&source, "code")?), - ("line", native_field(&source, "line")?), + ("file_path", field(source, "file_path")?), + ("code", field(source, "code")?), + ("line", field(source, "line")?), ("status", status.into_pyobject(py)?.into_any()), ("reason", reason.into_pyobject(py)?.into_any()), ], @@ -627,7 +670,7 @@ fn report<'py>( } fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult> { - if !native_field(&report, "status")?.eq("applied")? { + if !field(report, "status")?.eq("applied")? { return Ok(report.clone()); } replace( @@ -700,6 +743,5 @@ fn result<'py>( /// Whether a fix verdict proved every edit: nothing failed and nothing was unattributed. fn verified(verdict: &Bound<'_, PyAny>) -> PyResult { - Ok(!native_field(&verdict, "failing")?.is_truthy()? - && !native_field(&verdict, "unattributed")?.is_truthy()?) + Ok(!field(verdict, "failing")?.is_truthy()? && !field(verdict, "unattributed")?.is_truthy()?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index bba9cb572d..58fccc2ad9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index f0120f2865..8c71f4a136 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -3,11 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; +use std::path::Path; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::catalogue::{ @@ -15,21 +18,21 @@ use crate::bindings::_helpers::rules::catalogue::{ verify_custom_rules, }; use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; -use crate::bindings::_helpers::rules::fact_replay::project_models; +use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; -use crate::bindings::_helpers::records::fields::field as native_field; -const DATACLASSES_MODULE: &str = "dataclasses"; -const PATHLIB_MODULE: &str = "pathlib"; +const MODEL_RESOURCE_TYPE: &str = "model"; /// Build the catalogue and select the rules, project slice and settings one run evaluates. #[pyfunction] #[pyo3(signature = (**options))] -fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn plan_rule_run<'py>( + options: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let options = Request::new("plan_rule_run", options)?; let py: Python<'py> = options.py(); let graph: Bound<'py, PyAny> = options.get("graph")?; - let project: Bound<'py, PyAny> = native_field(&graph, "project")?; + let project: Bound<'py, PyAny> = field(&graph, "project")?; freed_memory_released(py); let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(&options.get("config")?, &project)?; @@ -53,14 +56,11 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> = Vec::new(); if options.get("no_sql_analysis")?.is_truthy()? - || !project - .getattr("settings")? - .getattr("sql_analysis")? - .is_truthy()? + || !field(&project_attribute(&project, "settings")?, "sql_analysis")?.is_truthy()? { let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { - let code: Bound<'py, PyAny> = native_field(&rule, "code")?; + let code: Bound<'py, PyAny> = field(&rule, "code")?; if TYPE_PROOF_RULE_CODES.contains(&code.extract::()?.as_str()) { skipped_type_proof_rules.push(code); } else { @@ -71,7 +71,7 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> = Vec::new(); for rule in &selected { - if native_field(&rule, "custom")?.is_truthy()? { + if field(rule, "custom")?.is_truthy()? { custom_rules.push(rule.clone()); } } @@ -86,19 +86,16 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> = Vec::with_capacity(selected.len()); for rule in &selected { - codes.push(native_field(&rule, "code")?); + codes.push(field(rule, "code")?); } let selection_config: Bound<'py, PyAny> = selection_rules_config(&effective_config, model_paths.as_ref(), &selected_project)?; - let selected_config: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?.call( - (selection_config,), - Some(&keywords( - py, - &[ - ("select", PyTuple::new(py, codes)?.into_any()), - ("ignore", PyTuple::empty(py).into_any()), - ], - )?), + let selected_config: Bound<'py, PyAny> = replaced( + &selection_config, + &[ + ("select", PyTuple::new(py, codes)?.into_any()), + ("ignore", PyTuple::empty(py).into_any()), + ], )?; call_with( ®istry::class(py, RecordId::RuleRunPlan)?, @@ -131,17 +128,15 @@ fn selected_model_paths<'py>( if selected_keys.is_none() { return Ok(None); } - let model_type: Bound<'py, PyAny> = - registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let names: Bound<'py, PySet> = PySet::empty(py)?; for key in selected_keys.try_iter()? { let key: Bound<'py, PyAny> = key?; - if native_field(&key, "resource_type")?.is(&model_type) { - names.add(native_field(&key, "name")?)?; + if field(&key, "resource_type")?.eq(MODEL_RESOURCE_TYPE)? { + names.add(field(&key, "name")?)?; } } let paths: Bound<'py, PySet> = PySet::empty(py)?; - for model in project_models(&native_field(&graph, "project")?)? { + for model in project_models(&field(graph, "project")?)? { if names.contains(&model.name)? { paths.add(model.posix)?; } @@ -180,43 +175,38 @@ fn selection_rules_config<'py>( let Some(model_paths) = model_paths else { return Ok(config.clone()); }; - let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; let model_names: Bound<'py, PySet> = PySet::empty(py)?; for model in project_models(project)? { model_names.add(model.name)?; } let mut rule_exceptions: Vec> = Vec::new(); - for entry in native_field(&config, "rule_exceptions")?.try_iter()? { + for entry in field(config, "rule_exceptions")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; - let path: Bound<'py, PyAny> = path_class - .call1((native_field(&entry, "path")?,))? - .call_method0("as_posix")?; + let path: String = field(&entry, "path")?.extract()?; + let path: String = String::from_utf8_lossy(&pure_path_posix(Path::new(&path))).into_owned(); if model_paths.contains(path)? { rule_exceptions.push(entry); } } let mut graph_edge_exceptions: Vec> = Vec::new(); - for entry in native_field(&config, "graph_edge_exceptions")?.try_iter()? { + for entry in field(config, "graph_edge_exceptions")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; - if model_names.contains(native_field(&entry, "consumer")?)? { + if model_names.contains(field(&entry, "consumer")?)? { graph_edge_exceptions.push(entry); } } - imported(py, DATACLASSES_MODULE, "replace")?.call( - (config,), - Some(&keywords( - py, - &[ - ( - "rule_exceptions", - PyTuple::new(py, rule_exceptions)?.into_any(), - ), - ( - "graph_edge_exceptions", - PyTuple::new(py, graph_edge_exceptions)?.into_any(), - ), - ], - )?), + replaced( + config, + &[ + ( + "rule_exceptions", + PyTuple::new(py, rule_exceptions)?.into_any(), + ), + ( + "graph_edge_exceptions", + PyTuple::new(py, graph_edge_exceptions)?.into_any(), + ), + ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 5588c9f8b5..7ea3883b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridg use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_cache::{ @@ -30,7 +31,6 @@ use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use pyo3::IntoPyObjectExt; -use crate::bindings::_helpers::records::fields::field as native_field; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index a3d84bf69b..4d3645ec91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -2,11 +2,11 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; -use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs index a47736e73d..5d8cccd3ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs @@ -2,8 +2,8 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::bindings::_helpers::records::fields::field as native_field; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; const UNKNOWN: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index 886cfeb6e1..fe32823a87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -14,13 +14,13 @@ use crate::bindings::_helpers::analysis::schema_request::{ }; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::semantic_shapes::shapes::{ model_inputs, project_shapes, published_shape, }; -use crate::bindings::_helpers::records::fields::field as native_field; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -131,12 +131,12 @@ fn rebound_models<'py>( ) -> PyResult>> { let py: Python<'py> = project.py(); let by_name: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(&project, "models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Object<'py> = model?; by_name.set_item(native_field(&model, "name")?, &model)?; } let graph: Bound<'py, PyDict> = PyDict::new(py); - for model in native_field(&project, "models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Object<'py> = model?; let mut upstream: Vec> = Vec::new(); for reference in native_field(&model, "references")?.try_iter()? { @@ -165,7 +165,9 @@ fn rebound_models<'py>( .getattr("config")? .getattr("values")? .call_method1("get", ("sql_analysis",))?; - if !selected_keys.contains(native_field(&model, "key")?)? || opted.is(PyBool::new(py, false)) { + if !selected_keys.contains(native_field(&model, "key")?)? + || opted.is(PyBool::new(py, false)) + { continue; } let mut reads_changed: bool = false; @@ -255,17 +257,19 @@ fn validated_diagnostics<'py>( dialect: &Object<'py>, ) -> PyResult> { let py: Python<'py> = project.py(); - let functions: Object<'py> = native_field(&project, "functions")?; + let functions: Object<'py> = native_field(project, "functions")?; let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let ignores: bool = - binding_catalog::ignores_quoted_case(&native_field(&project, "effective_connection")?, dialect)?; + let ignores: bool = binding_catalog::ignores_quoted_case( + &native_field(project, "effective_connection")?, + dialect, + )?; let mut requests: Vec> = Vec::with_capacity(models.len()); for model in models { let query_sql: String = analysis_cursor_sql( py, - &native_field(&model, "query_sql")?.extract::()?, + &native_field(model, "query_sql")?.extract::()?, &model .getattr("config")? .getattr("values")? @@ -305,7 +309,7 @@ fn validated_diagnostics<'py>( let mut diagnostics: Vec> = Vec::new(); for ((model, request), result) in models.iter().zip(&requests).zip(results.try_iter()?) { let result: Object<'py> = result?; - let expansion: Option> = expansions.get_item(native_field(&model, "name")?)?; + let expansion: Option> = expansions.get_item(native_field(model, "name")?)?; let mut mapper: Option> = None; for diagnostic in native_field(&result, "diagnostics")?.try_iter()? { let diagnostic: Object<'py> = diagnostic?; @@ -314,7 +318,7 @@ fn validated_diagnostics<'py>( } let built: Object<'py> = match mapper.take() { Some(built) => built, - None => positions(model, &native_field(&request, "sql")?, expansion.as_ref())?, + None => positions(model, &native_field(request, "sql")?, expansion.as_ref())?, }; let position: Object<'py> = native_field(&built, "position")?; mapper = Some(built); @@ -339,8 +343,8 @@ fn validated_diagnostics<'py>( ("code", native_field(&diagnostic, "code")?), ("message", native_field(&diagnostic, "message")?), ("resource_type", model_type.clone()), - ("resource_name", native_field(&model, "name")?), - ("path", native_field(&model, "relative_path")?), + ("resource_name", native_field(model, "name")?), + ("path", native_field(model, "relative_path")?), ("line", line), ("column", column), ], @@ -357,12 +361,12 @@ fn positions<'py>( expansion: Option<&Object<'py>>, ) -> PyResult> { let py: Python<'py> = model.py(); - let authored_query: Object<'py> = native_field(&model, "authored_query_sql")?; + let authored_query: Object<'py> = native_field(model, "authored_query_sql")?; let mut passes: Vec> = Vec::new(); let expanded: Object<'py> = match expansion { None => authored_query.clone(), Some(expansion) => { - for spans in native_field(&expansion, "passes")?.try_iter()? { + for spans in native_field(expansion, "passes")?.try_iter()? { let mut rows: Vec> = Vec::new(); for span in spans?.try_iter()? { let span: Object<'py> = span?; @@ -375,11 +379,11 @@ fn positions<'py>( } passes.push(PyTuple::new(py, rows)?.into_any()); } - native_field(&expansion, "expanded_sql")? + native_field(expansion, "expanded_sql")? } }; let request: Bound<'py, PyDict> = PyDict::new(py); - request.set_item("authored", native_field(&model, "authored_sql")?)?; + request.set_item("authored", native_field(model, "authored_sql")?)?; request.set_item("query", authored_query)?; request.set_item("expanded", expanded)?; request.set_item("cleaned", cleaned)?; @@ -443,8 +447,10 @@ fn placeholders<'py>(model: &Object<'py>) -> PyResult> { } fn ignores_quoted_case<'py>(project: &Object<'py>, dialect: &Object<'py>) -> PyResult> { - let ignores: bool = - binding_catalog::ignores_quoted_case(&native_field(&project, "effective_connection")?, dialect)?; + let ignores: bool = binding_catalog::ignores_quoted_case( + &native_field(project, "effective_connection")?, + dialect, + )?; Ok(PyBool::new(project.py(), ignores).to_owned().into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index d392e80625..c7cb9d5ec8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -5,12 +5,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::assembly::{ AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, }; -use crate::bindings::_helpers::records::fields::field as native_field; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ @@ -128,9 +128,9 @@ fn compiled_payload<'py>( copied(payload, &DIRECT_PAYLOAD_FIELDS)?, )?; let resource_type: Bound<'py, PyAny> = classes.model(RecordId::CompiledSqlTestResource)?; - let mode: Bound<'py, PyAny> = native_field(&payload, "mode")?; + let mode: Bound<'py, PyAny> = native_field(payload, "mode")?; let mut resources: Vec> = Vec::new(); - for name in native_field(&payload, "tested_resource_names")?.try_iter()? { + for name in native_field(payload, "tested_resource_names")?.try_iter()? { resources.push(keyword_call( &resource_type, vec![("kind", mode.clone()), ("name", name?)], @@ -162,7 +162,8 @@ fn keyed_diagnostic<'py>( (line, column, end_line, end_column, message, help): DiagnosticRow, ) -> PyResult> { let py: Python<'py> = classes.py(); - let path: Bound<'py, PyAny> = native_field(&test_input, "test_file")?.getattr("relative_path")?; + let path: Bound<'py, PyAny> = + native_field(test_input, "test_file")?.getattr("relative_path")?; let key: Bound<'py, PyTuple> = PyTuple::new( py, [ @@ -211,31 +212,41 @@ fn compiled_test<'py>( .iter() .map(|(resource_type, dep)| classes.object_key(resource_type, dep)) .collect::>()?; - let (payload, payload_fields) = compiled_payload(classes, &native_field(&test_input, "payload")?)?; - let test_file: Bound<'py, PyAny> = native_field(&test_input, "test_file")?; - let test_block: Bound<'py, PyAny> = native_field(&test_input, "test_block")?; + let (payload, payload_fields) = + compiled_payload(classes, &native_field(test_input, "payload")?)?; + let test_file: Bound<'py, PyAny> = native_field(test_input, "test_file")?; + let test_block: Bound<'py, PyAny> = native_field(test_input, "test_block")?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("key", classes.object_key("sql_test", &name)?), ("scope_deps", PyTuple::new(py, scope_deps)?.into_any()), ("name", text(py, &name)), ("test_file", test_file.clone()), ("test_block", test_block.clone()), - ("sql_body", native_field(&test_input, "sql_body")?), - ("mode", native_field(&test_input, "mode")?), + ("sql_body", native_field(test_input, "sql_body")?), + ("mode", native_field(test_input, "mode")?), ("payload", payload), ("source_path", native_field(&test_file, "relative_path")?), - ("ownership_root", native_field(&test_file, "ownership_root")?), + ( + "ownership_root", + native_field(&test_file, "ownership_root")?, + ), ("block_index", native_field(&test_block, "test_index")?), ("explicit_name", native_field(&test_block, "name")?), - ("parent_name", native_field(&test_input, "parent_name")?), - ("case_name", native_field(&test_input, "case_name")?), - ("case_index", native_field(&test_input, "case_index")?), + ("parent_name", native_field(test_input, "parent_name")?), + ("case_name", native_field(test_input, "case_name")?), + ("case_index", native_field(test_input, "case_index")?), ( "case_fingerprint", case_fingerprint.into_pyobject(py)?.into_any(), ), - ("parameter_schema", native_field(&test_input, "parameter_schema")?), - ("parameter_values", native_field(&test_input, "parameter_values")?), + ( + "parameter_schema", + native_field(test_input, "parameter_schema")?, + ), + ( + "parameter_values", + native_field(test_input, "parameter_values")?, + ), ( "target_model_names", PyTuple::new(py, target_model_names)?.into_any(), @@ -284,9 +295,9 @@ pub(crate) fn assemble_native_sql_tests<'py>( py: Python<'py>, inputs: &Bound<'py, PyAny>, ) -> PyResult> { - let test_inputs: Vec> = native_field(&inputs, "test_inputs")?.extract()?; + let test_inputs: Vec> = native_field(inputs, "test_inputs")?.extract()?; let request = AssemblyRequest { - model_inputs: native_field(&inputs, "model_inputs")?.extract()?, + model_inputs: native_field(inputs, "model_inputs")?.extract()?, test_inputs: test_inputs.clone(), lexical_syntax: inputs .getattr("sql_lexical_syntax")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs index cf276ef8d7..01856fb93e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -16,10 +16,10 @@ use crate::bindings::_helpers::planner_strategy::fixture_sql::{ }; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::constants::FIXTURE_PREFIXES; -use crate::bindings::_helpers::records::fields::field as native_field; /// Report every statically provable fixture defect or return the completed fixture SQL groups. pub(super) fn validated_fixtures<'py>( @@ -50,7 +50,7 @@ pub(super) fn validated_fixtures<'py>( let completion = complete_fixtures(context, adapter, (&chain, groups.as_any()))?; let mut errors: BTreeSet = BTreeSet::new(); for diagnostic in &completion.diagnostics { - let key = native_field(&diagnostic, "key")?; + let key = native_field(diagnostic, "key")?; errors.insert(format!( "{}: {}", fixture_location( @@ -58,7 +58,7 @@ pub(super) fn validated_fixtures<'py>( &key.get_item(0)?, &key.get_item(1)?.extract::()? )?, - native_field(&diagnostic, "message")?.extract::()? + native_field(diagnostic, "message")?.extract::()? )); } let inferred_by_fixture = &completion.inferred_by_fixture; @@ -142,7 +142,7 @@ pub(super) fn validated_fixtures<'py>( let label = if expected_output { "expected output".to_owned() } else { - format!("mock {}", native_field(&kind, "value")?.extract::()?) + format!("mock {}", native_field(kind, "value")?.extract::()?) }; errors.insert(format!( "{}: {label} '{name}' column '{}' has incompatible type {}; resource type is {}. Use an explicit CAST, ARRAY_CONSTRUCT, or PARSE_JSON as appropriate", @@ -155,7 +155,7 @@ pub(super) fn validated_fixtures<'py>( } } if !errors.is_empty() { - let test_name: String = native_field(&test, "name")?.extract()?; + let test_name: String = native_field(test, "name")?.extract()?; return Ok(Err(errors .into_iter() .map(|error| format!("SQL test '{test_name}': {error}")) @@ -185,12 +185,12 @@ pub(super) fn fixture_location( resource_type: &Bound<'_, PyAny>, name: &str, ) -> PyResult { - let value: String = native_field(&resource_type, "value")?.extract()?; + let value: String = native_field(resource_type, "value")?.extract()?; let prefix = FIXTURE_PREFIXES .iter() .find_map(|(kind, prefix)| (*kind == value).then_some(*prefix)) .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value))?; - let test_file = native_field(&test, "test_file")?; + let test_file = native_field(test, "test_file")?; let line = fixture_line( &native_field(&test_file, "contents")?.extract::()?, &format!("{prefix}{name}"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 6e30a5a7cb..38ee42ab5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -6,6 +6,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; @@ -15,7 +16,6 @@ use crate::bindings::_helpers::sql_test_glue::planning::{ }; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyTuple}; -use crate::bindings::_helpers::records::fields::field as native_field; pub(super) fn chains<'py>( project: &Bound<'py, PyAny>, @@ -26,7 +26,7 @@ pub(super) fn chains<'py>( return Ok(PyTuple::empty(py).into_any()); } let request = ChainRequest { - models: native_field(&project, "models")?.extract()?, + models: native_field(project, "models")?.extract()?, tests: tests.extract()?, lexical_syntax: project .getattr("sql_lexical_syntax")? @@ -76,7 +76,7 @@ pub(crate) fn planned<'py>( let functions = super::plan_rendering::functions(project, adapter)?; let overrides = super::plan_rendering::overrides(project, tests, adapter)?; let request = PlanningRequest { - models: native_field(&project, "models")?.extract()?, + models: native_field(project, "models")?.extract()?, rendered_model_sql: model_sql.extract()?, functions: functions.extract()?, tests: tests.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 29da1d46bb..b59b1d4c69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -9,17 +9,20 @@ use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; -use crate::bindings::_helpers::records::fields::field as native_field; fn prefixed_names<'py>(test: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { let py = test.py(); let mut names: Vec = if model_payload(test)? { - prefixed_ctes(&native_field(&test, "payload")?.getattr("authored_ctes")?, prefix)? - .keys() - .extract()? + prefixed_ctes( + &native_field(test, "payload")?.getattr("authored_ctes")?, + prefix, + )? + .keys() + .extract()? } else { Vec::new() }; @@ -36,7 +39,7 @@ fn function_deps<'py>( let py = test.py(); let deps = PyDict::new(py); if model_payload(test)? { - let names: Vec = native_field(&plan, "model_names")?.extract()?; + let names: Vec = native_field(plan, "model_names")?.extract()?; let mocked: std::collections::HashSet = inputs.0.extract::>()?.into_iter().collect(); let models = inputs.1; @@ -46,7 +49,9 @@ fn function_deps<'py>( }; for dep in native_field(&model, "deps")?.try_iter()? { let dep = dep?; - let resource: String = native_field(&dep, "resource_type")?.getattr("value")?.extract()?; + let resource: String = native_field(&dep, "resource_type")? + .getattr("value")? + .extract()?; if resource != UDF && resource != TABLE_FN { continue; } @@ -59,15 +64,17 @@ fn function_deps<'py>( } } } else { - let payload = native_field(&test, "payload")?; - let mode: String = native_field(&payload, "mode")?.getattr("value")?.extract()?; + let payload = native_field(test, "payload")?; + let mode: String = native_field(&payload, "mode")? + .getattr("value")? + .extract()?; if mode == UDF || mode == TABLE_FN { let names: std::collections::HashSet = payload .getattr("tested_resource_names")? .extract::>()? .into_iter() .collect(); - for function in native_field(&project, "functions")?.try_iter()? { + for function in native_field(project, "functions")?.try_iter()? { let function = function?; let key = native_field(&function, "key")?; if key @@ -124,8 +131,8 @@ pub(super) fn entry<'py>( "function_deps", function_deps(test, plan, inputs.0, (&mocks, inputs.1))?, )?; - fields.set_item("chain", native_field(&plan, "chain")?)?; - fields.set_item("assertions", native_field(&plan, "assertions")?)?; + fields.set_item("chain", native_field(plan, "chain")?)?; + fields.set_item("assertions", native_field(plan, "assertions")?)?; fields.set_item("sql_analysis_enabled", analysis)?; registry::class(py, RecordId::SqlTestPlanEntry)?.call((), Some(&fields)) } @@ -141,7 +148,7 @@ pub(super) fn expected_diagnostics( }; let py = test.py(); let resource_types = registry::class(py, RecordId::CompiledResourceType)?; - for step in native_field(&plan, "chain")?.try_iter()? { + for step in native_field(plan, "chain")?.try_iter()? { let step = step?; let name = native_field(&step, "model_name")?; let model_name: String = name.extract()?; @@ -169,7 +176,7 @@ pub(super) fn expected_diagnostics( )?; result.push(format!( "SQL test '{}': {}: {}", - native_field(&test, "name")?.extract::()?, + native_field(test, "name")?.extract::()?, location, message )); From 8bbac27272cddd0b669c2fc95a0c429be1fa33c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:11:55 +0100 Subject: [PATCH 10/11] fix: keep Ctrl-C raised while releasing freed memory after rules, and make the compile interrupt test deterministic --- .../src/bindings/_helpers/rules/rule_plan.rs | 2 +- .../src/bindings/_helpers/rules/rule_run.rs | 10 +++++++--- .../src/sqlbuild/cli/commands/main/compile/helpers.py | 11 ++++++++++- .../commands/main/compile/test_compile_early_lint.py | 2 ++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 8c71f4a136..126272ba37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -33,7 +33,7 @@ pub(crate) fn plan_rule_run<'py>( let py: Python<'py> = options.py(); let graph: Bound<'py, PyAny> = options.get("graph")?; let project: Bound<'py, PyAny> = field(&graph, "project")?; - freed_memory_released(py); + freed_memory_released(py)?; let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(&options.get("config")?, &project)?; let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index ab494cc30c..3b3961fc8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; @@ -145,16 +146,19 @@ fn rules_bulk_cache_path(project_dir: PathBuf, file_name: &str) -> PathBuf { } /// Hand freed glibc heap pages back to the OS; a no-op elsewhere or without `malloc_trim`. -pub(crate) fn freed_memory_released(py: Python<'_>) { +pub(crate) fn freed_memory_released(py: Python<'_>) -> PyResult<()> { if !cfg!(target_os = "linux") { - return; + return Ok(()); } let trim = || -> PyResult<()> { let process: Bound<'_, PyAny> = imported(py, CTYPES_MODULE, "CDLL")?.call1((py.None(),))?; let _ = native_field(&process, "malloc_trim")?.call1((0,))?; Ok(()) }; - let _ = trim(); + match trim() { + Err(error) if !error.is_instance_of::(py) => Err(error), + _ => Ok(()), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 94d94f9f18..71b02a24a6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4966,6 +4966,8 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: "import time\n\n\n" "def slow_total(amount: str) -> str:\n" ' """Return the amount, pausing for the macro tests that only SQL lint expands."""\n' + " if amount == 'model_amount':\n" + " time.sleep(2.0)\n" " if amount == 'slow_amount':\n" " with open(os.environ['SLOW_MACRO_MARKER'], 'a', encoding='utf-8') as marker:\n" " marker.write('expanding\\n')\n" @@ -4985,7 +4987,8 @@ def slow_lint_model_files(*, model_count: int) -> dict[str, str]: ), "models/slow_total.sql": ( 'MODEL (description "Slow total");\n' - 'SELECT @slow_total("amount") AS amount FROM (SELECT 5 AS amount) AS input_values\n' + 'SELECT @slow_total("model_amount") AS amount\n' + "FROM (SELECT 5 AS model_amount) AS input_values\n" ), **{ f"tests/slow_total_{index}.sql": ( @@ -5001,3 +5004,9 @@ def slow_lint_model_files(*, model_count: int) -> dict[str, str]: for index in range(model_count) }, } + + +def restore_default_interrupt() -> None: + """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py index 8ca339f922..77d262f4b3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py @@ -19,6 +19,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( compile_json_payload, diagnostic_location_keys, + restore_default_interrupt, slow_lint_model_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project @@ -132,6 +133,7 @@ def test_given_slow_background_lint_when_interrupted_then_compile_stops_promptly "SLOW_MACRO_MARKER": str(marker), "SLOW_MACRO_SECONDS": str(test_case.macro_seconds), }, + preexec_fn=restore_default_interrupt, ) deadline: float = time.monotonic() + 60.0 while not marker.exists() and time.monotonic() < deadline and process.poll() is None: From 70a58ebbc4b45465a7a4085c982e7ce26a78ca85 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:14:09 +0100 Subject: [PATCH 11/11] fix: report SQL test planning panics as native compiler errors --- .../_helpers/planner_entries/test_planning.rs | 19 +++++++++++-------- python-boundary.toml | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs index 08cbc3a1b3..ed2e2d34d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs @@ -13,6 +13,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -159,15 +160,17 @@ pub(crate) struct PlanningRequest { pub(crate) sql_test_selection: Py, } -/// Plan the request's selected SQL tests. +/// Plan the request's selected SQL tests, reporting a panic as a native compiler error. fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { - plan_selected_sql_tests( - request.project.bind(py), - request.adapter.bind(py), - request.selected_keys.bind(py), - request.sql_test_selection.bind(py), - ) - .map(Bound::unbind) + compiler_guard(|| { + plan_selected_sql_tests( + request.project.bind(py), + request.adapter.bind(py), + request.selected_keys.bind(py), + request.sql_test_selection.bind(py), + ) + .map(Bound::unbind) + }) } /// Plan on a native thread when `enabled`, or resolve at once to no planned tests. diff --git a/python-boundary.toml b/python-boundary.toml index 84f3282619..5857186522 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -745,7 +745,7 @@ note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; con [[stays]] path = "compiler/compile/types.py" rule = 1 -note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." +note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." [[stays]] path = "rule_engine/types.py"