From e643366929893172f59e0daa34d4f0c7ed5ade81 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:29:49 +0000 Subject: [PATCH 1/3] refactor: retain native compiler and planner ownership --- CONTRIBUTING.md | 22 + .../edge_cases/rpt_edge_header_contracts.sql | 23 + .../test_rpt_edge_header_contracts.sql | 15 + .../src/graph/_helpers/indexes.rs | 39 +- .../src/graph/main/downstream_edges.rs | 24 + .../sqlbuild-analysis/src/graph/main/mod.rs | 3 + .../src/graph/main/reachable_graph_keys.rs | 23 + .../src/graph/main/reachable_keys.rs | 21 + crates/sqlbuild-analysis/src/graph/models.rs | 7 + .../main/execution_upstream.rs | 58 +++ .../src/planner_identity/main/mod.rs | 1 + .../src/planner_identity/models.rs | 19 + .../src/model_files/main/config_value.rs | 43 ++ .../src/model_files/main/mod.rs | 1 + .../src/header_metadata/constants.rs | 5 + .../analysis_session/model_analyses.rs | 28 +- .../_helpers/analysis_session/session.rs | 13 + .../analysis_session/stage_functions.rs | 18 +- .../_helpers/attachments/argument_values.rs | 33 +- .../compile_driver/assembly_analysis.rs | 24 +- .../_helpers/compile_driver/assembly_facts.rs | 91 +++- .../compile_driver/selection_phases.rs | 24 +- .../compiled_views/compiled_model_view.rs | 11 +- .../compiled_views/model_input_view.rs | 28 +- .../_helpers/compiled_views/relation_keys.rs | 14 +- .../_helpers/compiled_views/resource_reads.rs | 36 +- .../src/bindings/_helpers/graph/keys.rs | 29 +- .../_helpers/input_loops/audit_loop.rs | 83 ++-- .../_helpers/input_loops/model_loop.rs | 9 +- .../_helpers/input_loops/model_loop_config.rs | 29 +- .../input_loops/model_loop_contracts.rs | 86 ++-- .../_helpers/input_loops/model_loop_header.rs | 100 ++-- .../_helpers/input_loops/model_loop_host.rs | 4 +- .../input_loops/resource_attachment.rs | 3 +- .../_helpers/model_config/header_audits.rs | 445 ++++++++++++++++++ .../_helpers/model_config/header_families.rs | 65 +++ .../_helpers/model_config/header_metadata.rs | 247 ++++++++-- .../src/bindings/_helpers/model_config/mod.rs | 2 + .../model_config/model_config_builder.rs | 34 +- .../_helpers/native_host/header_projection.rs | 54 +++ .../src/bindings/_helpers/native_host/mod.rs | 2 + .../_helpers/native_host/model_lookup.rs | 88 ++++ .../_helpers/planner_changes/actions.rs | 19 +- .../_helpers/planner_changes/detect.rs | 27 +- .../_helpers/planner_changes/renames.rs | 8 +- .../_helpers/planner_entries/clone_plans.rs | 19 +- .../_helpers/planner_entries/contexts.rs | 2 +- .../_helpers/planner_entries/display_plan.rs | 13 +- .../planner_entries/execution_plan.rs | 16 +- .../_helpers/planner_entries/model_entries.rs | 30 +- .../_helpers/planner_entries/models.rs | 3 +- .../_helpers/planner_entries/relations.rs | 45 +- .../_helpers/planner_identity/graph.rs | 223 +++++---- .../_helpers/planner_identity/identities.rs | 47 +- .../_helpers/planner_identity/staleness.rs | 71 +-- .../planner_migrations/fingerprint.rs | 30 +- .../_helpers/planner_phases/buildability.rs | 88 ++-- .../_helpers/planner_phases/contexts.rs | 16 + .../_helpers/planner_phases/getters.rs | 15 + .../_helpers/planner_phases/identities.rs | 34 +- .../_helpers/planner_phases/models.rs | 2 +- .../_helpers/planner_phases/objects.rs | 13 - .../planner_phases/output_assembly.rs | 56 +-- .../_helpers/planner_phases/retention.rs | 14 +- .../_helpers/planner_phases/scopes.rs | 71 +-- .../planner_phases/warehouse_state.rs | 19 +- .../_helpers/planner_scenario/execution.rs | 25 +- .../_helpers/planner_scope/loaders.rs | 9 +- .../bindings/_helpers/planner_scope/mod.rs | 3 +- .../_helpers/planner_scope/retained.rs | 198 ++++++++ .../bindings/_helpers/planner_scope/scope.rs | 174 +++---- .../_helpers/planner_scope/sources.rs | 34 +- .../_helpers/planner_selection/selectors.rs | 20 +- .../planner_strategy/audit_entries.rs | 81 ++-- .../_helpers/planner_strategy/audits.rs | 97 ++-- .../planner_strategy/cursor_inputs.rs | 26 +- .../planner_strategy/fixture_context.rs | 33 +- .../planner_strategy/fixture_reads.rs | 6 +- .../_helpers/planner_strategy/lineage.rs | 17 +- .../_helpers/planner_strategy/models.rs | 12 +- .../_helpers/planner_warehouse/cursors.rs | 2 +- .../_helpers/planner_warehouse/snapshot.rs | 18 +- .../planner_warehouse/source_freshness.rs | 17 +- .../planner_warehouse/source_tables.rs | 25 +- .../project_assembly/declared_relations.rs | 2 +- .../project_assembly/syntax_checks.rs | 26 +- .../_helpers/semantic_checks/metadata.rs | 100 ++-- fensu.toml | 36 +- .../test_native_compiled_project.py | 17 + .../compiler/model_config/_test_types.py | 9 + .../model_config/test_native_model_config.py | 39 ++ .../compiler/planner/_helpers/helpers.py | 10 +- .../planner/_helpers/planning/test_scopes.py | 11 + 93 files changed, 2707 insertions(+), 1125 deletions(-) create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_header_contracts.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_header_contracts.sql create mode 100644 crates/sqlbuild-analysis/src/graph/main/downstream_edges.rs create mode 100644 crates/sqlbuild-analysis/src/graph/main/reachable_graph_keys.rs create mode 100644 crates/sqlbuild-analysis/src/graph/main/reachable_keys.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/execution_upstream.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/main/config_value.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c75806fac3..f0ab39173e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,6 +111,28 @@ uv run fensu dupes --since origin/main - Use project-local config files for local overrides, and do not commit credentials or personal settings. - Follow existing code style and module organization before introducing new patterns. +### Native ownership and Python interfaces + +Compiler and planner algorithms belong in the Rust crates. Keep parsed header metadata, config +nodes, dependency graphs and selections native between their producers and consumers. Constructing +a Python container or public record only to extract it again is not an interface requirement. + +The supported Python interfaces have specific owners: + +| Interface | Why Python remains | Ownership rule | +| --- | --- | --- | +| Adapters and warehouse execution | Projects subclass adapters; the executor runs warehouse operations | Call through the adapter/executor boundary, retaining their public inputs and results | +| User functions, factories, hooks, rules and custom constructors | These execute project-authored Python | Preserve invocation timing, errors and opaque return values; do not execute them speculatively | +| Public records and caller-authored protocol objects | Python callers access, replace, compare and serialize documented records | Project native values on public access; import caller-authored values at entry, not repeatedly inside native algorithms | +| Config and sample values | Python can supply values without a lossless native representation | Retain opaque values and their Python semantics; keep ordinary builtin containers native | +| Streams, progress, logging and interpreter shutdown | These integrate with Python's runtime and caller-owned resources | Keep resource lifetime and error propagation at the owning boundary | + +`python-boundary.toml` lists the Python source that intentionally stays. The native crossing, +storage and layer rules in `fensu.toml` prevent new accidental dependencies; pending budgets are +an inventory to reduce, not permission to add callbacks. `native_host` owns genuine host edges, +not a destination for movable compiler or planner algorithms. Remove an obsolete private +interface with its last caller, while retaining supported public facades and stubs. + ## Commits And Pull Requests - Use concise conventional commit messages, such as `fix: ...`, `feat: ...`, `test: ...`, or `docs: ...`. diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_header_contracts.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_header_contracts.sql new file mode 100644 index 0000000000..5b34b39789 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_header_contracts.sql @@ -0,0 +1,23 @@ +MODEL ( + description "Order contract metadata retained beside a templated destination alias", + tags [edge_case], + alias "rpt_edge_header_contracts_${reporting_currency}", + contract enforced, + columns ( + order_id (type INTEGER, nullable false, audits [not_null, unique]), + status (type order_status, nullable false, audits [not_null]), + amount (type DECIMAL(10, 2)), + ), +); + +WITH final AS ( + SELECT + 7::INTEGER AS order_id, + @enum("order_status").PENDING::VARCHAR AS status, + 12.50::DECIMAL(10, 2) AS amount + FROM __seed("currencies") + WHERE currency_code = 'USD' +) + +SELECT order_id, status, amount +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_header_contracts.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_header_contracts.sql new file mode 100644 index 0000000000..3d80151069 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_header_contracts.sql @@ -0,0 +1,15 @@ +TEST (name "rpt_edge_header_contracts__keeps_enum_contract_with_templated_alias"); + +WITH __seed__currencies AS ( + SELECT 'USD' AS currency_code + UNION ALL + SELECT 'EUR' AS currency_code +), +__expected__rpt_edge_header_contracts AS ( + SELECT + 7::INTEGER AS order_id, + 'PENDING'::VARCHAR AS status, + 12.50::DECIMAL(10, 2) AS amount +) + +SELECT 1 diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs b/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs index b5689372f9..1601c07502 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs @@ -4,20 +4,14 @@ use std::collections::{HashMap, HashSet}; use crate::assembly::project::types::ObjectKey; use crate::graph::constants::{MODEL_ROOT, MODEL_ROOT_PREFIX, SQL_TEST_RESOURCE}; -use crate::graph::models::{GraphIndexes, GraphResource, ProjectGraph}; +use crate::graph::main::downstream_edges::downstream_edges; +use crate::graph::models::{GraphIndexes, GraphResource, OrderedEdges, ProjectGraph}; type Edges = Vec<(ObjectKey, Vec)>; -/// Insertion-ordered key lists, as Python's `dict.setdefault(key, []).append(...)` builds them. -#[derive(Default)] -struct OrderedEdges { - entries: Edges, - index: HashMap, -} - impl OrderedEdges { /// The list stored for `key`, appended empty in insertion order when the key is new. - fn list(&mut self, key: &ObjectKey) -> &mut Vec { + pub(crate) fn list(&mut self, key: &ObjectKey) -> &mut Vec { let position: usize = match self.index.get(key) { Some(&position) => position, None => { @@ -28,6 +22,13 @@ impl OrderedEdges { }; &mut self.entries[position].1 } + + /// The dependencies already stored for `key`, without inserting a missing target. + pub(crate) fn get(&self, key: &ObjectKey) -> Option<&Vec> { + self.index + .get(key) + .map(|&position| &self.entries[position].1) + } } /// Tag to keys in first-tagged order, each key once, as Python's tag sets hold them. @@ -70,7 +71,7 @@ pub(crate) fn build(resources: &[GraphResource]) -> ProjectGraph { { *upstream.list(&resource.key) = lineage_deps(&resource.deps); } - let downstream: Edges = downstream(&upstream.entries); + let downstream: Edges = downstream_edges(&upstream.entries); let mut tags: OrderedTags = OrderedTags::default(); let mut graph = ProjectGraph { downstream_index: positions(&downstream), @@ -125,24 +126,6 @@ fn lineage_deps(deps: &[ObjectKey]) -> Vec { .collect() } -/// Inverted edges keyed in upstream order, each list sorted by `(resource type, name)`. -fn downstream(upstream: &[(ObjectKey, Vec)]) -> Edges { - let mut inverted: OrderedEdges = OrderedEdges::default(); - for (key, _) in upstream { - let _ = inverted.list(key); - } - for (key, deps) in upstream { - for dep in deps { - inverted.list(dep).push(key.clone()); - } - } - let mut entries: Edges = inverted.entries; - for (_, keys) in &mut entries { - keys.sort(); - } - entries -} - fn positions(entries: &[(ObjectKey, Vec)]) -> HashMap { entries .iter() diff --git a/crates/sqlbuild-analysis/src/graph/main/downstream_edges.rs b/crates/sqlbuild-analysis/src/graph/main/downstream_edges.rs new file mode 100644 index 0000000000..b34d065dfc --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/downstream_edges.rs @@ -0,0 +1,24 @@ +//! Reverse ordered execution or lineage edges without projecting Python graph maps. + +use crate::assembly::project::types::ObjectKey; +use crate::graph::models::OrderedEdges; + +/// Inverted edges keyed in upstream order, each list sorted by `(resource type, name)`. +pub fn downstream_edges( + upstream: &[(ObjectKey, Vec)], +) -> Vec<(ObjectKey, Vec)> { + let mut inverted = OrderedEdges::default(); + for (key, _) in upstream { + let _ = inverted.list(key); + } + for (key, deps) in upstream { + for dep in deps { + inverted.list(dep).push(key.clone()); + } + } + let mut entries = inverted.entries; + for (_, keys) in &mut entries { + keys.sort(); + } + entries +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index f461ab6399..65c5eea9e7 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,10 +1,13 @@ pub mod build_project_graph; pub mod build_resources; +pub mod downstream_edges; pub mod edge_keys; pub mod graph_from_indexes; pub mod match_selector; pub mod model_layer_count; pub mod parse_selector; +pub mod reachable_graph_keys; +pub mod reachable_keys; pub mod resolve_selector_tokens; pub mod resolve_selectors; pub mod selector_name_help; diff --git a/crates/sqlbuild-analysis/src/graph/main/reachable_graph_keys.rs b/crates/sqlbuild-analysis/src/graph/main/reachable_graph_keys.rs new file mode 100644 index 0000000000..b01f621b23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/reachable_graph_keys.rs @@ -0,0 +1,23 @@ +//! Reachable keys over the project's existing edge indexes. + +use std::collections::HashSet; +use std::convert::Infallible; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::main::edge_keys::edge_keys; +use crate::graph::models::{EdgeDirection, ProjectGraph}; + +/// The starts and every key reachable in the given direction. +pub fn reachable_graph_keys( + starts: &[ObjectKey], + graph: &ProjectGraph, + direction: EdgeDirection, +) -> HashSet { + let mut visited: HashSet<&ObjectKey> = starts.iter().collect(); + let Ok(()) = breadth_first_reach(starts, None, &mut visited, |current| { + Ok::<_, Infallible>(edge_keys(graph, current, direction)) + }); + visited.into_iter().cloned().collect() +} diff --git a/crates/sqlbuild-analysis/src/graph/main/reachable_keys.rs b/crates/sqlbuild-analysis/src/graph/main/reachable_keys.rs new file mode 100644 index 0000000000..c05009b6d5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/reachable_keys.rs @@ -0,0 +1,21 @@ +//! Reachable compiled resource keys without projecting an intermediate host graph. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::assembly::project::types::ObjectKey; + +/// The starts and every key reachable through the ordered dependency lists. +pub fn reachable_keys( + starts: &[ObjectKey], + edges: &[(ObjectKey, Vec)], +) -> HashSet { + let index: HashMap<_, _> = edges.iter().map(|(key, deps)| (key, deps)).collect(); + let mut visited: HashSet<&ObjectKey> = starts.iter().collect(); + let Ok(()) = breadth_first_reach(starts, None, &mut visited, |current| { + Ok::<_, Infallible>(index.get(current).map_or(&[][..], |deps| deps.as_slice())) + }); + visited.into_iter().cloned().collect() +} diff --git a/crates/sqlbuild-analysis/src/graph/models.rs b/crates/sqlbuild-analysis/src/graph/models.rs index e13685f2e6..61ffc53442 100644 --- a/crates/sqlbuild-analysis/src/graph/models.rs +++ b/crates/sqlbuild-analysis/src/graph/models.rs @@ -4,6 +4,13 @@ use std::collections::HashMap; use crate::assembly::project::types::ObjectKey; +/// Dependency lists indexed without losing project insertion order. +#[derive(Debug, Default)] +pub(crate) struct OrderedEdges { + pub(crate) entries: Vec<(ObjectKey, Vec)>, + pub(crate) index: HashMap, +} + /// One graph resource in project order: its key, upstream keys, tags and model folder. #[derive(Clone, Debug, PartialEq, Eq)] pub struct GraphResource { diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/execution_upstream.rs b/crates/sqlbuild-analysis/src/planner_identity/main/execution_upstream.rs new file mode 100644 index 0000000000..17cbe67103 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/execution_upstream.rs @@ -0,0 +1,58 @@ +//! Execution edges in project order, including SQL tests and injected reads. + +use std::collections::{HashMap, HashSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::models::OrderedEdges; +use crate::planner_identity::constants::{TABLE_FN_RESOURCE, UDF_RESOURCE}; +use crate::planner_identity::models::ExecutionTest; + +/// Weave tests before their targets and append distinct gate and hook dependencies. +pub fn execution_upstream( + resources: Vec<(ObjectKey, Vec)>, + tests: &[ExecutionTest], + injected: &[(ObjectKey, ObjectKey)], +) -> Vec<(ObjectKey, Vec)> { + let mut upstream = OrderedEdges::default(); + for (key, deps) in resources { + *upstream.list(&key) = deps; + } + for test in tests { + let mut seen: HashSet = HashSet::new(); + let deps: Vec = if test.table_function { + test.scope_deps + .iter() + .filter(|key| matches!(key.0.as_str(), UDF_RESOURCE | TABLE_FN_RESOURCE)) + .cloned() + .collect() + } else { + test.scope_deps + .iter() + .filter_map(|target| upstream.get(target)) + .flatten() + .filter(|key| matches!(key.0.as_str(), UDF_RESOURCE | TABLE_FN_RESOURCE)) + .filter(|key| seen.insert((*key).clone())) + .cloned() + .collect() + }; + *upstream.list(&test.key) = deps; + if !test.table_function { + for target in &test.scope_deps { + if upstream.get(target).is_some() { + upstream.list(target).push(test.key.clone()); + } + } + } + } + let mut injected_seen: HashMap> = HashMap::new(); + for (gated, read) in injected { + let deps = upstream.list(gated); + let seen = injected_seen + .entry(gated.clone()) + .or_insert_with(|| deps.iter().cloned().collect()); + if seen.insert(read.clone()) { + deps.push(read.clone()); + } + } + upstream.entries +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs index 219d458e53..94c535cd51 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs @@ -1,4 +1,5 @@ pub mod classify_selection_staleness; +pub mod execution_upstream; pub mod expected_graph_identity_hashes; pub mod function_fingerprint_sql; pub mod graph_write_identity_hashes; diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index 54a84b5fda..f11d73b77b 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -2,6 +2,8 @@ use std::collections::{HashMap, HashSet}; +use crate::assembly::project::types::ObjectKey; + /// A graph node identity: resource type (or fingerprint node type) and name, ordered as Python's `(resource_type, name)`. #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct GraphKey { @@ -9,6 +11,23 @@ pub struct GraphKey { pub name: String, } +impl From<&ObjectKey> for GraphKey { + fn from((kind, name): &ObjectKey) -> Self { + Self { + kind: kind.clone(), + name: name.clone(), + } + } +} + +/// One SQL test woven into its targets' execution dependencies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExecutionTest { + pub key: ObjectKey, + pub scope_deps: Vec, + pub table_function: bool, +} + /// One node of an identity graph: its upstream keys in order and its local hash. #[derive(Clone, Debug, PartialEq, Eq)] pub struct IdentityNode { diff --git a/crates/sqlbuild-discovery/src/model_files/main/config_value.rs b/crates/sqlbuild-discovery/src/model_files/main/config_value.rs new file mode 100644 index 0000000000..51960dcae2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/config_value.rs @@ -0,0 +1,43 @@ +//! Authored header values retained as config nodes without reading their Python projection. + +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; + +use crate::model_files::_helpers::native_nodes::dict_entries; + +/// Retain builtin values natively; hook objects, declaration markers and huge integers need host identity. +pub fn config_value(value: &AuthoredValue) -> Option> { + Some(match value { + AuthoredValue::Null => ConfigNode::Null, + AuthoredValue::Boolean(value) => ConfigNode::Bool(*value), + AuthoredValue::String(value) => ConfigNode::Str(value.clone()), + AuthoredValue::BareWord(value) => match header_word_value(value) { + HeaderWord::Bool(value) => ConfigNode::Bool(value), + HeaderWord::Null => ConfigNode::Null, + HeaderWord::Integer(value) => match value.parse() { + Ok(value) => ConfigNode::Int(value), + Err(_) => return None, + }, + HeaderWord::Float(value) => ConfigNode::Float(value), + HeaderWord::Text => ConfigNode::Str(value.clone()), + }, + AuthoredValue::List(values) => { + ConfigNode::List(values.iter().map(config_value).collect::>()?) + } + AuthoredValue::Tuple(values) => { + ConfigNode::Tuple(values.iter().map(config_value).collect::>()?) + } + AuthoredValue::Map(values) => ConfigNode::Map( + dict_entries(values) + .into_iter() + .map(|(key, value)| Some((ConfigNode::Str(key.to_owned()), config_value(value)?))) + .collect::>()?, + ), + AuthoredValue::Set(_) + | AuthoredValue::TypedConstant(_) + | AuthoredValue::InlineSqlHook(_) + | AuthoredValue::NamedSqlHook(_, _) + | AuthoredValue::PythonHook(_, _) => return None, + }) +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index 4d936ccc0a..d9a6179c01 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,3 +1,4 @@ +pub mod config_value; pub mod discover_model_files; pub mod header_facts; pub mod model_file_options; diff --git a/crates/sqlbuild-model-config/src/header_metadata/constants.rs b/crates/sqlbuild-model-config/src/header_metadata/constants.rs index cc093ed623..ca620d491d 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/constants.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/constants.rs @@ -28,6 +28,11 @@ pub const ABOVE_OPERATOR: &str = "above"; pub const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; /// The audit that contradicts `nullable = true`. pub const NOT_NULL_AUDIT_NAME: &str = "not_null"; +/// The enum membership audit and its ordered values argument. +pub const ACCEPTED_VALUES_AUDIT: &str = "accepted_values"; +pub const ACCEPTED_VALUES_ARGUMENT: &str = "values"; +/// The audit collection on a schema column or model. +pub const AUDITS_KEY: &str = "audits"; /// The help every `ResourceIdentityError` shows. pub const IDENTITY_HELP: &str = "Rename the authored identity and update its references, selectors, \ and integration keys. SQLBuild does not silently normalize resource identities. Double \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 9d387684be..e964962a2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::analysis_session::session::{ CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, start_model_analysis_session, }; -use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; use crate::bindings::_helpers::compile_driver::analysis_cache::{ @@ -34,6 +33,7 @@ use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; const LOGGING_MODULE: &str = "logging"; @@ -50,7 +50,7 @@ pub(crate) struct ModelAnalysisRequest<'a, 'py> { pub(crate) column_types: Bound<'py, PyAny>, pub(crate) nullability: Bound<'py, PyAny>, pub(crate) complete_schemas: Bound<'py, PyAny>, - pub(crate) dynamic_families: Bound<'py, PyAny>, + pub(crate) dynamic_families: &'a [(String, Vec)], pub(crate) rich_type_inference: bool, /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. pub(crate) cache: Bound<'py, PyAny>, @@ -104,11 +104,16 @@ fn session_request<'py>( rules: Vec<(String, String)>, ) -> PyResult { let profile: &Bound<'py, PyAny> = &request.profile; - let mut families: Vec<(String, Vec)> = Vec::new(); - for item in mapping_items(&request.dynamic_families)?.try_iter()? { - let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; - families.push((name, family_rows(&declared)?)); - } + let families: Vec<(String, Vec)> = request + .dynamic_families + .iter() + .map(|(name, declared)| { + ( + name.clone(), + declared.iter().cloned().map(FamilyRow::from).collect(), + ) + }) + .collect(); let shapes: RelationShapes = match declared_rows(request, catalog)? { Some(rows) => rows, None => RelationShapes { @@ -352,7 +357,7 @@ fn model_rows<'py>( } } let references: Vec<(String, String)> = analysed_references(model_input)?; - let schema: Option> = attached_schema(model_input)?; + let schema: Option = attached_schema(model_input)?; let recovers: bool = get("contract")?.eq(&enforced)? || schema .as_ref() @@ -366,7 +371,12 @@ fn model_rows<'py>( None }; let families = match &schema { - Some(schema) => family_rows(&schema.dynamic_columns)?, + Some(schema) => schema + .dynamic_columns + .iter() + .cloned() + .map(FamilyRow::from) + .collect(), None => Vec::new(), }; rows.push(ModelRow { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index c2487bf4a1..096934a644 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -70,6 +70,19 @@ positional_record! { } } +impl From for FamilyRow { + fn from(family: sqlbuild_model_config::header_metadata::models::DynamicFamily) -> Self { + Self { + name: family.name, + pivot_column: family.pivot_column, + value_column: family.value_column, + aggregate: family.aggregate, + data_type: family.data_type, + name_pattern: family.name_pattern, + } + } +} + /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. pub(crate) struct ModelRow { pub(crate) name: String, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index d4025bf278..2f041b3a80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::stage_work::{ EXPRESSION_SHAPE_BATCHES, EXPRESSION_SHAPES_INFERRED, count_stage, }; +use crate::bindings::_helpers::model_config::header_families::family_specs; use crate::bindings::_helpers::records::construct::{enum_from_value, new_record}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -251,19 +252,10 @@ pub(crate) fn dynamic_contract_proofs<'py>( /// Declared dynamic families as native request rows. pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult> { - let mut rows: Vec = Vec::new(); - for family in families.try_iter()? { - let family: Bound<'_, PyAny> = family?; - rows.push(FamilyRow { - name: native_field(&family, "name")?.extract()?, - pivot_column: native_field(&family, "pivot_column")?.extract()?, - value_column: native_field(&family, "value_column")?.extract()?, - aggregate: native_field(&family, "aggregate")?.extract()?, - data_type: native_field(&family, "type")?.extract()?, - name_pattern: native_field(&family, "name_pattern")?.extract()?, - }); - } - Ok(rows) + Ok(family_specs(families)? + .into_iter() + .map(FamilyRow::from) + .collect()) } fn or_empty<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs index a721e0b06b..cad5b27291 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/argument_values.rs @@ -1,11 +1,42 @@ //! Python audit argument values read as plain argument values. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_object}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; use sqlbuild_attachments::audits::models::ArgumentValue; +use sqlbuild_model_config::config_values::models::ConfigNode; + +/// Native-built argument containers need no intermediate Python list or tuple. +pub(crate) fn config_argument_value( + py: Python<'_>, + value: &PyConfigNode, +) -> PyResult { + Ok(match value { + ConfigNode::Null => ArgumentValue::Null, + ConfigNode::Bool(value) => ArgumentValue::Boolean(*value), + ConfigNode::Int(value) => ArgumentValue::Number(value.to_string()), + ConfigNode::Str(value) => ArgumentValue::Text(value.clone()), + ConfigNode::List(items) => ArgumentValue::List( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Tuple(items) => ArgumentValue::Tuple( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Map(_) => ArgumentValue::Opaque, + ConfigNode::Float(_) | ConfigNode::Host(_) => { + return argument_value(&config_object(py, value)?); + } + }) +} /// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. pub(crate) fn argument_pairs( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 6e73eb8d79..642ba9517f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -39,7 +39,9 @@ use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use crate::bindings::_helpers::sqltext::lexical_syntax::optional_record_syntax; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::HashMap; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; @@ -52,7 +54,7 @@ pub(crate) struct AnalysisContext<'py> { pub(crate) column_types: Bound<'py, PyAny>, pub(crate) nullability: Bound<'py, PyAny>, pub(crate) complete_schemas: Bound<'py, PyAny>, - pub(crate) dynamic_families: Bound<'py, PyAny>, + pub(crate) dynamic_families: Vec<(String, Vec)>, /// The model analyses, or `None` when SQL analysis does not run. pub(crate) analyses: Option>, /// The native rows of `column_types`, `nullability` and `complete_schemas`. @@ -114,7 +116,7 @@ pub(crate) fn analysis_context<'py>( nullability.into_any(), complete_schemas.into_any(), ); - let dynamic_families: Bound<'py, PyAny> = dynamic_families_by_table(inputs)?.into_any(); + let dynamic_families = dynamic_families_by_table(inputs)?; let profile: Bound<'py, PyAny> = profile_with_catalog(inputs, request.inference_profile.bind(py), &column_types)?; let mut context: AnalysisContext<'py> = AnalysisContext { @@ -141,10 +143,13 @@ pub(crate) fn analysis_context<'py>( } /// `_build_dynamic_families_by_table`: enforced contract models' dynamic column families. -fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { +fn dynamic_families_by_table<'py>( + inputs: &Bound<'py, PyAny>, +) -> PyResult)>> { let py: Python<'py> = inputs.py(); let enforced: Bound<'py, PyAny> = enum_member(py, RecordId::ContractPolicy, "ENFORCED")?; - let families: Bound<'py, PyDict> = PyDict::new(py); + let mut families: Vec<(String, Vec)> = Vec::new(); + let mut positions: HashMap = HashMap::new(); for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let contract: Bound<'py, PyAny> = native_field(&model_input, "config")? @@ -156,8 +161,13 @@ fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( column_types: context.column_types.clone(), nullability: context.nullability.clone(), complete_schemas: context.complete_schemas.clone(), - dynamic_families: context.dynamic_families.clone(), + dynamic_families: &context.dynamic_families, rich_type_inference: rich, cache, declared: Some(&context.declared), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 60ae289112..dc0fcec4b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -2,18 +2,23 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use crate::bindings::_helpers::analysis_session::stage_functions::dynamic_contract_proofs; +use crate::bindings::_helpers::analysis_session::profile_inputs::dialect_or_generic; +use crate::bindings::_helpers::analysis_session::session::{ + FamilyRow, NativeModelAnalysisSession, PivotRequestRow, prove_dynamic_column_contracts, +}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::stage_work::{ PIVOT_PROOFS, SESSION_PIVOT_PROOFS, count_stage, }; +use crate::bindings::_helpers::model_config::header_families::HeaderFamilies; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; -use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; +use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_rows; use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::IntoPyObjectExt; @@ -47,13 +52,7 @@ pub(crate) fn native_project_facts<'py>( &syntax_checks, ) .and_then(|value| value.into_bound_py_any(py))?; - let proofs: Bound<'py, PyDict> = pivot_proofs( - &standalone_pivot_models(&model_inputs.clone(), &positions) - .and_then(|value| value.into_bound_py_any(py))? - .cast_into::()?, - context, - session, - )?; + let proofs = pivot_proofs(&model_inputs, &positions, context, session)?; let deps: Bound<'py, PyAny> = native_field(&resources, "model_deps")?; let syntax_valid: Bound<'py, PyAny> = native_field(&resources, "model_syntax_valid")?; let mut models: Vec> = Vec::new(); @@ -87,6 +86,14 @@ pub(crate) fn native_project_facts<'py>( /// The declared relation facts standalone pivot proofs read. pub(crate) fn pivot_tables<'py>(context: &AnalysisContext<'py>) -> PyResult> { let py: Python<'py> = context.profile.py(); + let families = PyDict::new(py); + let class = record_class(py, RecordId::SchemaDynamicColumnFamily)?; + for (name, specs) in &context.dynamic_families { + families.set_item( + name, + HeaderFamilies::new(&class, specs.clone())?.project(py)?, + )?; + } new_record_with( py, RecordId::NativePivotTables, @@ -101,41 +108,73 @@ pub(crate) fn pivot_tables<'py>(context: &AnalysisContext<'py>) -> PyResult( - pivots: &Bound<'py, PyDict>, + model_inputs: &Bound<'py, PyAny>, + positions: &Bound<'py, PyAny>, context: &AnalysisContext<'py>, session: &Bound<'py, PyAny>, ) -> PyResult> { - let tables: Bound<'py, PyAny> = pivot_tables(context)?; - let py: Python<'py> = pivots.py(); - let proven: Vec> = dynamic_contract_proofs( - session, - &tables, - &PyTuple::new(py, pivots.values())?.into_any(), - )? - .iter() - .collect(); + let py = model_inputs.py(); + let pivots = standalone_pivot_rows(model_inputs, positions)?; + let indexes: Vec<_> = pivots.iter().map(|(index, _)| *index).collect(); + let models = pivots + .into_iter() + .map(|(_, (sql, families))| (sql, families.into_iter().map(FamilyRow::from).collect())) + .collect::>(); + let contracts = if models.is_empty() { + Vec::new() + } else if session.is_none() { + prove_dynamic_column_contracts( + py, + PivotRequestRow { + dialect: dialect_or_generic(&native_field( + &context.profile, + "sql_analysis_dialect", + )?)?, + column_types: context.declared.types.rows().clone(), + authoritative_types: context.declared.complete.rows().clone(), + column_nullability: context.declared.nullability.rows().clone(), + families_by_table: context + .dynamic_families + .iter() + .map(|(name, families)| { + ( + name.clone(), + families.iter().cloned().map(FamilyRow::from).collect(), + ) + }) + .collect(), + models, + }, + )? + } else { + session + .cast::()? + .borrow() + .prove_dynamic_contracts(py, models)? + }; + let proven = contracts + .into_iter() + .map(|(_, proof)| proof.into_bound_py_any(py)) + .collect::>>()?; count_stage(py, PIVOT_PROOFS, proven.len() as u64)?; if !session.is_none() { let session_proofs: usize = proven.iter().filter(|proof| !proof.is_none()).count(); count_stage(py, SESSION_PIVOT_PROOFS, session_proofs as u64)?; } - if proven.len() != pivots.len() { + if proven.len() != indexes.len() { return Err(PyValueError::new_err( "native pivot proofs do not match the standalone pivot models", )); } let proofs: Bound<'py, PyDict> = PyDict::new(py); - for (index, proof) in pivots.keys().iter().zip(proven) { + for (index, proof) in indexes.into_iter().zip(proven) { proofs.set_item(index, proof)?; } Ok(proofs) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index d8f52aed64..bb68f5bba6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -1,5 +1,7 @@ //! Unit-test selector splitting, analysis and rule selection, and static command compilation. +use std::sync::Arc; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyFrozenSet; use pyo3::{pyfunction, wrap_pyfunction}; @@ -8,10 +10,10 @@ use crate::bindings::_helpers::boundary::python_calls::{imported, keywords}; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; use crate::bindings::_helpers::records::construct::{new_record, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_values; use crate::bindings::_helpers::records::ids::RecordId; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -85,9 +87,9 @@ fn planner_scopes<'py>( ) } -/// The stale-warning scope's models of `scopes`. -fn scoped_models<'py>(scopes: &Bound<'py, PyAny>) -> PyResult> { - native_field(scopes, "stale_warning_scope")?.getattr("models_by_name") +/// The native scope whose models receive deep analysis. +fn analysis_scope(scopes: &Bound<'_, PyAny>) -> PyResult> { + scope_data(&native_field(scopes, "stale_warning_scope")?) } /// Return options whose selectors name only resources, with unit tests selected apart. @@ -119,10 +121,11 @@ pub(crate) fn resolve_configured_rule_selection<'py>( if selection.is_none() || unselective(&selection)? { return Ok(py.None().into_bound(py)); } - let models: Bound<'py, PyAny> = scoped_models(&planner_scopes(project, &selection)?)?; - let keys: Vec> = mapping_values(&models)? - .try_iter()? - .map(|model| native_field(&model?, "key")) + let scope = analysis_scope(&planner_scopes(project, &selection)?)?; + let keys: Vec> = scope + .models + .values() + .map(|model| native_field(&model.bind(py), "key")) .collect::>()?; Ok(PyFrozenSet::new(py, keys)?.into_any()) } @@ -154,9 +157,8 @@ pub(crate) fn resolve_analysis_model_names<'py>( ], )?), )?; - let models: Bound<'py, PyAny> = - scoped_models(&planner_scopes(&structural_project, selection)?)?; - Ok(PyFrozenSet::new(py, models.try_iter()?.collect::>>()?)?.into_any()) + let scope = analysis_scope(&planner_scopes(&structural_project, selection)?)?; + Ok(PyFrozenSet::new(py, scope.models.keys())?.into_any()) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 2bbd812f1a..441e7d723d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -1,5 +1,6 @@ //! A compiled model, read from the native compiled project that owns it. +use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; use sqlbuild_project::models as project_models; use sqlbuild_project::models::{DestinationFact, ModelMetadataTexts}; use sqlbuild_project::types::InferredRow; @@ -7,7 +8,7 @@ use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{FromPyObject, pyclass, pymethods}; use sqlbuild_analysis::assembly::analysis_session::main::contract_proof_json::contract_proof_json; @@ -60,7 +61,7 @@ pub(crate) struct SqlRuleKey { } /// A schema's entry audits, column specs and each audited column's index and name. -pub(crate) type AuditedSchema = (Py, Arc, Vec<(usize, String)>); +pub(crate) type AuditedSchema = (Arc, Arc, Vec<(usize, String)>); /// `(name, type)` of a declared column. pub(crate) type DeclaredRow = (String, Option); @@ -367,7 +368,7 @@ impl NativeCompiledModel { .map(|(index, column)| (index, column.name.clone())) .collect(); Ok(Some(( - schema.host.audits.clone_ref(py), + schema.host.audits.clone(), schema.host.columns.clone(), audited, ))) @@ -665,11 +666,11 @@ impl NativeCompiledModel { /// Whether the attached schema declares dynamic columns. #[getter(has_dynamic_columns)] pub(crate) fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, py| { + self.read(py, |record, _py| { Ok(record .schema .as_ref() - .is_some_and(|schema| !schema.host.dynamic_columns.bind(py).is_empty())) + .is_some_and(|schema| !schema.host.dynamic_columns.specs.is_empty())) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index 2c801e2ef9..3396a1a71d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -21,9 +21,12 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::{ use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; use crate::bindings::_helpers::compiled_views::relation_keys::{RelationKeyRow, relation_key_row}; use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; +use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; +use crate::bindings::_helpers::model_config::header_families::{HeaderFamilies, family_specs}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_tuple}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; const MODEL_RELATION_KIND: &str = "ref"; @@ -49,8 +52,8 @@ pub(crate) type ModelInputRecord = project_models::ModelInputRecord, - pub(crate) audits: Py, - pub(crate) dynamic_columns: Py, + pub(crate) audits: Arc, + pub(crate) dynamic_columns: Arc, } /// A model input's user values and the attribute projections its views build. @@ -411,10 +414,9 @@ pub(crate) fn model_relation_key_row( ) -> PyResult { let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), "values")?; let declared = match (&record.schema, owner.input_schema_fact(record)) { - (Some(schema), Some(fact)) => Some(( - fact.columns.as_slice(), - schema.host.audits.bind(py).as_any(), - )), + (Some(schema), Some(fact)) => { + Some((fact.columns.as_slice(), schema.host.audits.key_rows(py)?)) + } _ => None, }; relation_key_row( @@ -450,8 +452,8 @@ pub(crate) fn schema_entry( fields.set_item("description", &schema.description)?; fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; fields.set_item("columns", schema.host.columns.project(py)?)?; - fields.set_item("dynamic_columns", schema.host.dynamic_columns.bind(py))?; - fields.set_item("audits", schema.host.audits.bind(py))?; + fields.set_item("dynamic_columns", schema.host.dynamic_columns.project(py)?)?; + fields.set_item("audits", schema.host.audits.project(py)?)?; Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) }) .map(|entry| entry.clone_ref(py)) @@ -484,16 +486,16 @@ fn expansion_object( } /// A model input's attached schema: its name and dynamic column families. -pub(crate) struct AttachedSchema<'py> { +pub(crate) struct AttachedSchema { pub(crate) name: String, - pub(crate) dynamic_columns: Bound<'py, PyAny>, + pub(crate) dynamic_columns: Vec, pub(crate) type_enforcement: bool, } /// The attached schema of `model_input`; a native input answers without building its entry. pub(crate) fn attached_schema<'py>( model_input: &Bound<'py, PyAny>, -) -> PyResult>> { +) -> PyResult> { let py: Python<'py> = model_input.py(); if let Ok(native) = model_input.cast::() { return native.get().read(py, |record, py| { @@ -502,7 +504,7 @@ pub(crate) fn attached_schema<'py>( }; Ok(Some(AttachedSchema { name: record.name.clone(), - dynamic_columns: schema.host.dynamic_columns.bind(py).clone().into_any(), + dynamic_columns: schema.host.dynamic_columns.read(py)?, type_enforcement: schema.type_enforcement, })) }); @@ -513,7 +515,7 @@ pub(crate) fn attached_schema<'py>( } Ok(Some(AttachedSchema { name: native_field(&entry, "name")?.extract()?, - dynamic_columns: native_field(&entry, "dynamic_columns")?, + dynamic_columns: family_specs(&native_field(&entry, "dynamic_columns")?)?, type_enforcement: native_field(&entry, "type_enforcement")?.is_truthy()?, })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs index 4622ba1f67..8967d13c1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs @@ -32,6 +32,9 @@ const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; const RELATION_KIND_SEED: &str = "seed"; const RELATION_KIND_SOURCE: &str = "source"; +/// Native declared columns and model audits used to derive relation keys. +type DeclaredKeyFacts<'a> = (&'a [SchemaColumnFact], Vec<(String, Option)>); + positional_record! { extract, into; /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. @@ -53,7 +56,7 @@ pub(crate) fn relation_key_row( kind: &str, name: String, values: &Bound<'_, PyDict>, - declared: Option<(&[SchemaColumnFact], &Bound<'_, PyAny>)>, + declared: Option>, ) -> PyResult { let mut columns: Vec<(String, Vec, bool)> = Vec::new(); let mut audits: Vec<(String, Option)> = Vec::new(); @@ -65,7 +68,7 @@ pub(crate) fn relation_key_row( column.nullable == Some(false), )); } - audits = audit_key_rows(declared_audits)?; + audits = declared_audits; } Ok(RelationKeyRow { kind: kind.to_owned(), @@ -180,7 +183,12 @@ pub(crate) fn entry_relation_key_row( let columns: Vec = python_column_facts(&declared_specs(entry, "columns")?, None)?; let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; - relation_key_row(kind, name, values, Some((&columns, &audits))) + relation_key_row( + kind, + name, + values, + Some((&columns, audit_key_rows(&audits)?)), + ) } /// `getattr(entry, name, None) or ()`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs index fef6bdadc4..39c57fe4c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs @@ -1,36 +1,44 @@ //! A compiled resource's attributes read alike for models, functions, seeds and sources. use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::compiled_model_view::{ + NativeCompiledModel, model_graph_row, +}; use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; -/// A resource's `deps` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn deps<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); +/// A resource's graph facts without constructing its public keys or dependency tuple. +pub(crate) fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { + let py = resource.py(); if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view.deps(py).and_then(|value| value.into_bound_py_any(py)); + return model_graph_row(view.get(), py); } if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view.deps(py).and_then(|value| value.into_bound_py_any(py)); + return view.get().graph_row(py); } if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view.deps(py).into_bound_py_any(py); + return view.get().graph_row(py); } if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSource = view.get(); - return view.deps(py).into_bound_py_any(py); + return view.get().graph_row(py); } count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "deps") + Ok(ResourceRow { + key: native_key(&native_field(resource, "key")?)?, + deps: native_field(resource, "deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + tags: Vec::new(), + folder: None, + }) } /// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs index b2e2363db6..230e469567 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs @@ -1,16 +1,18 @@ //! `CompiledObjectKey`s for native `(resource type, name)` pairs, converted once per pair. +use std::cell::RefCell; +use std::collections::HashMap; + use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; /// One `CompiledObjectKey` per native pair, so shared keys convert once. pub(crate) struct KeyCache<'py> { key_type: Bound<'py, PyAny>, resource_type: Bound<'py, PyAny>, - keys: Bound<'py, PyDict>, + keys: RefCell>>, } impl<'py> KeyCache<'py> { @@ -18,12 +20,12 @@ impl<'py> KeyCache<'py> { Ok(Self { key_type: record_class(py, RecordId::CompiledObjectKey)?, resource_type: record_class(py, RecordId::CompiledResourceType)?, - keys: PyDict::new(py), + keys: RefCell::new(HashMap::new()), }) } pub(crate) fn py(&self) -> Python<'py> { - self.keys.py() + self.key_type.py() } /// The `CompiledObjectKey` for one `(resource type, name)` pair. @@ -31,15 +33,16 @@ impl<'py> KeyCache<'py> { &self, (resource_type, name): &(String, String), ) -> PyResult> { - let pair = (resource_type.as_str(), name.as_str()); - if let Some(existing) = self.keys.get_item(pair)? { - return Ok(existing); + let pair = (resource_type.clone(), name.clone()); + if let Some(existing) = self.keys.borrow().get(&pair) { + return Ok(existing.bind(self.py()).clone()); } - let keywords: Bound<'py, PyDict> = PyDict::new(self.keys.py()); - keywords.set_item("resource_type", self.resource_type.call1((resource_type,))?)?; - keywords.set_item("name", name)?; - let created: Bound<'py, PyAny> = self.key_type.call((), Some(&keywords))?; - self.keys.set_item(pair, &created)?; + let created: Bound<'py, PyAny> = self + .key_type + .call1((self.resource_type.call1((resource_type,))?, name))?; + self.keys + .borrow_mut() + .insert(pair, created.clone().unbind()); Ok(created) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index c4e9323075..0ef2383d5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -22,7 +22,6 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::attachments::argument_values::argument_pairs; use crate::bindings::_helpers::compile_context::delta::record_source_reads; use crate::bindings::_helpers::compiled_project::attachment_inputs::{ AuditBlockRow, AuditFileRow, AuditSettings, @@ -40,11 +39,11 @@ use crate::bindings::_helpers::input_loops::source_loop::{ AuthoredExpansion, expanded_authored_sql, file_declarations, }; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::header_audits::{AuditInstance, HeaderAudits}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::model_config::header_metadata::thresholds; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::result_output::integration_values::json_value; use sqlbuild_project::models::AttachedTarget; use sqlbuild_project::models::ReferenceRow; @@ -81,7 +80,7 @@ pub(crate) struct AttachmentOwner<'py> { pub(crate) relation: String, pub(crate) path: String, /// `(owner file, column, instance)` in attachment order. - pub(crate) instances: Vec<(String, Option, Bound<'py, PyAny>)>, + pub(crate) instances: Vec<(String, Option, AuditInstance<'py>)>, } /// Every singular audit, then each owner's attached audits, and the built-in shadow warnings. @@ -342,12 +341,11 @@ impl<'py> AuditScope<'_, 'py> { } /// The instance's `meta` as JSON, or `None`; a value JSON cannot carry fails the compile. - fn meta(&self, instance: &Bound<'py, PyAny>, label: &str) -> PyResult> { - let meta: Bound<'py, PyAny> = native_field(instance, "meta")?; - if meta.is_none() { + fn meta(&self, instance: &AuditInstance<'py>, label: &str) -> PyResult> { + if !instance.meta_present()? { return Ok(None); } - match json_value(&meta)? { + match instance.meta()? { Some(value) => Ok(Some(value)), None => { let message: String = format!("{label}: 'meta' must be a mapping of JSON values"); @@ -362,11 +360,11 @@ impl<'py> AuditScope<'_, 'py> { definitions: &HashMap>, owner: &AttachmentOwner<'py>, attachment: (&str, Option<&str>), - instance: &Bound<'py, PyAny>, + instance: &AuditInstance<'py>, ) -> PyResult { let py: Python<'py> = self.context.host.py(); let (owner_file, column) = attachment; - let definition_name: String = native_field(instance, "definition_name")?.extract()?; + let definition_name: String = instance.field("definition_name")?.extract()?; let Some(definition) = definitions.get(&definition_name) else { let message: String = format!("{owner_file} references unknown generic audit '{definition_name}'"); @@ -383,9 +381,9 @@ impl<'py> AuditScope<'_, 'py> { &consumer, (owner.kind, &owner.name, &owner.path), )?; - let thresholds: Bound<'py, PyAny> = native_field(instance, "thresholds")?; - let minimum_samples: Bound<'py, PyAny> = native_field(instance, "minimum_samples")?; - let instance_severity: Option = native_field(instance, "severity")?.extract()?; + let (has_thresholds, threshold_error) = instance.threshold_policy()?; + let minimum_samples = instance.field("minimum_samples")?; + let instance_severity: Option = instance.field("severity")?.extract()?; let label: String = format!("{owner_file} audit '{definition_name}'"); let measurement: bool = definition.block.measurement; if measurement { @@ -396,13 +394,13 @@ impl<'py> AuditScope<'_, 'py> { ); return Err(compile_error(self.context.classes, &message, None)); } - if thresholds.is_none() { + if !has_thresholds { let message: String = format!( "{label}: measurement audit attachments must define at least one threshold" ); return Err(compile_error(self.context.classes, &message, None)); } - } else if !thresholds.is_none() || !minimum_samples.is_none() { + } else if has_thresholds || !minimum_samples.is_none() { let message: String = format!( "{label}: thresholds and minimum_samples are only valid for measurement audits" ); @@ -422,13 +420,12 @@ impl<'py> AuditScope<'_, 'py> { sql_body: definition.block.sql_body.clone(), evidence_sql: definition.block.evidence_sql.clone(), implicit_arguments: implicit, - explicit_arguments: argument_pairs(native_field(instance, "arguments")?.downcast()?)?, - has_thresholds: !thresholds.is_none(), - threshold_error: !thresholds.is_none() - && !native_field(&thresholds, "error")?.is_none(), + explicit_arguments: instance.arguments()?, + has_thresholds, + threshold_error, instance_severity, default_severity: self.settings.default_severity.clone(), - instance_run_scope: native_field(instance, "run_scope")?.extract()?, + instance_run_scope: instance.field("run_scope")?.extract()?, default_run_scope: self.settings.default_run_scope.clone(), }); let rendered = match rendering { @@ -454,7 +451,7 @@ impl<'py> AuditScope<'_, 'py> { .policies .map_err(|message| compile_error(self.context.classes, &message, None))?; let run_scope: String = match source { - PolicySource::Instance => native_field(instance, "run_scope")?.extract()?, + PolicySource::Instance => instance.field("run_scope")?.extract()?, PolicySource::Default => self.settings.default_run_scope.clone().unwrap_or_default(), PolicySource::Fallback => "delta_and_final".to_owned(), }; @@ -464,11 +461,11 @@ impl<'py> AuditScope<'_, 'py> { return Err(compile_error(self.context.classes, &message, None)); } usages.extend(expanded.usages); - let evidence_limit: Py = native_field(instance, "evidence_limit")?.unbind(); - let always_run: bool = native_field(instance, ALWAYS_RUN_KEY)?.is_truthy()?; + let evidence_limit: Py = instance.field("evidence_limit")?.unbind(); + let always_run: bool = instance.field(ALWAYS_RUN_KEY)?.is_truthy()?; let usages: Py = PyTuple::new(py, usages)?.unbind(); - let name: Option = native_field(instance, "name")?.extract()?; - let description: Option = native_field(instance, "description")?.extract()?; + let name: Option = instance.field("name")?.extract()?; + let description: Option = instance.field("description")?.extract()?; let meta: Option = self.meta(instance, &label)?; Ok(AttachedAudit { definition_name: definition @@ -494,7 +491,7 @@ impl<'py> AuditScope<'_, 'py> { host: AuditHandles { audit_file: definition.file.file.clone_ref(py), audit_block: definition.block.block.clone_ref(py), - thresholds: thresholds.unbind(), + thresholds: instance.field("thresholds")?.unbind(), minimum_samples: minimum_samples.unbind(), evidence_limit, usages, @@ -703,13 +700,14 @@ fn header_text(header: &Bound<'_, PyDict>, key: &str) -> PyResult /// The attachment owners of every model, source and seed, as the audit loop reads them. pub(crate) fn model_owner<'py>( + py: Python<'py>, name: String, path: String, - schema: (&Bound<'py, PyTuple>, (&HeaderColumns, &[SchemaColumnFact])), + schema: (&HeaderAudits, (&HeaderColumns, &[SchemaColumnFact])), ) -> PyResult> { let (audits, (columns, facts)) = schema; - let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); - for audit in audits.iter() { + let mut instances: Vec<(String, Option, AuditInstance<'py>)> = Vec::new(); + for audit in audits.instances(py) { instances.push((path.clone(), explicit_column(&audit)?, audit)); } for (index, fact) in facts.iter().enumerate() { @@ -717,11 +715,7 @@ pub(crate) fn model_owner<'py>( continue; } let column_owner: &str = fact.location_path.as_deref().unwrap_or(&path); - for audit in columns.specs[index] - .field(audits.py(), "audits")? - .try_iter()? - { - let audit: Bound<'py, PyAny> = audit?; + for audit in columns.specs[index].audit_instances(py)? { let owner_file: String = location_path(&audit)?.unwrap_or_else(|| column_owner.to_owned()); instances.push((owner_file, Some(fact.name.clone()), audit)); @@ -745,9 +739,9 @@ pub(crate) fn resource_owner<'py>( ) -> PyResult> { let (path, owner_file) = paths; let (audits, columns) = schema; - let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); + let mut instances: Vec<(String, Option, AuditInstance<'py>)> = Vec::new(); for audit in audits.try_iter()? { - let audit: Bound<'py, PyAny> = audit?; + let audit = AuditInstance::Python(audit?); let column: Option = if kind == SEED_TARGET { explicit_column(&audit)? } else { @@ -759,7 +753,11 @@ pub(crate) fn resource_owner<'py>( let column: Bound<'py, PyAny> = column?; let column_name: String = native_field(&column, "name")?.extract()?; for audit in native_field(&column, "audits")?.try_iter()? { - instances.push((owner_file.clone(), Some(column_name.clone()), audit?)); + instances.push(( + owner_file.clone(), + Some(column_name.clone()), + AuditInstance::Python(audit?), + )); } } let function: &str = if kind == SOURCE_TARGET { @@ -777,14 +775,17 @@ pub(crate) fn resource_owner<'py>( } /// `_explicit_audit_column_name`: the instance's `column` argument when it is a string. -fn explicit_column(audit: &Bound<'_, PyAny>) -> PyResult> { - let arguments: Bound<'_, PyDict> = native_field(audit, "arguments")?.downcast_into()?; - header_text(&arguments, COLUMN_ARGUMENT) +fn explicit_column(audit: &AuditInstance<'_>) -> PyResult> { + audit + .argument(COLUMN_ARGUMENT)? + .filter(|value| value.is_instance_of::()) + .map(|value| value.extract()) + .transpose() } /// The path of a spec's authored location, if it has one. -fn location_path(spec: &Bound<'_, PyAny>) -> PyResult> { - let location: Bound<'_, PyAny> = native_field(spec, "location")?; +fn location_path(spec: &AuditInstance<'_>) -> PyResult> { + let location = spec.field("location")?; if location.is_none() { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 78ef4e81d3..b62cb5e827 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -391,12 +391,8 @@ impl NativeModelLoop { local: (enums.as_any(), constants.as_any()), }, )?; - let config: Bound<'py, PyAny> = configs.effective( - &context, - &record.header_value_dict(py)?, - (name, relative_path), - row.header.as_ref(), - )?; + let config: Bound<'py, PyAny> = + configs.effective(&context, record, (name, relative_path), row.header.as_ref())?; let model: LoopModel<'_, 'py> = LoopModel { row: &row, config, @@ -784,6 +780,7 @@ impl NativeModelLoop { let source: HeaderSource<'_, 'py> = HeaderSource { name, values: &values, + config: &config, model_file: model_file.get(), relative_path, visible: (&visible.declarations, &visible.consumer), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs index ecd4af2157..2cb2ebd0e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs @@ -6,11 +6,14 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_discovery::model_files::main::config_value::config_value; use sqlbuild_discovery::model_files::models::HeaderFacts; use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; use sqlbuild_model_config::config_presence::main::contains_template::contains_template; use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_config_value; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, config_error}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_classes::{ConfigClasses, config_fields, view}; @@ -48,18 +51,22 @@ impl<'py> ModelConfigs<'py> { pub(crate) fn effective( &self, context: &LoopContext<'_, 'py>, - header: &Bound<'py, PyDict>, + model_file: &NativeModelFile, model: (&str, &str), facts: Option<&HeaderFacts>, ) -> PyResult> { let (name, relative_path) = model; - let py: Python<'py> = header.py(); + let py: Python<'py> = context.host.py(); + let header = model_file.header_value_dict(py)?; + let authored = model_file + .native_header_values() + .map(|values| values.as_ref()); let classes: &ConfigClasses<'py> = &context.classes.config; let builder: &NativeModelConfigBuilder = classes.builder.get(); let matched: Option = builder .selected_path_default(py, relative_path)? .map_err(|error| config_error(error.bind(py).as_any(), None))?; - let metadata: Option> = reusable_metadata(header, facts)?; + let metadata: Option> = reusable_metadata(&header, facts)?; let reusable: bool = match metadata { Some(_) => self.reusable(context, &matched)?, None => false, @@ -70,11 +77,11 @@ impl<'py> ModelConfigs<'py> { { let values: Bound<'py, PyDict> = cached.values.copy()?; values.update(metadata.as_mapping())?; - let nodes = metadata_nodes(cached.nodes.as_ref(), metadata, metadata)?; + let nodes = metadata_nodes(cached.nodes.as_ref(), metadata, metadata, authored)?; return view(classes, &values, &cached.fields, nodes.as_ref()); } let built: ConfigBuild<'py> = - builder.built(py, (header, facts), matched.as_deref(), name)?; + builder.built(py, (&header, facts, authored), matched.as_deref(), name)?; let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { @@ -146,7 +153,7 @@ impl<'py> ModelConfigs<'py> { ReusedConfig { values: reusable_values, fields: fields.clone(), - nodes: metadata_nodes(built.nodes.as_ref(), metadata, &inherited.values)?, + nodes: metadata_nodes(built.nodes.as_ref(), metadata, &inherited.values, None)?, }, ); Ok(()) @@ -158,6 +165,7 @@ fn metadata_nodes( base: Option<&BuiltConfigValues>, metadata: &Bound<'_, PyDict>, replacements: &Bound<'_, PyDict>, + authored: Option<&[(String, AuthoredValue)]>, ) -> PyResult> { let Some(base) = base else { return Ok(None); @@ -170,7 +178,14 @@ fn metadata_nodes( let text = text.to_str()?; nodes.remove(text); if let Some(value) = replacements.get_item(&key)? { - nodes.insert(text.to_owned(), Arc::new(carried_node(&value)?)); + let native = authored + .and_then(|entries| entries.iter().rev().find(|(key, _)| key == text)) + .and_then(|(_, value)| config_value(value)); + let node = match native { + Some(node) => node, + None => carried_node(&value)?, + }; + nodes.insert(text.to_owned(), Arc::new(node)); } } Ok(Some(Arc::new(nodes))) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs index b9a8d2c3fb..fa29ec7fb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs @@ -6,12 +6,14 @@ use pyo3::BoundObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; use crate::bindings::_helpers::input_loops::model_loop::LoopContext; use crate::bindings::_helpers::input_loops::model_loop_schema::DeclaredColumns; +use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_node}; use crate::bindings::_helpers::model_config::header_metadata::{HeaderColumn, HeaderColumns}; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; const CASES_ATTRIBUTE: &str = "cases"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; @@ -20,7 +22,6 @@ const DEFINITION_ATTRIBUTE: &str = "definition"; const DEFINITION_NAME_FIELD: &str = "definition_name"; const ARGUMENTS_FIELD: &str = "arguments"; const NAME_FIELD: &str = "name"; -const TYPE_FIELD: &str = "type"; const AUDITS_FIELD: &str = "audits"; const MEMBERS_FIELD: &str = "members"; const VALUE_FIELD: &str = "value"; @@ -48,10 +49,6 @@ const COPIED_CASE_FIELDS: [&str; 9] = [ "evidence_limit", "meta", ]; -const PIVOT_COLUMN_FIELD: &str = "pivot_column"; -const VALUE_COLUMN_FIELD: &str = "value_column"; -const AGGREGATE_FIELD: &str = "aggregate"; -const NAME_PATTERN_FIELD: &str = "name_pattern"; /// A model's enum-typed columns resolved to scalar types, and the usages they generate. pub(crate) struct EnumColumns<'py> { @@ -118,42 +115,6 @@ fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { native_field(audit, DEFINITION_NAME_FIELD)?.extract() } -/// The parsed families as `SchemaDynamicColumnFamily` objects. -pub(crate) fn dynamic_family_objects<'py>( - py: Python<'py>, - context: &LoopContext<'_, 'py>, - families: &[DynamicFamily], -) -> PyResult>> { - families - .iter() - .map(|family| { - keywords( - &context.contracts.schema_dynamic_column_family, - &[ - (NAME_FIELD, family.name.as_str().into_pyobject_any(py)?), - ( - PIVOT_COLUMN_FIELD, - family.pivot_column.as_str().into_pyobject_any(py)?, - ), - ( - VALUE_COLUMN_FIELD, - family.value_column.as_str().into_pyobject_any(py)?, - ), - ( - AGGREGATE_FIELD, - family.aggregate.as_str().into_pyobject_any(py)?, - ), - (TYPE_FIELD, family.data_type.as_str().into_pyobject_any(py)?), - ( - NAME_PATTERN_FIELD, - family.name_pattern.as_deref().into_pyobject_any(py)?, - ), - ], - ) - }) - .collect() -} - /// `resolve_enum_contract_columns` and `_generated_enum_usages` for a header schema entry. pub(crate) fn enum_columns<'py>( context: &LoopContext<'_, 'py>, @@ -189,6 +150,47 @@ pub(crate) fn enum_columns<'py>( continue; }; resolved.set_item(&fact.name, &declaration)?; + if column.retains_audits() && context.contracts.uses_default_classes()? { + let mut generated = None; + if enforced { + let members = native_field(&declaration, MEMBERS_FIELD)? + .try_iter()? + .map(|member| config_node(&native_field(&member?, VALUE_FIELD)?)) + .collect::>>()?; + let mut present = false; + for audit in column.audit_instances(py)? { + if audit.accepts_enum(&members)? { + present = true; + break; + } + } + if !present { + generated = Some(ParsedAudit { + definition_name: ConfigNode::Str(ACCEPTED_VALUES_AUDIT.to_owned()), + arguments: vec![( + ConfigNode::Str(VALUES_ARGUMENT.to_owned()), + ConfigNode::Tuple(members), + )], + name: None, + description: None, + severity: None, + run_scope: None, + always_run: None, + thresholds: None, + minimum_samples: None, + evidence_limit: None, + meta: None, + }); + fact.audit_definitions + .push(ACCEPTED_VALUES_AUDIT.to_owned()); + } + } + let scalar_type = native_field(&declaration, SCALAR_TYPE_ATTRIBUTE)?; + fact.data_type = scalar_type.extract()?; + resolved_columns.push(column.enum_type(py, scalar_type, generated)?); + facts.push(fact); + continue; + } let mut audits: Vec> = column .field(py, AUDITS_FIELD)? .try_iter()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index cf3ba5e5b8..65e83ce79a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -3,17 +3,19 @@ use std::sync::Arc; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_analysis::compiled_project::models::{ DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, }; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::main::audit_factory_references::audit_factory_references; use sqlbuild_model_config::header_metadata::main::merged_audit_identities::merged_audit_identities; use sqlbuild_model_config::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns; use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; -use sqlbuild_model_config::header_metadata::models::{DynamicFamily, ParsedColumn}; +use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_model_config::model_validation::models::ColumnMigration; +use sqlbuild_model_config::types::NodeKind; use crate::bindings::_helpers::compile_driver::stage_work::{ NATIVE_HEADER_COLUMNS, PARSED_HEADER_COLUMNS, count_stage, @@ -24,19 +26,21 @@ use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, compile_error, config_error, }; use crate::bindings::_helpers::input_loops::model_loop_contracts::{ - EnumColumns, dynamic_family_objects, enum_columns, factory_audits, + EnumColumns, enum_columns, factory_audits, }; use crate::bindings::_helpers::input_loops::model_loop_inputs::present; use crate::bindings::_helpers::input_loops::model_loop_schema::{ DeclaredColumns, ModelSchema, merged_columns, schema_enums, }; -use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::header_audits::HeaderAudits; +use crate::bindings::_helpers::model_config::header_families::HeaderFamilies; use crate::bindings::_helpers::model_config::header_metadata::{ - HeaderColumn, HeaderColumns, NativeColumnSpec, audit_instances, native_schema_column, + HeaderColumn, HeaderColumns, HeaderValue, NativeColumnSpec, native_schema_column, parsed_audit_identity, }; +use crate::bindings::_helpers::model_config::typed_config::{config_node_of, typed_values}; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; @@ -52,6 +56,7 @@ const ENUMS_FIELD: &str = "enums"; pub(crate) struct HeaderSource<'a, 'py> { pub(crate) name: &'a str, pub(crate) values: &'a Bound<'py, PyDict>, + pub(crate) config: &'a Bound<'py, PyAny>, pub(crate) model_file: &'a NativeModelFile, pub(crate) relative_path: &'a str, /// The model's visible declarations and the identity its usages are recorded for. @@ -71,8 +76,8 @@ pub(crate) struct AttachedHeader<'py> { /// A header schema entry's Python contract objects, enum columns and retained fact. pub(crate) struct HeaderSchema<'py> { pub(crate) columns: Arc, - pub(crate) audits: Bound<'py, PyTuple>, - pub(crate) dynamic_columns: Bound<'py, PyTuple>, + pub(crate) audits: Arc, + pub(crate) dynamic_columns: Arc, pub(crate) type_enforcement: bool, pub(crate) enum_columns: Bound<'py, PyDict>, pub(crate) generated_usages: Vec>, @@ -106,8 +111,7 @@ pub(crate) fn header_schema<'py>( schema: None, }); } - let none: Bound<'py, PyAny> = py.None().into_bound(py); - let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); + let typed = typed_values(source.config)?; let model: (&str, &str) = (source.name, source.relative_path); let native: Option<&[NativeColumn]> = native_columns(context, source, columns.as_ref())?; if columns.is_some() { @@ -117,9 +121,17 @@ pub(crate) fn header_schema<'py>( }; count_stage(py, stage, 1)?; } - let python_columns: Option> = if native.is_some() { None } else { columns }; - let metadata = - parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); + let columns_node = if native.is_some() { + Arc::new(ConfigNode::Null) + } else { + config_node_of(&typed, COLUMNS_KEY)? + }; + let audits_node = config_node_of(&typed, AUDITS_KEY)?; + let metadata = parse_header_metadata( + &columns_node.as_ref(), + &audits_node.as_ref(), + source.relative_path, + ); let parsed_columns = metadata .columns .map_err(|error| raised_config_error(py, error))?; @@ -133,7 +145,7 @@ pub(crate) fn header_schema<'py>( .iter() .map(|column| native_schema_column(py, context.contracts, column, &locations)) .collect::>()?, - facts: local_column_facts(&parsed_columns)?, + facts: local_column_facts(py, &parsed_columns)?, } } }; @@ -146,24 +158,31 @@ pub(crate) fn header_schema<'py>( )?, None => local_columns, }; - let families = dynamic_families(py, &node(dynamic), model)?; + let dynamic_node = config_node_of(&typed, DYNAMIC_COLUMNS_KEY)?; + let families = parse_dynamic_columns(&dynamic_node.as_ref(), model) + .map_err(|error| raised_config_error(py, error))?; let parsed_audits = metadata .audits .map_err(|error| raised_config_error(py, error))?; let references: Vec = match factories { - Some(factories) => { + Some(_) => { let available: Vec = context.classes.audit_factories.keys().extract()?; - audit_factory_references(&PyNode(factories), model, &available) + let factories = config_node_of(&typed, AUDIT_FACTORIES_KEY)?; + audit_factory_references(&factories.as_ref(), model, &available) .map_err(|error| raised_config_error(py, error))? } None => Vec::new(), }; - let mut direct_audits: Vec> = - audit_instances(py, context.contracts, &parsed_audits)?; + let mut audits = HeaderAudits::new( + py, + context.contracts, + &parsed_audits, + &py.None().into_bound(py), + )?; let (generated, generated_identities) = factory_audits(py, context, &references)?; let direct_identities: Vec = parsed_audits .iter() - .map(parsed_audit_identity) + .map(|audit| parsed_audit_identity(py, audit)) .collect::>()?; merged_audit_identities(&direct_identities, &generated_identities, model) .map_err(|error| raised_config_error(py, error))?; @@ -185,9 +204,12 @@ pub(crate) fn header_schema<'py>( .iter() .any(|column| column.data_type.is_some()); let family_facts: Vec = families.iter().map(family_fact).collect(); - direct_audits.extend(generated); - let audits: Bound<'py, PyTuple> = PyTuple::new(py, direct_audits)?; - let families: Vec> = dynamic_family_objects(py, context, &families)?; + audits.extend(generated); + let audits = Arc::new(audits); + let families = Arc::new(HeaderFamilies::new( + &context.contracts.schema_dynamic_column_family, + families, + )?); let enums: Bound<'py, PyDict> = native_field(source.visible.0, ENUMS_FIELD)?.downcast_into()?; let enums: Bound<'py, PyDict> = match source.schema { Some(schema) => { @@ -211,7 +233,7 @@ pub(crate) fn header_schema<'py>( schema: Some(HeaderSchema { columns: resolved.columns, audits, - dynamic_columns: PyTuple::new(py, families)?, + dynamic_columns: families, type_enforcement, enum_columns: resolved.declarations, generated_usages: resolved.usages, @@ -237,9 +259,10 @@ fn native_columns<'a>( return Ok(None); }; let header: Bound<'_, PyDict> = source.model_file.header_value_dict(columns.py())?; - let own: bool = header - .get_item(COLUMNS_KEY)? - .is_some_and(|value| value.is(columns)); + let own: bool = match header.get_item(COLUMNS_KEY)? { + Some(value) => value.is(columns) || value.eq(columns)?, + None => false, + }; Ok(own.then_some(parsed.as_slice())) } @@ -283,15 +306,6 @@ fn native_declared_columns( Ok(DeclaredColumns { specs, facts }) } -/// The header's dynamic column families, each `name_pattern` compiled natively. -fn dynamic_families( - py: Python<'_>, - raw: &PyNode<'_>, - model: (&str, &str), -) -> PyResult> { - parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, error)) -} - /// The header description: absent, or a non-blank string by Python's `str.strip`. fn description( context: &LoopContext<'_, '_>, @@ -323,26 +337,28 @@ fn raised_config_error( } /// The facts of the header's own columns, which are declared in the model file. -fn local_column_facts(parsed: &[ParsedColumn>]) -> PyResult> { +fn local_column_facts( + py: Python<'_>, + parsed: &[ParsedColumn], +) -> PyResult> { let mut facts: Vec = Vec::with_capacity(parsed.len()); for column in parsed { - let text = |node: Option<&PyNode<'_>>| -> PyResult> { - node.map(|node| node.0.extract()).transpose() + let text = |node: Option<&N>| -> PyResult> { + node.map(|node| node.exact_text(py)).transpose() }; facts.push(SchemaColumnFact { - name: column.name.0.str()?.extract()?, + name: column.name.exact_text(py)?, data_type: text(column.column_type.as_ref())?, nullable: column .nullable .as_ref() - .map(|node| node.0.extract()) - .transpose()?, + .map(|node| node.kind() == NodeKind::Bool(true)), described: column.description.is_some(), migrate_from: text(column.migrate_from.as_ref())?, audit_definitions: column .audits .iter() - .map(|audit| audit.definition_name.0.extract()) + .map(|audit| audit.definition_name.exact_text(py)) .collect::>()?, location_path: None, }); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index 2b141d0568..a8816208d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -237,8 +237,8 @@ pub(crate) fn finished_record<'py>( model_schema: schema.model_schema.clone(), host: SchemaHandles { columns: schema.columns.clone(), - audits: schema.audits.clone().unbind(), - dynamic_columns: schema.dynamic_columns.clone().unbind(), + audits: schema.audits.clone(), + dynamic_columns: schema.dynamic_columns.clone(), }, }); let authored: Arc = authored_query_sql.clone(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index 95469078fa..cea062d742 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -146,10 +146,11 @@ fn attachment_owners<'py>( continue; }; owners.push(model_owner( + py, record.name.clone(), record.host.model_file.get().relative_text(py)?, ( - schema.host.audits.bind(py), + schema.host.audits.as_ref(), (schema.host.columns.as_ref(), &fact.columns), ), )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs new file mode 100644 index 0000000000..2136827d23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_audits.rs @@ -0,0 +1,445 @@ +//! Retained audit contracts; Python-authored factory results remain opaque values. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::types::{PyString, PyStringMethods, PyTuple}; +use sqlbuild_attachments::audits::models::ArgumentValue; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_discovery::model_files::main::config_value::config_value; +use sqlbuild_discovery::model_files::models::NativeAudit; +use sqlbuild_model_config::config_values::models::ConfigNode; +use sqlbuild_model_config::header_metadata::constants::{ + ACCEPTED_VALUES_ARGUMENT, ACCEPTED_VALUES_AUDIT, +}; +use sqlbuild_model_config::header_metadata::models::ParsedAudit; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use std::sync::Arc; + +use crate::bindings::_helpers::attachments::argument_values::{ + argument_pairs, config_argument_value, +}; +use crate::bindings::_helpers::model_config::config_nodes::{ + PyConfigNode, config_node, config_object, +}; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; +use crate::bindings::_helpers::model_config::header_metadata::{ + ContractClasses, HeaderValue, audit_instance, thresholds, +}; +use crate::bindings::_helpers::native_host::header_projection::AuditProjection; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::result_output::integration_values::json_value; +use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; + +/// Native-built specs never become Python audit records just to be read by attachment. +pub(crate) enum AuditInstance<'py> { + Native(Arc>, Bound<'py, PyAny>), + Python(Bound<'py, PyAny>), +} + +impl<'py> AuditInstance<'py> { + /// Equality with the builtin generated enum audit, without constructing argument containers. + pub(crate) fn accepts_enum(&self, values: &[PyConfigNode]) -> PyResult { + let Self::Native(spec, location) = self else { + return Ok(false); + }; + let py = location.py(); + if spec.definition_name.exact_text(py)? != ACCEPTED_VALUES_AUDIT + || spec.thresholds.is_some() + { + return Ok(false); + } + for name in [ + "name", + "description", + "severity", + "run_scope", + "minimum_samples", + "evidence_limit", + "meta", + ] { + if !self.field(name)?.is_none() { + return Ok(false); + } + } + if !self.field("always_run")?.eq(false)? || spec.arguments.len() != 1 { + return Ok(false); + } + let (key, authored) = &spec.arguments[0]; + let matches = match key { + ConfigNode::Str(key) => key == ACCEPTED_VALUES_ARGUMENT, + ConfigNode::Host(_) => { + let key = key.object(py)?; + key.is_instance_of::() && key.eq(ACCEPTED_VALUES_ARGUMENT)? + } + _ => false, + }; + if !matches { + return Ok(false); + } + let ConfigNode::Tuple(authored) = authored else { + return Ok(false); + }; + if authored.len() != values.len() { + return Ok(false); + } + for (left, right) in authored.iter().zip(values) { + if !left.object(py)?.eq(right.object(py)?)? { + return Ok(false); + } + } + Ok(true) + } + + pub(crate) fn meta_present(&self) -> PyResult { + match self { + Self::Native(spec, _) => Ok(spec + .meta + .as_ref() + .is_some_and(|value| !matches!(value, ConfigNode::Null))), + Self::Python(value) => Ok(!field(value, "meta")?.is_none()), + } + } + + pub(crate) fn threshold_policy(&self) -> PyResult<(bool, bool)> { + match self { + Self::Native(spec, _) => Ok(( + spec.thresholds.is_some(), + spec.thresholds.is_some_and(|value| value.error.is_some()), + )), + Self::Python(value) => { + let thresholds = field(value, "thresholds")?; + Ok(( + !thresholds.is_none(), + !thresholds.is_none() && !field(&thresholds, "error")?.is_none(), + )) + } + } + } + + pub(crate) fn field(&self, name: &str) -> PyResult> { + let (spec, location) = match self { + Self::Native(spec, location) => (spec, location), + Self::Python(value) => return field(value, name), + }; + let py = location.py(); + let value = match name { + "definition_name" => return spec.definition_name.object(py), + "name" => &spec.name, + "description" => &spec.description, + "run_scope" => &spec.run_scope, + "minimum_samples" => &spec.minimum_samples, + "evidence_limit" => &spec.evidence_limit, + "meta" => &spec.meta, + "always_run" => { + return match &spec.always_run { + Some(value) => value.object(py), + None => false.into_bound_py_any(py), + }; + } + "severity" => { + let classes = contract_classes(py)?; + let classes: ContractClasses<'_> = classes.extract()?; + return match &spec.severity { + Some(value) => Ok(classes + .severities + .get_item(value.object(py)?)? + .unwrap_or_else(|| py.None().into_bound(py))), + None => Ok(py.None().into_bound(py)), + }; + } + "thresholds" => { + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + return thresholds(py, &classes, spec.thresholds.as_ref()); + } + "location" => return Ok(location.clone()), + _ => return Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), + }; + value + .as_ref() + .map(|value| value.object(py)) + .transpose() + .map(|value| value.unwrap_or_else(|| py.None().into_bound(py))) + } + + pub(crate) fn argument(&self, name: &str) -> PyResult>> { + match self { + Self::Native(spec, location) => { + for (key, value) in &spec.arguments { + let matches = match key { + ConfigNode::Str(key) => key == name, + ConfigNode::Host(_) => { + let key = key.object(location.py())?; + key.is_instance_of::() && key.eq(name)? + } + _ => false, + }; + if matches { + return value.object(location.py()).map(Some); + } + } + Ok(None) + } + Self::Python(value) => field(value, "arguments")?.cast::()?.get_item(name), + } + } + + pub(crate) fn arguments(&self) -> PyResult> { + match self { + Self::Native(spec, location) => { + let mut arguments = Vec::with_capacity(spec.arguments.len()); + for (key, value) in &spec.arguments { + let key = match key { + ConfigNode::Str(key) => key.clone(), + ConfigNode::Host(_) => { + let key = key.object(location.py())?; + let Ok(key) = key.cast::() else { + continue; + }; + key.to_string_lossy().into_owned() + } + _ => continue, + }; + arguments.push((key, config_argument_value(location.py(), value)?)); + } + Ok(arguments) + } + Self::Python(value) => argument_pairs(field(value, "arguments")?.cast()?), + } + } + + pub(crate) fn meta(&self) -> PyResult> { + match self { + Self::Native(spec, location) => spec + .meta + .as_ref() + .map(|value| config_json_value(location.py(), value)) + .transpose() + .map(Option::flatten), + Self::Python(value) => { + let meta = field(value, "meta")?; + if meta.is_none() { + Ok(None) + } else { + json_value(&meta) + } + } + } + } +} + +/// Native discovery facts keep their authored values rather than building audit records. +pub(crate) fn authored_audit( + py: Python<'_>, + spec: &NativeAudit, +) -> PyResult> { + let value = |value: &AuthoredValue| match config_value(value) { + Some(value) => Ok(value), + None => config_node(authored_value_to_python(py, value)?.bind(py)), + }; + Ok(ParsedAudit { + definition_name: ConfigNode::Str(spec.definition_name.clone()), + arguments: spec + .arguments + .iter() + .map(|(key, item)| Ok((ConfigNode::Str(key.clone()), value(item)?))) + .collect::>()?, + name: spec.name.clone().map(ConfigNode::Str), + description: spec.description.clone().map(ConfigNode::Str), + severity: spec.severity.clone().map(ConfigNode::Str), + run_scope: spec.run_scope.clone().map(ConfigNode::Str), + always_run: spec.always_run.map(ConfigNode::Bool), + thresholds: spec.thresholds, + minimum_samples: spec.minimum_samples.as_ref().map(value).transpose()?, + evidence_limit: spec.evidence_limit.as_ref().map(value).transpose()?, + meta: spec.meta.as_ref().map(value).transpose()?, + }) +} + +/// The parsed specs and genuine Python audit objects of one model or column. +pub(crate) struct HeaderAudits { + specs: Vec>, + native_generated: Vec>, + public: AuditProjection, +} + +impl HeaderAudits { + pub(crate) fn new( + py: Python<'_>, + classes: &ContractClasses<'_>, + specs: &[ParsedAudit], + location: &Bound<'_, PyAny>, + ) -> PyResult { + let custom = if classes.uses_default_classes()? { + None + } else { + Some( + specs + .iter() + .map(|spec| audit_instance(py, classes, spec, location).map(Bound::unbind)) + .collect::>()?, + ) + }; + Ok(Self { + specs: specs + .iter() + .map(|spec| retain_audit(py, spec)) + .collect::>()?, + native_generated: Vec::new(), + public: AuditProjection { + custom, + generated: Vec::new(), + location: location.clone().unbind(), + records: PyOnceLock::new(), + }, + }) + } + + pub(crate) fn extend(&mut self, generated: Vec>) { + self.public + .generated + .extend(generated.into_iter().map(Bound::unbind)); + } + + pub(crate) fn push_native(&mut self, spec: ParsedAudit) { + self.native_generated.push(spec); + } + + pub(crate) fn len(&self) -> usize { + self.specs.len() + self.public.generated.len() + self.native_generated.len() + } + + pub(crate) fn native_specs(&self) -> &[ParsedAudit] { + if self.public.custom.is_some() { + &[] + } else { + &self.specs + } + } + + pub(crate) fn python_specs<'py>(&self, py: Python<'py>) -> Vec> { + self.public + .custom + .iter() + .flatten() + .chain(&self.public.generated) + .map(|spec| spec.bind(py).clone()) + .collect() + } + + pub(crate) fn instances<'py>(&self, py: Python<'py>) -> Vec> { + self.native_specs() + .iter() + .map(|spec| { + AuditInstance::Native( + Arc::new(spec.clone()), + self.public.location.bind(py).clone(), + ) + }) + .chain(self.python_specs(py).into_iter().map(AuditInstance::Python)) + .chain(self.native_generated.iter().map(|spec| { + AuditInstance::Native(Arc::new(spec.clone()), py.None().into_bound(py)) + })) + .collect() + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.public + .records + .get_or_try_init(py, || { + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + let mut audits = self + .native_specs() + .iter() + .map(|spec| audit_instance(py, &classes, spec, self.public.location.bind(py))) + .collect::>>()?; + audits.extend(self.python_specs(py)); + for spec in &self.native_generated { + audits.push(audit_instance( + py, + &classes, + spec, + &py.None().into_bound(py), + )?); + } + PyTuple::new(py, audits).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } + + pub(crate) fn key_rows(&self, py: Python<'_>) -> PyResult)>> { + let mut rows = Vec::with_capacity(self.len()); + for spec in self.instances(py) { + let column = spec.argument("column")?; + rows.push(( + spec.field("definition_name")?.extract()?, + column + .filter(|value| value.is_instance_of::()) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) + } +} + +/// Preserve parsed leaves without rebuilding their public argument dictionaries. +pub(crate) fn retain_audit( + py: Python<'_>, + spec: &ParsedAudit, +) -> PyResult> { + let optional = |node: &Option| node.as_ref().map(|node| node.retained(py)).transpose(); + Ok(ParsedAudit { + definition_name: spec.definition_name.retained(py)?, + arguments: spec + .arguments + .iter() + .map(|(key, value)| Ok((key.retained(py)?, value.retained(py)?))) + .collect::>()?, + name: optional(&spec.name)?, + description: optional(&spec.description)?, + severity: optional(&spec.severity)?, + run_scope: optional(&spec.run_scope)?, + always_run: optional(&spec.always_run)?, + thresholds: spec.thresholds, + minimum_samples: optional(&spec.minimum_samples)?, + evidence_limit: optional(&spec.evidence_limit)?, + meta: optional(&spec.meta)?, + }) +} + +/// Read metadata without projecting retained containers; opaque user values keep Python semantics. +fn config_json_value(py: Python<'_>, value: &PyConfigNode) -> PyResult> { + Ok(Some(match value { + ConfigNode::Null => JsonValue::Null, + ConfigNode::Bool(value) => JsonValue::Bool(*value), + ConfigNode::Int(value) => JsonValue::Integer(JsonInteger::from(*value)), + ConfigNode::Float(value) => JsonValue::Float(*value), + ConfigNode::Str(value) => JsonValue::String(value.clone()), + ConfigNode::List(items) | ConfigNode::Tuple(items) => { + let mut result: Vec = Vec::with_capacity(items.len()); + for item in items { + let Some(value) = config_json_value(py, item)? else { + return Ok(None); + }; + result.push(value); + } + JsonValue::Array(result) + } + ConfigNode::Map(entries) => { + let mut result: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); + for (key, item) in entries { + let ConfigNode::Str(key) = key else { + return json_value(&config_object(py, value)?); + }; + let Some(value) = config_json_value(py, item)? else { + return Ok(None); + }; + result.push((key.clone(), value)); + } + JsonValue::Object(result) + } + ConfigNode::Host(_) => return json_value(&config_object(py, value)?), + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs new file mode 100644 index 0000000000..be8d53470a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_families.rs @@ -0,0 +1,65 @@ +//! Native dynamic-column contracts with a shared, lazy public projection. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +use crate::bindings::_helpers::native_host::header_projection::FamilyProjection; +use crate::bindings::_helpers::records::construct::record_class; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +/// Parsed family specs stay native until a public schema attribute needs records. +pub(crate) struct HeaderFamilies { + pub(crate) specs: Vec, + public: FamilyProjection, +} + +impl HeaderFamilies { + pub(crate) fn new(class: &Bound<'_, PyAny>, specs: Vec) -> PyResult { + let result = Self { + specs, + public: FamilyProjection::new( + class, + !class.is(&record_class( + class.py(), + RecordId::SchemaDynamicColumnFamily, + )?), + ), + }; + if result.public.custom { + let _ = result.project(class.py())?; + } + Ok(result) + } + + pub(crate) fn read(&self, py: Python<'_>) -> PyResult> { + if self.public.custom { + family_specs(self.project(py)?.as_any()) + } else { + Ok(self.specs.clone()) + } + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.public.project(py, &self.specs) + } +} + +/// A Python-authored family contract crosses into native ownership once. +pub(crate) fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { + families + .try_iter()? + .map(|family| { + let family = family?; + Ok(DynamicFamily { + name: field(&family, "name")?.extract()?, + pivot_column: field(&family, "pivot_column")?.extract()?, + value_column: field(&family, "value_column")?.extract()?, + aggregate: field(&family, "aggregate")?.extract()?, + data_type: field(&family, "type")?.extract()?, + name_pattern: field(&family, "name_pattern")?.extract()?, + }) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 881b358b83..df8b27f346 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -9,14 +9,23 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt}; use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_model_config::header_metadata::constants::AUDITS_KEY; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; +use sqlbuild_model_config::types::AuthoredNode; use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_nodes::{ + PyConfigNode, config_node, config_object, +}; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; +use crate::bindings::_helpers::model_config::header_audits::{ + AuditInstance, HeaderAudits, authored_audit, +}; +use crate::bindings::_helpers::model_config::typed_config::node_text; use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::fields::field; @@ -61,10 +70,58 @@ const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), ]; +/// A parsed leaf projected only when a public contract needs its Python value. +pub(crate) trait HeaderValue: AuthoredNode { + fn object<'py>(&self, py: Python<'py>) -> PyResult>; + + fn retained(&self, py: Python<'_>) -> PyResult { + config_node(&self.object(py)?) + } + + fn exact_text(&self, py: Python<'_>) -> PyResult { + self.object(py)?.extract() + } +} + +impl HeaderValue for PyNode<'_> { + fn object<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self.0.as_unbound().bind(py).clone()) + } +} + +impl HeaderValue for &PyConfigNode { + fn object<'py>(&self, py: Python<'py>) -> PyResult> { + config_object(py, self) + } + + fn exact_text(&self, py: Python<'_>) -> PyResult { + node_text(py, self) + } + + fn retained(&self, _py: Python<'_>) -> PyResult { + Ok((*self).clone()) + } +} + +impl HeaderValue for PyConfigNode { + fn object<'py>(&self, py: Python<'py>) -> PyResult> { + config_object(py, self) + } + + fn exact_text(&self, py: Python<'_>) -> PyResult { + node_text(py, self) + } + + fn retained(&self, _py: Python<'_>) -> PyResult { + Ok(self.clone()) + } +} + /// Native column fields; a Python record is constructed only for public consumers. pub(crate) struct HeaderColumn { fields: ColumnFields, class: Py, + audits: Option, projection: PyOnceLock>, } @@ -86,6 +143,81 @@ pub(crate) struct NativeColumnSpec { } impl HeaderColumn { + /// Retype a builtin column without discarding its retained audit specs. + pub(crate) fn enum_type( + mut self, + py: Python<'_>, + scalar_type: Bound<'_, PyAny>, + generated: Option>, + ) -> PyResult { + if self.audits.is_none() + && let ColumnFields::Native(spec) = &self.fields + { + let audits = spec + .column + .audits + .iter() + .map(|audit| authored_audit(py, audit)) + .collect::>>()?; + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + self.audits = Some(HeaderAudits::new( + py, + &classes, + &audits, + &native_location(py, spec)?, + )?); + } + if let Some(audits) = &mut self.audits { + if let Some(generated) = generated { + audits.push_native(generated); + } + match &mut self.fields { + ColumnFields::Native(spec) => spec.column.column_type = scalar_type.extract()?, + ColumnFields::Python(values) => values[1] = scalar_type.unbind(), + ColumnFields::Projected => { + return Err(PyRuntimeError::new_err( + "enum column has no retained fields", + )); + } + } + self.projection = PyOnceLock::new(); + return Ok(self); + } + Err(PyRuntimeError::new_err( + "enum column has no retained audits", + )) + } + + pub(crate) fn retains_audits(&self) -> bool { + self.audits.is_some() || matches!(self.fields, ColumnFields::Native(_)) + } + + pub(crate) fn audit_instances<'py>( + &self, + py: Python<'py>, + ) -> PyResult>> { + if let Some(audits) = &self.audits { + return Ok(audits.instances(py)); + } + if let ColumnFields::Native(spec) = &self.fields { + let audits = spec + .column + .audits + .iter() + .map(|audit| authored_audit(py, audit)) + .collect::>>()?; + let classes: ContractClasses<'_> = contract_classes(py)?.extract()?; + return Ok( + HeaderAudits::new(py, &classes, &audits, &native_location(py, spec)?)? + .instances(py), + ); + } + self.field(py, "audits")? + .try_iter()? + .map(|value| value.map(AuditInstance::Python)) + .collect() + } + pub(crate) fn replaced<'py>( class: &Bound<'py, PyAny>, values: [Bound<'py, PyAny>; 8], @@ -111,6 +243,11 @@ impl HeaderColumn { if let Some(projected) = self.projection.get(py) { return field(projected.bind(py), name); } + if name == AUDITS_KEY + && let Some(audits) = &self.audits + { + return Ok(audits.project(py)?.into_any()); + } match &self.fields { ColumnFields::Python(values) => Ok(values[index()?].bind(py).clone()), ColumnFields::Native(spec) => native_value(py, spec, index()?), @@ -121,7 +258,7 @@ impl HeaderColumn { pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { self.projection .get_or_try_init(py, || { - let values: [Bound<'py, PyAny>; 8] = match &self.fields { + let mut values: [Bound<'py, PyAny>; 8] = match &self.fields { ColumnFields::Python(values) => { values.each_ref().map(|value| value.bind(py).clone()) } @@ -130,6 +267,9 @@ impl HeaderColumn { return Err(PyRuntimeError::new_err("missing column projection")); } }; + if let Some(audits) = &self.audits { + values[5] = audits.project(py)?.into_any(); + } contract_object(self.class.bind(py), values).map(Bound::unbind) }) .map(|value| value.bind(py).clone()) @@ -143,6 +283,7 @@ impl HeaderColumn { Ok(Self { fields: ColumnFields::Projected, class, + audits: None, projection, }) } @@ -158,6 +299,7 @@ impl HeaderColumn { Ok(Self { fields: ColumnFields::Python(values.map(Bound::unbind)), class: class.clone().unbind(), + audits: None, projection: PyOnceLock::new(), }) } @@ -172,14 +314,18 @@ impl HeaderColumn { Ok(Self { fields: ColumnFields::Native(Box::new(spec)), class: class.clone().unbind(), + audits: None, projection: PyOnceLock::new(), }) } } impl HeaderColumn { - /// The natively parsed column, or `None` when Python built its values or record. + /// The original native column, if no retained audit edits changed its public contract. pub(crate) fn native_spec(&self) -> Option<&NativeColumnSpec> { + if self.audits.is_some() { + return None; + } match &self.fields { ColumnFields::Native(spec) => Some(spec), ColumnFields::Python(_) | ColumnFields::Projected => None, @@ -369,7 +515,7 @@ pub(crate) struct ContractClasses<'py> { pub(crate) source_location: Bound<'py, PyAny>, measurement_thresholds: Bound<'py, PyAny>, measurement_threshold_bound: Bound<'py, PyAny>, - severities: Bound<'py, PyDict>, + pub(crate) severities: Bound<'py, PyDict>, threshold_operators: Bound<'py, PyDict>, } @@ -426,10 +572,10 @@ pub(crate) fn audit_tuple<'py>( } /// The parsed audits as `SchemaAuditInstance` objects without a location, in order. -pub(crate) fn audit_instances<'py>( +pub(crate) fn audit_instances<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, - parsed: &[ParsedAudit>], + parsed: &[ParsedAudit], ) -> PyResult>> { let no_location: Bound<'py, PyAny> = py.None().into_bound(py); let mut audits: Vec> = Vec::with_capacity(parsed.len()); @@ -440,13 +586,16 @@ pub(crate) fn audit_instances<'py>( } /// `audit.name or audit.definition_name` of the instance `audit` builds. -pub(crate) fn parsed_audit_identity(audit: &ParsedAudit>) -> PyResult { +pub(crate) fn parsed_audit_identity( + py: Python<'_>, + audit: &ParsedAudit, +) -> PyResult { if let Some(name) = &audit.name - && name.0.is_truthy()? + && name.object(py)?.is_truthy()? { - return name.0.extract(); + return name.exact_text(py); } - audit.definition_name.0.extract() + audit.definition_name.exact_text(py) } /// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. @@ -491,8 +640,14 @@ fn mismatched_record() -> Option<&'static str> { None } -fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { - value.map_or_else(|| py.None().into_bound(py), |node| node.0.clone()) +fn optional<'py, N: HeaderValue>( + py: Python<'py>, + value: Option<&N>, +) -> PyResult> { + Ok(value + .map(|node| node.object(py)) + .transpose()? + .unwrap_or_else(|| py.None().into_bound(py))) } fn schema_column<'py>( @@ -504,70 +659,78 @@ fn schema_column<'py>( native_schema_column(py, classes, column, locations)?.project(py) } -pub(crate) fn native_schema_column<'py>( +pub(crate) fn native_schema_column<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, - column: &ParsedColumn>, + column: &ParsedColumn, locations: &Bound<'py, PyDict>, ) -> PyResult { let location = locations - .get_item(&column.name.0)? + .get_item(column.name.object(py)?)? .unwrap_or_else(|| py.None().into_bound(py)); - let audits = column - .audits - .iter() - .map(|audit| audit_instance(py, classes, audit, &location)) - .collect::>>()?; - HeaderColumn::new( + let audits = HeaderAudits::new(py, classes, &column.audits, &location)?; + let projected_audits = if classes + .schema_column + .is(&record_class(py, RecordId::SchemaColumn)?) + { + PyTuple::empty(py) + } else { + audits.project(py)? + }; + let mut result = HeaderColumn::new( &classes.schema_column, [ - column.name.0.clone(), - optional(py, column.column_type.as_ref()), - optional(py, column.nullable.as_ref()), - optional(py, column.description.as_ref()), + column.name.object(py)?, + optional(py, column.column_type.as_ref())?, + optional(py, column.nullable.as_ref())?, + optional(py, column.description.as_ref())?, PyDict::new(py).into_any(), - PyTuple::new(py, audits)?.into_any(), + projected_audits.into_any(), location, - optional(py, column.migrate_from.as_ref()), + optional(py, column.migrate_from.as_ref())?, ], - ) + )?; + if !matches!(result.fields, ColumnFields::Projected) { + result.audits = Some(audits); + } + Ok(result) } -fn audit_instance<'py>( +pub(crate) fn audit_instance<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, - audit: &ParsedAudit>, + audit: &ParsedAudit, location: &Bound<'py, PyAny>, ) -> PyResult> { let arguments = PyDict::new(py); for (key, value) in &audit.arguments { - arguments.set_item(&key.0, &value.0)?; + arguments.set_item(key.object(py)?, value.object(py)?)?; } let severity = match &audit.severity { Some(severity) => classes .severities - .get_item(&severity.0)? + .get_item(severity.object(py)?)? .unwrap_or_else(|| py.None().into_bound(py)), None => py.None().into_bound(py), }; - let always_run = audit.always_run.as_ref().map_or_else( - || PyBool::new(py, false).to_owned().into_any(), - |node| node.0.clone(), - ); + let always_run = match &audit.always_run { + Some(node) => node.object(py)?, + None => PyBool::new(py, false).to_owned().into_any(), + }; contract_object( &classes.schema_audit_instance, [ - audit.definition_name.0.clone(), + audit.definition_name.object(py)?, arguments.into_any(), - optional(py, audit.name.as_ref()), - optional(py, audit.description.as_ref()), + optional(py, audit.name.as_ref())?, + optional(py, audit.description.as_ref())?, severity, - optional(py, audit.run_scope.as_ref()), + optional(py, audit.run_scope.as_ref())?, always_run, thresholds(py, classes, audit.thresholds.as_ref())?, - optional(py, audit.minimum_samples.as_ref()), - optional(py, audit.evidence_limit.as_ref()), - optional(py, audit.meta.as_ref()), + optional(py, audit.minimum_samples.as_ref())?, + optional(py, audit.evidence_limit.as_ref())?, + optional(py, audit.meta.as_ref())?, location.clone(), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 4ebfd08885..ce22851d76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -9,7 +9,9 @@ pub(crate) mod contract_classes; pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod getters; +pub(crate) mod header_audits; pub(crate) mod header_columns; +pub(crate) mod header_families; pub(crate) mod header_metadata; pub(crate) mod model_config_builder; pub(crate) mod model_validator; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 46b5f31fe9..fe0737c151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -1,9 +1,12 @@ //! The native model config build handed Python inputs once, and its results projected for Python. +use std::collections::HashMap; + use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{PyErr, pyclass, pymethods}; +use sqlbuild_discovery::model_files::main::config_value::config_value; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; use sqlbuild_model_config::config_presence::models::MacroPath; use sqlbuild_model_config::config_values::models::ConfigNode; @@ -18,6 +21,7 @@ use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::compiler_guard; @@ -36,6 +40,13 @@ use crate::bindings::_helpers::model_config::typed_config::{BuiltConfigValues, b const CONTEXT_KEYS: [&str; 3] = ["database", "schema", "alias"]; const BUILD_STOPPED: &str = "a config value read as text without an error"; +/// A header's public mapping, parsed facts and native authored values. +type HeaderBuildInputs<'a, 'py> = ( + &'a Bound<'py, PyDict>, + Option<&'a HeaderFacts>, + Option<&'a [(String, AuthoredValue)]>, +); + /// The project-wide inputs every model's config build reads, converted once per compile. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct NativeModelConfigBuilder { @@ -153,9 +164,26 @@ fn carried_whole(key: &str, facts: Option<&[(String, HeaderEntryFacts)]>) -> boo fn header_entries( header: &Bound<'_, PyDict>, facts: Option<&[(String, HeaderEntryFacts)]>, + authored: Option<&[(String, AuthoredValue)]>, ) -> PyResult { + let authored: HashMap<&str, &AuthoredValue> = authored + .into_iter() + .flatten() + .map(|(key, value)| (key.as_str(), value)) + .collect(); let mut entries: PyConfigEntries = Vec::with_capacity(header.len()); for (key, value) in header.iter() { + let native: Option = match key.downcast_exact::() { + Ok(key) => match key.to_str() { + Ok(key) => authored.get(key).and_then(|value| config_value(value)), + Err(_) => None, + }, + Err(_) => None, + }; + if let Some(native) = native { + entries.push((config_node(&key)?, native)); + continue; + } let whole: bool = is_exact_container(&value) && key .downcast_exact::() @@ -234,7 +262,7 @@ impl NativeModelConfigBuilder { let (matched_path_default, model_name) = model; let matched_path_default: Option<&str> = matched_path_default.as_deref(); compiler_guard(|| { - match self.built(py, (&header, None), matched_path_default, &model_name)? { + match self.built(py, (&header, None, None), matched_path_default, &model_name)? { ConfigBuild::Built(built) => { Ok(built_config(&classes, &built, matched_path_default)?.unbind()) } @@ -340,13 +368,13 @@ impl NativeModelConfigBuilder { pub(crate) fn built<'py>( &self, py: Python<'py>, - (header, facts): (&Bound<'py, PyDict>, Option<&HeaderFacts>), + (header, facts, authored): HeaderBuildInputs<'_, 'py>, matched_path_default: Option<&str>, model_name: &str, ) -> PyResult> { let facts: Option> = facts.map(|facts| facts.entries.iter().map(entry_facts).collect()); - let entries: PyConfigEntries = header_entries(header, facts.as_deref())?; + let entries: PyConfigEntries = header_entries(header, facts.as_deref(), authored)?; let model_header = ModelHeader { entries: &entries, facts: facts.as_deref(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs new file mode 100644 index 0000000000..b40db7b080 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/header_projection.rs @@ -0,0 +1,54 @@ +//! Public header records and user-supplied constructors at the Python boundary. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +/// Factory results, source locations and the cached public audit tuple are Python-owned edges. +pub(crate) struct AuditProjection { + pub(crate) custom: Option>>, + pub(crate) generated: Vec>, + pub(crate) location: Py, + pub(crate) records: PyOnceLock>, +} + +/// A supplied constructor and its public family records; native callers retain the family specs. +pub(crate) struct FamilyProjection { + class: Py, + pub(crate) custom: bool, + records: PyOnceLock>, +} + +impl FamilyProjection { + pub(crate) fn new(class: &Bound<'_, PyAny>, custom: bool) -> Self { + Self { + class: class.clone().unbind(), + custom, + records: PyOnceLock::new(), + } + } + + pub(crate) fn project<'py>( + &self, + py: Python<'py>, + specs: &[DynamicFamily], + ) -> PyResult> { + self.records + .get_or_try_init(py, || { + let mut records: Vec> = Vec::with_capacity(specs.len()); + for family in specs { + let fields = PyDict::new(py); + fields.set_item("name", &family.name)?; + fields.set_item("pivot_column", &family.pivot_column)?; + fields.set_item("value_column", &family.value_column)?; + fields.set_item("aggregate", &family.aggregate)?; + fields.set_item("type", &family.data_type)?; + fields.set_item("name_pattern", &family.name_pattern)?; + records.push(self.class.bind(py).call((), Some(&fields))?); + } + PyTuple::new(py, records).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 4737c79a25..8f9ded74e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,7 +1,9 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod connections; +pub(crate) mod header_projection; pub(crate) mod host; +pub(crate) mod model_lookup; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs new file mode 100644 index 0000000000..ce905aaebc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs @@ -0,0 +1,88 @@ +//! Public model-map inputs and projections around the native planner's model lookup. + +use std::sync::Arc; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; + +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::planner_scope::retained::ScopeData; +use crate::bindings::_helpers::records::fields::mapping_get; + +/// Native compiled views stay typed; caller-authored protocol implementations remain public values. +#[derive(Debug)] +pub(crate) enum ModelHandle { + View(Py), + Public(Py), +} + +impl ModelHandle { + pub(crate) fn read(value: &Bound<'_, PyAny>) -> Self { + match value.cast::() { + Ok(value) => Self::View(value.clone().unbind()), + Err(_) => Self::Public(value.clone().unbind()), + } + } + + pub(crate) fn bind<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + match self { + Self::View(value) => value.bind(py).clone().into_any(), + Self::Public(value) => value.bind(py).clone(), + } + } + + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + match self { + Self::View(value) => Self::View(value.clone_ref(py)), + Self::Public(value) => Self::Public(value.clone_ref(py)), + } + } +} + +/// A public caller's mapping, or the compiler-owned models shared by native planning. +#[derive(Debug)] +pub(crate) enum ModelLookup { + Public(Py), + Native(Arc), +} + +impl ModelLookup { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + match self { + Self::Public(value) => Self::Public(value.clone_ref(py)), + Self::Native(value) => Self::Native(value.clone()), + } + } + + pub(crate) fn get<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + match self { + Self::Public(value) => mapping_get(value.bind(py), name), + Self::Native(value) => Ok(value + .model(py, name) + .unwrap_or_else(|| py.None().into_bound(py))), + } + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Public(value) => Ok(value.bind(py).clone()), + Self::Native(value) => value.field(py, "models_by_name"), + } + } +} + +impl<'py> From> for ModelLookup { + fn from(value: Bound<'py, PyAny>) -> Self { + Self::Public(value.unbind()) + } +} + +impl<'py> IntoPyObject<'py> for &ModelLookup { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + self.project(py) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 9e98797d20..8232f95282 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -3,8 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_scope::constants::MODEL; -use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; +use crate::bindings::_helpers::planner_scope::retained::scope_data; +use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; @@ -18,19 +20,16 @@ pub(crate) fn resolve_model_actions<'py>( changes: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = scope.py(); - let model_kind: Bound<'py, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + let scope = scope_data(scope)?; let model_changes: Bound<'py, PyAny> = native_field(changes, "models")?; let resolved: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(scope, "execution_order")?.try_iter()? { - let key: Bound<'py, PyAny> = key?; - if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { + for key in &scope.order { + if !scope.selected.contains(key) || key.0 != MODEL { continue; } - let name: Bound<'py, PyAny> = native_field(&key, "name")?; - let change: Bound<'py, PyAny> = mapping_get(&model_changes, &name)?; - if !models_by_name.contains(&name)? || change.is_none() { + let name = &key.1; + let change: Bound<'py, PyAny> = mapping_get(&model_changes, name)?; + if !scope.models.contains_key(name) || change.is_none() { continue; } let backfill: Bound<'py, PyAny> = native_field(&change, "backfill")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 6fb7570f0b..ae467ab91e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,5 +1,6 @@ //! Detect how each selected model and function changed since its recorded build. +use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::function_reads; @@ -8,7 +9,7 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; -use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; +use crate::bindings::_helpers::records::construct::new_record_with; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -43,6 +44,7 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; @@ -103,7 +105,7 @@ pub(crate) fn detect_changes<'py>( let models: Bound<'py, PyDict> = scope_model_changes((project, scope, snapshot), None, expected)?; let functions: Bound<'py, PyDict> = PyDict::new(py); - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; + let selected = scope_data(scope)?; let recorded: Bound<'py, PyAny> = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let query_change_tracking: bool = @@ -117,7 +119,12 @@ pub(crate) fn detect_changes<'py>( "fingerprint_sql", PyString::new(py, &fingerprint_sql).into_any(), )]; - if selected.contains(function_reads::key(&function)?)? { + if selected + .selected + .contains(&crate::bindings::_helpers::graph::keys::native_key( + &function_reads::key(&function)?, + )?) + { let reason: &str = if full_refresh { "full_refresh" } else { @@ -239,19 +246,15 @@ pub(crate) fn scope_model_changes<'py>( function_local_hashes, hook_version_hashes: hook_version_hashes.into_any(), }; - let model_kind: Bound<'py, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; - let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; - let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + let scope = scope_data(scope)?; let changes: Bound<'py, PyDict> = PyDict::new(py); - for key in native_field(scope, "execution_order")?.try_iter()? { - let key: Bound<'py, PyAny> = key?; - if !selected.contains(&key)? || !native_field(&key, "resource_type")?.eq(&model_kind)? { + for key in &scope.order { + if !scope.selected.contains(key) || key.0 != MODEL { continue; } - let model: Bound<'py, PyAny> = mapping_get(&models_by_name, native_field(&key, "name")?)?; - if model.is_none() { + let Some(model) = scope.model(py, &key.1) else { continue; - } + }; let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let expected: (Option, Option) = ( lookup(&version_hashes, &name)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 71faadc542..83c9adbe24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; @@ -219,11 +220,10 @@ fn changed_referencing_query( snapshot: &Bound<'_, PyAny>, ) -> PyResult { let py: Python<'_> = scope.py(); - let models_by_name: Bound<'_, PyAny> = native_field(scope, "models_by_name")?; + let scope = scope_data(scope)?; let mut selected: Vec> = Vec::new(); - for key in native_field(scope, "selected_keys")?.try_iter()? { - let model: Bound<'_, PyAny> = mapping_get(&models_by_name, native_field(&key?, "name")?)?; - if !model.is_none() { + for (_, name) in &scope.selected { + if let Some(model) = scope.model(py, name) { selected.push(model); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 160b531330..b90ca8da71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -7,16 +7,15 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::compiled_views::source_reads; +use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; -use crate::bindings::_helpers::planner_identity::graph::{ - build_downstream_deps, build_execution_upstream_deps, -}; +use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; @@ -35,6 +34,7 @@ use pyo3::PyRef; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; use sqlbuild_model_config::config_values::models::ConfigNode; const ENABLED_KEY: &str = "enabled"; @@ -81,8 +81,11 @@ fn clone_plan_output<'py>( exclude: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let upstream = build_execution_upstream_deps(project)?.into_any(); - let downstream = build_downstream_deps(&upstream)?.into_any(); + let upstream = execution_deps(project)?; + let order = execution_order(py, &upstream, &[])?; + let cache = KeyCache::new(py)?; + let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); + let upstream = edge_dict(&cache, &upstream)?.into_any(); let selected = object( py, resolve_graph_selection( @@ -97,7 +100,7 @@ fn clone_plan_output<'py>( &[ ( "execution_order", - object(py, topologically_order_keys(&upstream, None)?)?, + PyTuple::new(py, cache.keys(&order)?)?.into_any(), ), ("selected_keys", selected), ("upstream_deps", upstream), @@ -256,7 +259,7 @@ fn clone_model_entries<'py>( py, ModelPlanContext { model_locations: resolution.model_locations.clone().unbind(), - models_by_name: PyDict::new(py).into_any().unbind(), + models_by_name: PyDict::new(py).into_any().into(), functions_by_name: functions.into_any().unbind(), seed_locations: resolution.seed_locations.clone().unbind(), function_locations: resolution.function_locations.clone().unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs index 5c0ac73afd..f24541703d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs @@ -96,7 +96,7 @@ impl ModelPlanContext { )?; Ok(Self { model_locations: args.value("model_locations")?.unbind(), - models_by_name: args.value("models_by_name")?.unbind(), + models_by_name: args.value("models_by_name")?.into(), seed_locations: args.value("seed_locations")?.unbind(), function_locations: args.value("function_locations")?.unbind(), source_map: args.value("source_map")?.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index 1d67e3d651..6fd160bdbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -2,13 +2,13 @@ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::{ - build_downstream_deps, build_execution_upstream_deps, -}; +use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; @@ -17,6 +17,7 @@ use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; @@ -149,8 +150,10 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .iter() .map(|entry| native_field(entry, "key")) .collect::>()?; - let upstream = build_execution_upstream_deps(project)?.into_any(); - let downstream = build_downstream_deps(&upstream)?.into_any(); + let upstream = execution_deps(project)?; + let cache = KeyCache::new(py)?; + let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); + let upstream = edge_dict(&cache, &upstream)?.into_any(); planner_model( py, RecordId::PlanOutput, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 49dfa8bfb9..2fe47c07d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,5 +1,6 @@ //! Top-level planner orchestration producing an execution plan. +use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; @@ -35,8 +36,8 @@ use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; -use crate::bindings::_helpers::records::construct::enum_member; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -57,16 +58,13 @@ fn scopes_and_cursor_overrides<'py>( &args.value("policies")?, )?; let selected = class_field(&scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let models_by_name = native_field(&selected, "models_by_name")?; - let model_type = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; + let selected = scope_data(&selected)?; let mut models: Vec> = Vec::new(); - for key in native_field(&selected, "selected_keys")?.try_iter()? { - let key = key?; - let name = native_field(&key, "name")?; - if native_field(&key, "resource_type")?.eq(&model_type)? - && models_by_name.contains(&name)? + for key in &selected.selected { + if key.0 == MODEL + && let Some(model) = selected.model(py, &key.1) { - models.push(models_by_name.get_item(&name)?); + models.push(model); } } let overrides = args.value("overrides")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 507b42a24e..37ddf4e00e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -45,7 +45,6 @@ use crate::bindings::_helpers::planner_strategy::strategy::{ use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::construct::enum_member; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::semantic_shapes::source_bindings::source_binding_diagnostics; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -73,7 +72,7 @@ impl<'py> ContextReads<'py> { let field = |value: &Py| value.bind(py).clone(); Ok(Self { lookups: CursorInputLookups { - models_by_name: field(&fields.models_by_name), + models_by_name: fields.models_by_name.clone_ref(py), functions_by_name: field(&fields.functions_by_name), source_map: field(&fields.source_map), source_warehouse_columns: field(&fields.source_warehouse_columns), @@ -155,7 +154,7 @@ fn plan_model_from_change<'py>( let (future_config, start_config) = cursor_safety_configs(model, context)?; reject_capped_watermark_inputs( model, - reads.lookups.models_by_name.clone(), + reads.lookups.models_by_name.clone_ref(py), Some(reads.lookups.functions_by_name.clone()), )?; let cursor_input_relations = @@ -433,12 +432,15 @@ pub(crate) fn plan_entries<'py>( functions_by_name.set_item(function_reads::name(&function)?, function)?; } let settings = native_field(&project, "settings")?; - let models_by_name = native_field(&scope, "models_by_name")?; + let scope_data = crate::bindings::_helpers::planner_scope::retained::scope_data(&scope)?; let context = Bound::new( py, ModelPlanContext { model_locations: relations_context.model_locations.clone_ref(py), - models_by_name: models_by_name.clone().unbind(), + models_by_name: + crate::bindings::_helpers::native_host::model_lookup::ModelLookup::Native( + scope_data.clone(), + ), functions_by_name: functions_by_name.into_any().unbind(), seed_locations: relations_context.seed_locations.clone_ref(py), function_locations: relations_context.function_locations.clone_ref(py), @@ -462,17 +464,15 @@ pub(crate) fn plan_entries<'py>( let microbatch = enum_member(py, RecordId::IncrementalMode, "MICROBATCH")?; let mut entries: Vec> = Vec::new(); let mut warnings: Vec> = Vec::new(); - for key in native_field(&scope, "execution_order")?.try_iter()? { - let key = key?; - let name = native_field(&key, "name")?; - if !selected_keys.contains(&key)? || !models.contains(&name)? { + for key in &scope_data.order { + let name = &key.1; + if !scope_data.selected.contains(key) || !models.contains(name)? { continue; } - let model = mapping_get(&models_by_name, &name)?; - let resolved = models.get_item(&name)?; - if model.is_none() { + let Some(model) = scope_data.model(py, name) else { continue; - } + }; + let resolved = models.get_item(name)?; let overrides = model_cursor_overrides( &model, &cursor_overrides, @@ -487,8 +487,8 @@ pub(crate) fn plan_entries<'py>( full_refresh: effective_full_refresh, overrides, change_result: native_field(&resolved, "change")?, - protect_full_refresh: !freshness_blocked.contains(&name)? - && !external_blocked.contains(&name)?, + protect_full_refresh: !freshness_blocked.contains(name)? + && !external_blocked.contains(name)?, })?; let entry_name = native_field(&entry, "name")?; if freshness_blocked.contains(&entry_name)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index 52e10ea560..ac269cd208 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -1,5 +1,6 @@ //! Typed requests of plan entry construction. +use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; @@ -68,7 +69,7 @@ pub(crate) struct PlannerRelationsContext { #[derive(Debug)] pub(crate) struct ModelPlanContext { pub(crate) model_locations: Py, - pub(crate) models_by_name: Py, + pub(crate) models_by_name: ModelLookup, pub(crate) seed_locations: Py, pub(crate) function_locations: Py, pub(crate) source_map: Py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index dfe63632bf..6a4904be61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -4,16 +4,19 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::audit_reads; use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; -use crate::bindings::_helpers::planner_entries::values::{none, planner_model}; +use crate::bindings::_helpers::planner_entries::values::planner_model; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; -use crate::bindings::_helpers::planner_scope::sources::build_source_load_map; +use crate::bindings::_helpers::planner_scope::retained::scope_data; +use crate::bindings::_helpers::planner_scope::sources::source_load_map; +use crate::bindings::_helpers::planner_scope::values::keys; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; @@ -23,25 +26,26 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_values; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use std::collections::HashSet; /// Relation keys needed to render selected audits without selecting those relations. -fn selected_audit_relation_keys<'py>( - project: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult> { - let py = project.py(); - let keys = PySet::empty(py)?; +fn selected_audit_relation_keys( + project: &Bound<'_, PyAny>, + selected_keys: &HashSet, +) -> PyResult> { + let mut keys: HashSet = HashSet::new(); for audit in project_reads::audits(project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, selected_keys)? { for dependency in audit_reads::scope_deps(&audit)?.try_iter()? { - keys.add(dependency?)?; + keys.insert(native_key(&dependency?)?); } } } - Ok(PyFrozenSet::new(py, keys.iter())?.into_any()) + Ok(keys) } /// Reuse already-gathered source columns for the read map, or gather them once. @@ -122,23 +126,20 @@ pub(crate) fn planner_relations_context<'py>( model_locations = applied.get_item(0)?; seed_locations = applied.get_item(1)?; } - let audit_keys = selected_audit_relation_keys(&project, &selected_keys)?; + let scope = scope_data(&scope)?; let load_keys = if relation_keys.is_none() { - selected_keys.call_method1("__or__", (&audit_keys,))? + let mut relevant = selected_audit_relation_keys(&project, &scope.selected)?; + relevant.extend(scope.selected.iter().cloned()); + relevant.into_iter().collect() } else { - relation_keys + keys(&relation_keys)? }; let source_map = object( py, - build_source_load_map( + source_load_map( &project, &load_keys, - Some(&if native_field(&scope, "all_keys")?.is_truthy()? { - native_field(&scope, "upstream_deps")? - } else { - none(py) - }) - .filter(|value| !value.is_none()), + (!scope.names.is_empty()).then_some(scope.upstream.as_slice()), )?, )?; let deferral_enabled = native_field(&deferral, "source_deferral_enabled")?.is_truthy()?; @@ -170,7 +171,7 @@ pub(crate) fn planner_relations_context<'py>( with_declared_source_reads( &project, source_map.downcast::()?, - &native_field(&scope, "python_read_source_names")?, + &scope.field(py, "python_read_source_names")?, )?, )?; let python_names = PyFrozenSet::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index f5c228d432..8137458eda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -5,25 +5,28 @@ use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, -}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySetMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_identity::constants::{ - MODEL_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, -}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; +use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; +use sqlbuild_analysis::planner_identity::models::{ + AvailableUpstreams, ExecutionTest, GraphKey, MissingUpstream, +}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; -use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_identity::hashing::graph_key; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; @@ -72,11 +75,6 @@ impl<'py> KeyObjects<'py> { } } -fn function_resource(key: &Bound<'_, PyAny>) -> PyResult { - let kind: String = native_field(key, "resource_type")?.str()?.extract()?; - Ok(kind == UDF_RESOURCE || kind == TABLE_FN_RESOURCE) -} - fn hook_label(hook_name: &str, kind: &str) -> String { if kind == INLINE_SQL_HOOK { return format!("inline SQL hook {hook_name}"); @@ -89,9 +87,15 @@ fn hook_label(hook_name: &str, kind: &str) -> String { pub(crate) fn build_execution_upstream_deps<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = project.py(); - let upstream: Bound<'py, PyDict> = PyDict::new(py); - let collections: [Bound<'py, PyAny>; 4] = [ + edge_dict(&KeyCache::new(project.py())?, &execution_deps(project)?) +} + +/// Execution dependencies kept native until a public graph or plan asks for its dict view. +pub(crate) fn execution_deps( + project: &Bound<'_, PyAny>, +) -> PyResult)>> { + let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); + let collections: [Bound<'_, PyAny>; 4] = [ project_reads::models(project)?, project_reads::sources(project)?, project_reads::seeds(project)?, @@ -99,78 +103,30 @@ pub(crate) fn build_execution_upstream_deps<'py>( ]; for collection in collections { for resource in collection.try_iter()? { - let resource: Bound<'py, PyAny> = resource?; - upstream.set_item( - resource_reads::key(&resource)?, - PyList::new( - py, - resource_reads::deps(&resource)? - .try_iter()? - .collect::>>()?, - )?, - )?; + let row = resource_reads::graph_row(&resource?)?; + resources.push((row.key, row.deps)); } } + let mut tests: Vec = Vec::new(); for test in project_reads::sql_tests(project)?.try_iter()? { - let test: Bound<'py, PyAny> = test?; - let test_key: Bound<'py, PyAny> = native_field(&test, "key")?; - let scope_deps: Bound<'py, PyAny> = native_field(&test, "scope_deps")?; - let deps: Bound<'py, PyList> = PyList::empty(py); - if native_field(&test, "mode")?.str()?.extract::()? == TABLE_FN_TEST_MODE { - for scope_dep in scope_deps.try_iter()? { - let scope_dep: Bound<'py, PyAny> = scope_dep?; - if function_resource(&scope_dep)? { - deps.append(scope_dep)?; - } - } - upstream.set_item(test_key, deps)?; - continue; - } - let seen: Bound<'py, PySet> = PySet::empty(py)?; - for target_key in scope_deps.try_iter()? { - let Some(target_deps) = upstream.get_item(target_key?)? else { - continue; - }; - for dep_key in target_deps.try_iter()? { - let dep_key: Bound<'py, PyAny> = dep_key?; - if !function_resource(&dep_key)? || seen.contains(&dep_key)? { - continue; - } - seen.add(&dep_key)?; - deps.append(dep_key)?; - } - } - upstream.set_item(&test_key, deps)?; - for target_key in scope_deps.try_iter()? { - if let Some(target_deps) = upstream.get_item(target_key?)? { - target_deps.downcast::()?.append(&test_key)?; - } - } + let test = test?; + tests.push(ExecutionTest { + key: native_key(&native_field(&test, "key")?)?, + scope_deps: native_field(&test, "scope_deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + table_function: native_field(&test, "mode")?.str()?.extract::()? + == TABLE_FN_TEST_MODE, + }); } let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let cache: KeyCache<'py> = KeyCache::new(py)?; - let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for gate in &gates { - extra.push((cache.key(&gate.gated)?, cache.key(&gate.read)?)); - } - for hook in &hooks { - extra.push((cache.key(&hook.gated)?, cache.key(&hook.read)?)); - } - for (gated, read) in extra { - let gated_upstream: Bound<'py, PyAny> = - upstream.call_method1("setdefault", (gated, PyList::empty(py)))?; - if !gated_upstream.contains(&read)? { - gated_upstream.downcast::()?.append(read)?; - } - } - let result: Bound<'py, PyDict> = PyDict::new(py); - for (key, deps) in upstream.iter() { - result.set_item( - key, - PyTuple::new(py, deps.try_iter()?.collect::>>()?)?, - )?; - } - Ok(result) + let injected: Vec<_> = gates + .into_iter() + .map(|gate| (gate.gated, gate.read)) + .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) + .collect(); + Ok(execution_upstream(resources, &tests, &injected)) } /// Human-readable origins for execution edges injected beyond pure lineage. @@ -179,27 +135,39 @@ pub(crate) fn build_execution_edge_origins<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); - let origins: Bound<'py, PyDict> = PyDict::new(py); + let result = PyDict::new(py); + let cache = KeyCache::new(py)?; + for ((gated, read), origin) in execution_origins(project)? { + result.set_item((cache.key(&gated)?, cache.key(&read)?), origin)?; + } + Ok(result) +} + +/// Injected-edge descriptions remain native for cycle and buildability diagnostics. +pub(crate) fn execution_origins( + project: &Bound<'_, PyAny>, +) -> PyResult> { + let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); for test in project_reads::sql_tests(project)?.try_iter()? { - let test: Bound<'py, PyAny> = test?; + let test = test?; let test_name: String = native_field(&test, "name")?.extract()?; for target_key in native_field(&test, "scope_deps")?.try_iter()? { - let target_key: Bound<'py, PyAny> = target_key?; - let target_name: String = native_field(&target_key, "name")?.extract()?; - origins.set_item( - (target_key, native_field(&test, "key")?), - format!("SQL test '{test_name}' runs before '{target_name}'"), - )?; + let target_key = native_key(&target_key?)?; + let target_name = &target_key.1; + let description = format!("SQL test '{test_name}' runs before '{target_name}'"); + origins.push(( + (target_key, native_key(&native_field(&test, "key")?)?), + description, + )); } } let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let cache: KeyCache<'py> = KeyCache::new(py)?; - let lineage: Bound<'py, PyAny> = native_field(project, "lineage_graph")?; - let is_lineage = |gated: &(String, String), read: &(String, String)| -> PyResult { - lineage - .call_method1("upstream", (gated.clone(),))? - .contains(read.clone()) + let lineage = native_field(project, "lineage_graph")?; + let lineage = lineage.cast::()?.borrow(); + let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { + edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) }; + let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); for GateRow { audit_name, target, @@ -207,16 +175,13 @@ pub(crate) fn build_execution_edge_origins<'py>( read, } in &gates { - if is_lineage(gated, read)? { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { continue; } - let _ = origins.call_method1( - "setdefault", - ( - (cache.key(gated)?, cache.key(read)?), - format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), - ), - )?; + origins.push(( + (gated.clone(), read.clone()), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + )); } for HookRow { hook_name, @@ -226,25 +191,47 @@ pub(crate) fn build_execution_edge_origins<'py>( .. } in &hooks { - if is_lineage(gated, read)? { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { continue; } - let _ = origins.call_method1( - "setdefault", - ( - (cache.key(gated)?, cache.key(read)?), - format!( - "{} on '{}' reads '{}'", - hook_label(hook_name, kind), - gated.1, - read.1 - ), + origins.push(( + (gated.clone(), read.clone()), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 ), - )?; + )); } Ok(origins) } +/// Stable native execution order, projecting no Python graph or key objects. +pub(crate) fn execution_order( + py: Python<'_>, + upstream: &[(ObjectKey, Vec)], + origins: &[((ObjectKey, ObjectKey), String)], +) -> PyResult> { + let key = |(kind, name): &ObjectKey| GraphKey { + kind: kind.clone(), + name: name.clone(), + }; + let edges: Vec<_> = upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) + .collect(); + let origins: HashMap<(GraphKey, GraphKey), String> = origins + .iter() + .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) + .collect(); + Ok(topological_order(&edges, &origins) + .map_err(|message| planner_input_error(py, message))? + .into_iter() + .map(|key| (key.kind, key.name)) + .collect()) +} + /// Downstream edges keyed by upstream key, each list sorted by `(resource_type, name)`. #[pyfunction] pub(crate) fn build_downstream_deps<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 9357c6f72e..bf4b2868d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_quer use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, }; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -150,18 +151,16 @@ pub(crate) fn direct_model_version_identities<'py>( } let model_metadata: Bound<'py, PyDict> = PyDict::new(py); let model_local: Bound<'py, PyDict> = PyDict::new(py); - let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; - for key in native_field(scope, "execution_order")?.try_iter()? { - let key: Bound<'py, PyAny> = key?; - let order_key: GraphKey = graph_key(&key)?; + let scope = scope_data(scope)?; + for key in &scope.order { + let order_key = GraphKey::from(key); graph.execution_order.push(order_key.clone()); if order_key.kind != MODEL_RESOURCE { continue; } - let model: Bound<'py, PyAny> = mapping_get(&models_by_name, &order_key.name)?; - if model.is_none() { + let Some(model) = scope.model(py, &order_key.name) else { continue; - } + }; let name: String = model_reads::model_name(&model)?; let metadata_json: String = model_version_identity_metadata_json( &model, @@ -229,21 +228,17 @@ pub(crate) fn with_honest_model_write_hashes<'py>( let snapshot: Bound<'py, PyAny> = arguments.value("snapshot")?; let changes: Bound<'py, PyAny> = arguments.value("changes")?; let identities: Bound<'py, PyAny> = arguments.value("version_identities")?; - let selected_python: Bound<'py, PyAny> = native_field(&scope, "selected_keys")?; - let selected: HashSet = graph_keys(&selected_python)?.into_iter().collect(); + let scope = scope_data(&scope)?; + let selected: HashSet = scope.selected.iter().map(GraphKey::from).collect(); let function_hashes: Vec<(String, String)> = string_pairs(&native_field(&identities, "function_local_hashes")?)?; let mut graph = IdentityGraph::default(); let mut base_hashes: Vec<(GraphKey, String)> = Vec::new(); let mut function_deps: HashMap> = HashMap::new(); - for item in native_field(&scope, "upstream_deps")? - .call_method0("items")? - .try_iter()? - { - let (key, deps): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - let key: GraphKey = graph_key(&key)?; + for (key, deps) in &scope.upstream { + let key = GraphKey::from(key); if key.kind == UDF_RESOURCE || key.kind == TABLE_FN_RESOURCE { - function_deps.insert(key, graph_keys(&deps)?); + function_deps.insert(key, deps.iter().map(GraphKey::from).collect()); } } for (name, function_hash) in &function_hashes { @@ -282,11 +277,8 @@ pub(crate) fn with_honest_model_write_hashes<'py>( base_hashes.push((seed_key, write_hash)); } let local_hashes: Bound<'py, PyAny> = native_field(&identities, "model_local_hashes")?; - for model in native_field(&scope, "models_by_name")? - .call_method0("values")? - .try_iter()? - { - let model: Bound<'py, PyAny> = model?; + for model in scope.models.values() { + let model = model.bind(py); let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), @@ -311,7 +303,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( base_hashes.push((key_of(MODEL_RESOURCE, &name), model_hash)); } } - graph.execution_order = graph_keys(&native_field(&scope, "execution_order")?)?; + graph.execution_order = scope.order.iter().map(GraphKey::from).collect(); let write_hashes: HashMap = graph_write_identity_hashes(&graph, &selected, base_hashes) .into_iter() @@ -322,17 +314,14 @@ pub(crate) fn with_honest_model_write_hashes<'py>( .downcast::()? .as_mapping(), )?; - let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; - for key in native_field(&scope, "execution_order")?.try_iter()? { - let key: Bound<'py, PyAny> = key?; - let order_key: GraphKey = graph_key(&key)?; + for key in &scope.order { + let order_key = GraphKey::from(key); if order_key.kind != MODEL_RESOURCE || !selected.contains(&order_key) { continue; } - let model: Bound<'py, PyAny> = mapping_get(&models_by_name, &order_key.name)?; - if model.is_none() { + let Some(model) = scope.model(py, &order_key.name) else { continue; - } + }; let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; if mapping_get(&local_hashes, &name)?.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index 7af8fb8aff..b81350512c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -20,6 +20,7 @@ use sqlbuild_analysis::planner_identity::models::{ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; use crate::bindings::_helpers::planner_identity::hashing::graph_key; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; @@ -39,17 +40,6 @@ fn own_change(change: &Bound<'_, PyAny>) -> PyResult { Ok(OWN_CHANGE_KINDS.contains(&kind.as_str())) } -fn names_of_kind(keys: &Bound<'_, PyAny>, kind: &str) -> PyResult> { - let mut names: HashSet = HashSet::new(); - for key in keys.try_iter()? { - let key: GraphKey = graph_key(&key?)?; - if key.kind == kind { - names.insert(key.name); - } - } - Ok(names) -} - fn upstream_graph(upstream_deps: &Bound<'_, PyAny>) -> PyResult>> { let mut deps: HashMap> = HashMap::new(); for item in mapping_items(upstream_deps)?.try_iter()? { @@ -93,15 +83,15 @@ fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult function_local_hashes: py.None().into_bound(py), hook_version_hashes: py.None().into_bound(py), }; - let models_by_name: Bound<'_, PyAny> = native_field(&inputs.original_scope, "models_by_name")?; - for item in mapping_items(&models_by_name)?.try_iter()? { - let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; - let selected_change: Bound<'_, PyAny> = mapping_get(&models, &name)?; + let scope = scope_data(&inputs.original_scope)?; + for (name, model) in &scope.models { + let model = model.bind(py); + let selected_change: Bound<'_, PyAny> = mapping_get(&models, name)?; if !selected_change.is_none() && (inputs.model_changes_complete || own_change(&selected_change)?) { if own_change(&selected_change)? { - changed.insert(name); + changed.insert(name.clone()); } continue; } @@ -112,12 +102,12 @@ fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult &pass, &model, ( - mapping_get(&version_hashes, &name)?.extract()?, - mapping_get(&metadata_jsons, &name)?.extract()?, + mapping_get(&version_hashes, name)?.extract()?, + mapping_get(&metadata_jsons, name)?.extract()?, ), )?; if own_change(&change)? { - changed.insert(name); + changed.insert(name.clone()); } } Ok(changed) @@ -137,7 +127,7 @@ fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> } let fingerprint: Bound<'_, PyAny> = mapping_get(&recorded, &name)?; if fingerprint.is_none() || !native_field(&fingerprint, "version_hash")?.eq(&expected)? { - changed.insert(name); + changed.insert(name.clone()); } } Ok(changed) @@ -203,9 +193,15 @@ pub(crate) fn build_stale_out_of_selection_warnings<'py>( identities: arguments.value("version_identities")?, model_changes_complete: arguments.flag("model_changes_complete")?, }; - let run_keys: Bound<'py, PyAny> = arguments - .value("execution_scope")? - .getattr("selected_keys")?; + let run = scope_data(&arguments.value("execution_scope")?)?; + let original = scope_data(&inputs.original_scope)?; + let names = |kind: &str| -> HashSet { + run.selected + .iter() + .filter(|key| key.0 == kind) + .map(|key| key.1.clone()) + .collect() + }; let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; let mut changed_sources: HashSet = HashSet::new(); if include_sources && !source_freshness.is_none() { @@ -214,17 +210,26 @@ pub(crate) fn build_stale_out_of_selection_warnings<'py>( } } let graph = StalenessGraph { - upstream_deps: upstream_graph(&native_field(&inputs.original_scope, "upstream_deps")?)?, - selected_model_names: names_of_kind( - &native_field(&inputs.original_scope, "selected_keys")?, - MODEL_RESOURCE, - )? - .into_iter() - .collect(), + upstream_deps: original + .upstream + .iter() + .map(|(key, deps)| { + ( + GraphKey::from(key), + deps.iter().map(GraphKey::from).collect(), + ) + }) + .collect(), + selected_model_names: original + .selected + .iter() + .filter(|key| key.0 == MODEL_RESOURCE) + .map(|key| key.1.clone()) + .collect(), run: StalenessNames { - models: names_of_kind(&run_keys, MODEL_RESOURCE)?, - seeds: names_of_kind(&run_keys, SEED_RESOURCE)?, - sources: names_of_kind(&run_keys, SOURCE_RESOURCE)?, + models: names(MODEL_RESOURCE), + seeds: names(SEED_RESOURCE), + sources: names(SOURCE_RESOURCE), }, changed: StalenessNames { models: changed_model_names(&inputs)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index e9fbd85573..43b0a04fa1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -2,10 +2,11 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migration_ref_names}; @@ -83,38 +84,29 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Attach persisted migration identities through the existing cache boundary. pub(crate) fn with_migration_fingerprints<'py>( - py: Python<'py>, - options: Option>, + entries: &Bound<'py, PyAny>, + models: &ModelLookup, + cache: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, ) -> PyResult> { use crate::bindings::_helpers::boundary::python_calls::call_with; - let arguments = KeywordArguments::read( - py, - "with_migration_fingerprints", - options, - ( - &["entries", "models_by_name", "fingerprints", "dialect"], - &[], - ), - )?; - let cache = arguments.value("fingerprints")?; + let py = entries.py(); let mut attached: Vec> = Vec::new(); - for entry in arguments.value("entries")?.try_iter()? { + for entry in entries.try_iter()? { let entry = entry?; - let model = arguments - .value("models_by_name")? - .call_method1("get", (native_field(&entry, "name")?,))?; + let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; let metadata = native_field(&entry, "fingerprint_metadata_json")?; if model.is_none() || metadata.is_none() || native_field(&entry, "action")?.eq("skip")? { attached.push(entry); continue; } let fingerprint = call_with( - &native_field(&cache, "fingerprint")?, + &native_field(cache, "fingerprint")?, &[ ("query_sql", model_reads::query_sql(&model)?), ("metadata_json", metadata), ("ref_identities", PyDict::new(py).into_any()), - ("dialect", arguments.value("dialect")?), + ("dialect", dialect.clone()), ], )?; let kwargs = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 8fc86e8ea3..a34ab70b03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,21 +1,22 @@ //! Selected-scope buildability validation before change planning. -use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::planner_identity::graph::build_execution_edge_origins; -use crate::bindings::_helpers::planner_identity::graph::check_buildability; -use crate::bindings::_helpers::planner_identity::graph::missing_upstream_message; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::planner_identity::graph::execution_origins; use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error}; +use crate::bindings::_helpers::planner_phases::objects::planner_error; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; +use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; +use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; pub(crate) fn check_selected_scope_buildability<'py>( runtime: &Bound<'py, PyAny>, @@ -26,50 +27,47 @@ pub(crate) fn check_selected_scope_buildability<'py>( let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let mut external_seeds: Vec<_> = Vec::new(); + let scope = scope_data(&selected_scope)?; + let mut satisfied_keys: std::collections::HashSet = std::collections::HashSet::new(); for seed in project_reads::seeds(&project)?.try_iter()? { let seed = seed?; if seed_reads::external(&seed)?.is_truthy()? { - external_seeds.push(seed_reads::key(&seed)?); + satisfied_keys.insert(GraphKey::from(&native_key(&seed_reads::key(&seed)?)?)); } } - let missing = object( - py, - check_buildability( - py, - Some(keywords( - py, - &[ - ( - "selected_keys", - native_field(&selected_scope, "selected_keys")?, - ), - ( - "upstream_deps", - native_field(&selected_scope, "upstream_deps")?, - ), - ( - "snapshot", - warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, - ), - ( - "deferred_relations", - native_field(deferral, "deferred_relations")?, - ), - ("satisfied_keys", frozenset(py, &external_seeds)?), - ], - )?), - )?, - )?; - if missing.is_truthy()? { - let message: String = object( - py, - missing_upstream_message( - &missing, - &object(py, build_execution_edge_origins(&project)?)?, - )?, - )? - .extract()?; + let snapshot = warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?; + let names = |value: Bound<'py, PyAny>| -> PyResult> { + value.try_iter()?.map(|value| value?.extract()).collect() + }; + let deferred = native_field(deferral, "deferred_relations")?; + let available = AvailableUpstreams { + satisfied_keys, + existing_relations: names(native_field(&snapshot, "existing_relations")?)?, + deferred_relations: if deferred.is_none() { + None + } else { + Some(names(deferred)?) + }, + }; + let selected: Vec<_> = scope.selected.iter().map(GraphKey::from).collect(); + let deps: std::collections::HashMap> = scope + .upstream + .iter() + .map(|(key, deps)| { + ( + GraphKey::from(key), + deps.iter().map(GraphKey::from).collect(), + ) + }) + .collect(); + let missing = missing_upstreams(&selected, &deps, &available); + if !missing.is_empty() { + let origins: std::collections::HashMap<(GraphKey, GraphKey), String> = + execution_origins(&project)? + .into_iter() + .map(|((key, read), label)| ((GraphKey::from(&key), GraphKey::from(&read)), label)) + .collect(); + let message = missing_upstream_message(&missing, &origins); return Err(planner_error( py, &message, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs index 7525a1875b..f665c6b7b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -7,6 +7,7 @@ use pyo3::{PyClass, pyclass::boolean_struct::True}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerScopeResolution, }; +use crate::bindings::_helpers::planner_scope::retained::project_scope; #[pymethods] impl PlannerScopeResolution { @@ -23,6 +24,21 @@ impl PlannerScopeResolution { inspection_scope, } } + + #[getter] + fn selected_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.selected_scope.bind(py)) + } + + #[getter] + fn stale_warning_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.stale_warning_scope.bind(py)) + } + + #[getter] + fn inspection_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.inspection_scope.bind(py)) + } } #[pymethods] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs index a5ec1af29e..6771762125 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs @@ -6,11 +6,16 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerRuntime, PlannerScopeResolution, PlannerWarehouseState, }; +use crate::bindings::_helpers::planner_scope::retained::RetainedScope; use crate::bindings::_helpers::records::getters::class_of; use crate::bindings::_helpers::records::models::GetterEntry; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_retained_scope, + }, GetterEntry { class: class_of::, read: read_planner_identity_context, @@ -29,6 +34,16 @@ pub(crate) const GETTERS: &[GetterEntry] = &[ }, ]; +fn read_retained_scope<'py>( + value: &Bound<'py, PyAny>, + name: &str, +) -> Option>> { + let Ok(cell) = value.cast::() else { + return None; + }; + Some(cell.get().data.field(value.py(), name)) +} + fn read_planner_identity_context<'py>( value: &Bound<'py, PyAny>, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 0d3ade77dd..eb83aaaada 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -8,7 +8,8 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; +use crate::bindings::_helpers::planner_phases::objects::native; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; @@ -18,6 +19,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{Py, PyRefMut, Python}; +use sqlbuild_analysis::planner_scope::constants::MODEL; use std::collections::{BTreeMap, HashSet}; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); @@ -159,37 +161,27 @@ pub(crate) fn detect_stale_warning_changes<'py>( native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?, )?; let stale_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?; - let mut remaining: Vec<_> = Vec::new(); - for key in native_field(&stale_scope, "all_keys")? - .call_method0("values")? - .try_iter()? - { - let key = key?; - if native_field(&key, "resource_type")?.eq("model")? - && !reusable.contains(&native_field(&key, "name")?.extract::()?) - { - remaining.push(key); - } - } + let stale_scope = scope_data(&stale_scope)?; + let remaining: HashSet<_> = stale_scope + .names + .iter() + .map(|(_, key)| key) + .filter(|key| key.0 == MODEL && !reusable.contains(&key.1)) + .cloned() + .collect(); let stale_identities = class_field(identities, |i: &PlannerIdentityContext| { &i.stale_warning_identities })?; { let mut work = identity_work(py)?; for key in &remaining { - *work - .stale_detections - .entry(native_field(key, "name")?.extract()?) - .or_default() += 1; + *work.stale_detections.entry(key.1.clone()).or_default() += 1; } } let remaining_changes = scope_model_changes( ( project, - &replace( - &stale_scope, - &[("selected_keys", frozenset(py, &remaining)?)], - )?, + &stale_scope.with_selected(py, remaining).object(py)?, snapshot, ), Some(true), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index ced5ea9dd5..fc33a2e8b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -91,7 +91,7 @@ pub(crate) struct PlannerWarehouseState { } /// Resolved planner scopes for one plan build. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] pub(crate) struct PlannerScopeResolution { pub(crate) selected_scope: Py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 78e3c4b3f2..8b0143ae88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::model_config::typed_config::{config_text, model_v use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{enum_member, new_record, new_record_with}; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -117,18 +116,6 @@ pub(crate) fn frozenset<'py>( Ok(PyFrozenSet::new(py, items)?.into_any()) } -/// Names of model keys among `keys`. -pub(crate) fn model_key_names(keys: &Bound<'_, PyAny>) -> PyResult> { - let mut names: Vec<_> = Vec::new(); - for key in keys.try_iter()? { - let key = key?; - if native_field(&key, "resource_type")?.eq("model")? { - names.push(native_field(&key, "name")?.extract()?); - } - } - Ok(names) -} - /// `sorted(items)` as a tuple of strings. pub(crate) fn sorted_names<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index f8405088ed..2a0726e922 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,8 +1,7 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. use crate::bindings::_helpers::boundary::python_calls::keywords; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; +use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; @@ -21,6 +20,7 @@ use crate::bindings::_helpers::planner_phases::objects::{ use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -28,6 +28,7 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::planner_scope::constants::MODEL; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -172,35 +173,14 @@ pub(crate) fn add_storage_policies<'py>( let py = runtime.py(); let mut entries = output.field("model_entries")?; if native_field(policies, "record_migration_fingerprints")?.is_truthy()? { - let models = PyDict::new(py); - for model in project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)? - .try_iter()? - { - let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; - } + let scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; entries = object( py, with_migration_fingerprints( - py, - Some(keywords( - py, - &[ - ("entries", entries), - ("models_by_name", models.into_any()), - ( - "fingerprints", - warehouse_field(warehouse, |warehouse| { - &warehouse.migration_fingerprints - })?, - ), - ( - "dialect", - native_field(runtime, "adapter")? - .call_method0("sql_analysis_dialect")?, - ), - ], - )?), + &entries, + &ModelLookup::Native(scope_data(&scope)?), + &warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, + &native_field(runtime, "adapter")?.call_method0("sql_analysis_dialect")?, )?, )?; } @@ -303,22 +283,16 @@ pub(crate) fn add_plan_warnings<'py>( /// The unselected models the selected scope reads, sorted. pub(crate) fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { - let py = scopes.py(); let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let selected_keys = native_field(&selected, "selected_keys")?; - let upstream_deps = native_field(&selected, "upstream_deps")?; - let model_type = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; + let selected = crate::bindings::_helpers::planner_scope::retained::scope_data(&selected)?; let mut names: Vec = Vec::new(); - for key in selected_keys.try_iter()? { - let Some(deps) = upstream_deps.downcast::()?.get_item(key?)? else { + for (key, deps) in &selected.upstream { + if !selected.selected.contains(key) { continue; - }; - for dep in deps.try_iter()? { - let dep = dep?; - if native_field(&dep, "resource_type")?.eq(&model_type)? - && !selected_keys.contains(&dep)? - { - names.push(native_field(&dep, "name")?.extract()?); + } + for dep in deps { + if dep.0 == MODEL && !selected.selected.contains(dep) { + names.push(dep.1.clone()); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 17b2e0d419..f6b5bf0ede 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -14,10 +14,11 @@ use crate::bindings::_helpers::planner_phases::constants::{ TRANSIENT_TABLE_TYPE, }; use crate::bindings::_helpers::planner_phases::objects::{ - construct, member, model_key_names, native, native_member, optional_text, planner_error, - planner_model, replace, sorted_names, text, tuple, + construct, member, native, native_member, optional_text, planner_error, planner_model, replace, + sorted_names, text, tuple, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -57,15 +58,16 @@ fn selected_models<'py>( runtime: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, ) -> PyResult>> { - let names: BTreeSet = model_key_names(&native_field(scope, "selected_keys")?)? - .into_iter() - .collect(); + let scope = scope_data(scope)?; let mut models: Vec<_> = Vec::new(); for model in project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let model = model?; - if names.contains(&model_reads::model_name(&model)?) { + if scope + .selected + .contains(&("model".to_owned(), model_reads::model_name(&model)?)) + { models.push(model); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 3e6c586595..7d9ebcab70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,16 +1,14 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::boundary::python_calls::keywords; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{replace, tuple}; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::records::fields::field as native_field; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::PyTuple; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] @@ -55,53 +53,20 @@ pub(crate) fn resolve_planner_scopes<'py>( ], )?), )?; - let stale_keys = object( - py, - transitive_closure_many( - &native_field(&selected, "selected_keys")?, - &native_field(&full, "upstream_deps")?, - true, - None, - )?, - )?; - let all_keys = PyDict::new(py); - for (name, key) in native_field(&full, "all_keys")? - .downcast::()? - .iter() - { - if stale_keys.contains(&key)? { - all_keys.set_item(name, key)?; - } - } - let models = PyDict::new(py); - for (name, model) in native_field(&full, "models_by_name")? - .downcast::()? - .iter() - { - if stale_keys.contains(model_reads::model_key(&model)?)? { - models.set_item(name, model)?; - } - } - let mut order: Vec<_> = Vec::new(); - for key in native_field(&full, "execution_order")?.try_iter()? { - let key = key?; - if stale_keys.contains(&key)? { - order.push(key); - } - } - let stale_scope = replace( - &full, - &[ - ("all_keys", all_keys.into_any()), - ("models_by_name", models.into_any()), - ("selected_keys", native_field(&selected, "selected_keys")?), - ("execution_order", tuple(py, order)?), - ( - "user_selected_keys", - native_field(&selected, "user_selected_keys")?, - ), - ], - )?; + let selected_data = scope_data(&selected)?; + let full_data = scope_data(&full)?; + let stale_keys = reachable_keys( + &selected_data.selected.iter().cloned().collect::>(), + &full_data.upstream, + ); + let mut stale = full_data.with_selected(py, selected_data.selected.clone()); + stale.names.retain(|(_, key)| stale_keys.contains(key)); + stale + .models + .retain(|name, _| stale_keys.contains(&("model".to_owned(), name.clone()))); + stale.order.retain(|key| stale_keys.contains(key)); + stale.user_selected = selected_data.user_selected.clone(); + let stale_scope = stale.object(py)?; Ok(Bound::new( py, PlannerScopeResolution { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index ad98aa1104..2553cb965a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,5 +1,6 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. +use crate::bindings::_helpers::graph::views::compiled_graph_keys; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; @@ -14,6 +15,7 @@ use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -80,15 +82,16 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ], )?; let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; - let stale_keys = pyo3::types::PyFrozenSet::new( + let stale_scope = scope_data(&class_field(scopes, |s: &PlannerScopeResolution| { + &s.stale_warning_scope + })?)?; + let stale_keys = compiled_graph_keys( py, - native_field( - &class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, - "all_keys", - )? - .call_method0("values")? - .try_iter()? - .collect::>>()?, + stale_scope + .names + .iter() + .map(|(_, key)| key.clone()) + .collect(), )? .into_any(); let snapshot = warehouse_snapshot(&SnapshotRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 4397d1a1fa..94772a0809 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -5,11 +5,11 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; +use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; -use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; +use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_scenario::constants::{ @@ -99,15 +99,15 @@ pub(crate) fn build_scenario_execution_plan<'py>( let models = by_attribute(&project_reads::models(&project)?, "name")?; let functions = by_attribute(&project_reads::functions(&project)?, "key")?; let locations = build_function_locations(&project_reads::functions(&project)?)?.into_any(); - let upstream = build_execution_upstream_deps(&project)?.into_any(); - let ordered = object(py, topologically_order_keys(&upstream, None)?)?; + let upstream = execution_deps(&project)?; + let ordered = execution_order(py, &upstream, &[])?; + let ordered = KeyCache::new(py)?.keys(&ordered)?; let mut model_keys: Vec> = Vec::new(); - for key in ordered.try_iter()? { - let key = key?; - if native_field(&key, "resource_type")?.eq("model")? - && native_field(&graph, "model_names")?.contains(native_field(&key, "name")?)? + for key in &ordered { + if native_field(key, "resource_type")?.eq("model")? + && native_field(&graph, "model_names")?.contains(native_field(key, "name")?)? { - model_keys.push(key); + model_keys.push(key.clone()); } } let overrides = if analysis { @@ -118,9 +118,8 @@ pub(crate) fn build_scenario_execution_plan<'py>( let fixtures = fixture_plans(&scenario, &graph, &context, &overrides)?; let seeds = seed_entries(&project, &graph, &context.relations)?; let mut function_entries: Vec> = Vec::new(); - for key in ordered.try_iter()? { - let key = key?; - if native_field(&graph, "function_deps")?.contains(&key)? + for key in &ordered { + if native_field(&graph, "function_deps")?.contains(key)? && let Some(function) = functions.get_item(key)? { function_entries.push(function_entry(&function, &context, &locations, &columns)?); @@ -271,7 +270,7 @@ fn model_context<'py>( py, ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), - models_by_name: models.clone().into_any().unbind(), + models_by_name: models.clone().into_any().into(), functions_by_name: by_attribute(&project_reads::functions(project)?, "name")? .into_any() .unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs index e3dde684cf..346d6e3453 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::main::{ expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, }; @@ -71,11 +72,11 @@ pub(crate) fn loader_to_source_entry<'py>( pub(crate) fn intermediate_map<'py>( project: &Bound<'py, PyAny>, - selected: &Bound<'py, PyAny>, + selected: &[ObjectKey], upstream: bool, ) -> PyResult> { let graph = project_graph(project)?; - let names = intermediate_sources(&graph, &keys(selected)?, upstream); + let names = intermediate_sources(&graph, selected, upstream); let loaders = PyDict::new(project.py()); for loader in native_field(project, "loader_functions")?.try_iter()? { let loader = loader?; @@ -96,14 +97,14 @@ pub(crate) fn build_intermediate_source_map<'py>( project: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult> { - intermediate_map(project, selected_keys, false) + intermediate_map(project, &keys(selected_keys)?, false) } pub(crate) fn build_upstream_intermediate_source_map<'py>( project: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult> { - intermediate_map(project, selected_keys, true) + intermediate_map(project, &keys(selected_keys)?, true) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs index 3ad6d7b82e..ad41a18f00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod loaders; pub(crate) mod registration; +pub(crate) mod retained; pub(crate) mod scope; pub(crate) mod sources; -mod values; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs new file mode 100644 index 0000000000..3cc9315dae --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs @@ -0,0 +1,198 @@ +//! Native planner scope ownership, with projection only at public record boundaries. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyclass; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::views::edge_dict; +use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; +use crate::bindings::_helpers::planner_scope::values::{edges, keys}; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::{field, mapping_items}; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::models::NativeRecord; + +/// The graph and selection used throughout planning; model values remain compiled-resource views. +#[derive(Debug)] +pub(crate) struct ScopeData { + pub(crate) upstream: Vec<(ObjectKey, Vec)>, + pub(crate) downstream: Vec<(ObjectKey, Vec)>, + pub(crate) names: Vec<(String, ObjectKey)>, + pub(crate) models: HashMap, + pub(crate) model_order: Vec, + pub(crate) selected: HashSet, + pub(crate) order: Vec, + pub(crate) user_selected: HashSet, + pub(crate) python_reads: HashSet, + pub(crate) tests: Py, + pub(crate) load_sources: bool, + pub(crate) end_audit_sources: HashSet, +} + +/// Internal carrier, never returned in place of a supported public PlannerScope record. +#[pyclass(frozen)] +pub(crate) struct RetainedScope { + pub(crate) data: Arc, + projection: PyOnceLock>, +} + +impl ScopeData { + pub(crate) fn object(self, py: Python<'_>) -> PyResult> { + Ok(Bound::new( + py, + RetainedScope { + data: Arc::new(self), + projection: PyOnceLock::new(), + }, + )? + .into_any()) + } + + pub(crate) fn model<'py>(&self, py: Python<'py>, name: &str) -> Option> { + self.models.get(name).map(|model| model.bind(py)) + } + + pub(crate) fn with_selected(&self, py: Python<'_>, selected: HashSet) -> Self { + Self { + upstream: self.upstream.clone(), + downstream: self.downstream.clone(), + names: self.names.clone(), + model_order: self.model_order.clone(), + models: self + .models + .iter() + .map(|(name, model)| (name.clone(), model.clone_ref(py))) + .collect(), + selected, + order: self.order.clone(), + user_selected: self.user_selected.clone(), + python_reads: self.python_reads.clone(), + tests: self.tests.clone_ref(py), + load_sources: self.load_sources, + end_audit_sources: self.end_audit_sources.clone(), + } + } + + /// Public field projection; internal graph consumers read the native fields above instead. + pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let cache = KeyCache::new(py)?; + let key_set = |keys: &HashSet| -> PyResult> { + Ok( + PyFrozenSet::new(py, cache.keys(&keys.iter().cloned().collect::>())?)? + .into_any(), + ) + }; + match name { + "upstream_deps" => Ok(edge_dict(&cache, &self.upstream)?.into_any()), + "downstream_deps" => Ok(edge_dict(&cache, &self.downstream)?.into_any()), + "selected_keys" => key_set(&self.selected), + "user_selected_keys" => key_set(&self.user_selected), + "end_audit_source_keys" => key_set(&self.end_audit_sources), + "execution_order" => Ok(PyTuple::new(py, cache.keys(&self.order)?)?.into_any()), + "all_keys" => { + let values = PyDict::new(py); + for (name, key) in &self.names { + values.set_item(name, cache.key(key)?)?; + } + Ok(values.into_any()) + } + "models_by_name" => { + let values = PyDict::new(py); + for name in &self.model_order { + if let Some(model) = self.models.get(name) { + values.set_item(name, model.bind(py))?; + } + } + Ok(values.into_any()) + } + "python_read_source_names" => Ok(PyFrozenSet::new(py, &self.python_reads)?.into_any()), + "sql_test_selection" => Ok(self.tests.bind(py).clone().into_any()), + "load_sources" => self.load_sources.into_bound_py_any(py), + _ => Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), + } + } +} + +/// Read native ownership directly, or import a caller-authored public scope at the API edge. +pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { + if let Ok(scope) = value.cast::() { + return Ok(scope.get().data.clone()); + } + let mut models: HashMap = HashMap::new(); + let mut model_order: Vec = Vec::new(); + for item in mapping_items(&field(value, "models_by_name")?)?.try_iter()? { + let item = item?; + let name: String = item.get_item(0)?.extract()?; + model_order.push(name.clone()); + models.insert(name, ModelHandle::read(&item.get_item(1)?)); + } + Ok(Arc::new(ScopeData { + upstream: edges(&field(value, "upstream_deps")?)?, + downstream: edges(&field(value, "downstream_deps")?)?, + names: mapping_items(&field(value, "all_keys")?)? + .try_iter()? + .map(|item| { + let item = item?; + Ok(( + item.get_item(0)?.extract()?, + crate::bindings::_helpers::graph::keys::native_key(&item.get_item(1)?)?, + )) + }) + .collect::>()?, + models, + model_order, + selected: keys(&field(value, "selected_keys")?)?.into_iter().collect(), + order: keys(&field(value, "execution_order")?)?, + user_selected: keys(&field(value, "user_selected_keys")?)? + .into_iter() + .collect(), + python_reads: field(value, "python_read_source_names")?.extract()?, + tests: field(value, "sql_test_selection")?.extract()?, + load_sources: field(value, "load_sources")?.is_truthy()?, + end_audit_sources: keys(&field(value, "end_audit_source_keys")?)? + .into_iter() + .collect(), + })) +} + +/// Preserve public record identity and dataclass behavior on explicit projection. +pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index 16797c3c0b..4c41b1ba9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -4,49 +4,29 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::lineage_graph_names; +use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; use crate::bindings::_helpers::planner_identity::graph::{ - build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, - topologically_order_keys, + execution_deps, execution_order, execution_origins, }; -use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; -use crate::bindings::_helpers::planner_scope::values::{edges, keys}; +use crate::bindings::_helpers::planner_scope::retained::ScopeData; +use crate::bindings::_helpers::planner_scope::values::{keys, project_graph}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; -use crate::bindings::_helpers::records::construct::{new_record, new_record_with}; +use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_keys; +use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDictMethods; -use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyTuple}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_analysis::planner_scope::main::{ - managed_sources::managed_sources, source_selectors::source_selectors, + expand_loaders::expand_loaders, managed_sources::managed_sources, + source_selectors::source_selectors, }; use std::collections::HashSet; -pub(crate) fn managed_source_upstream_keys<'py>( - selected_keys: &Bound<'py, PyAny>, - upstream_deps: &Bound<'py, PyAny>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let mut managed: HashSet = HashSet::new(); - for source in project_reads::sources(project)?.try_iter()? { - let source = source?; - if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { - managed.insert(source_reads::name(&source)?.extract()?); - } - } - let selected = managed_sources(&keys(selected_keys)?, &edges(upstream_deps)?, &managed); - PyFrozenSet::new( - project.py(), - KeyCache::new(project.py())?.keys(&selected.into_iter().collect::>())?, - ) -} - pub(crate) fn build_planner_scope<'py>( py: Python<'py>, options: Option>, @@ -71,65 +51,72 @@ pub(crate) fn build_planner_scope<'py>( let select = default_tuple(&args.value("select")?); let exclude = default_tuple(&args.value("exclude")?); let auto_load = args.value("auto_load_sources")?.is_truthy()?; - let mut upstream = build_execution_upstream_deps(&project)?.into_any(); - let mut downstream = build_downstream_deps(&upstream)?.into_any(); + let mut upstream = execution_deps(&project)?; let lineage: Bound<'py, NativeProjectGraph> = native_field(&project, "lineage_graph")?.cast_into()?; - let all_keys = lineage_graph_names(py, lineage.borrow())?.into_any(); - let mut selected = args.value("selected_keys")?; - if selected.is_none() { - selected = resolve_graph_selection(lineage.borrow(), &select, &exclude)?.into_any(); - } - let names: Vec<(String, (String, String))> = mapping_items(&all_keys)? - .try_iter()? - .map(|item| { - let (name, key): (String, Bound<'py, PyAny>) = item?.extract()?; - Ok(( - name, - crate::bindings::_helpers::graph::keys::native_key(&key)?, - )) - }) - .collect::>()?; + let lineage = lineage.borrow(); + let selected = args.value("selected_keys")?; + let mut selected = if selected.is_none() { + resolve_graph_keys(lineage.graph(), (&select, &exclude))? + } else { + keys(&selected)? + }; + let names = &lineage.graph().names; let texts = py_strings(&select)?; let carrier = Carrier::new(py, &texts, || { Ok(names.iter().map(|(name, _)| name.clone()).collect()) })?; let sent = carrier.send_all(&texts)?; - let selected_names = keys(&selected)?; - let source_keys = source_selectors(&sent, &names, &selected_names, true) - .map_err(|error| carrier.error(error))?; - let named_sources = source_selectors(&sent, &names, &selected_names, false) - .map_err(|error| carrier.error(error))?; - let cache = KeyCache::new(py)?; - let explicit_sources: Vec<(String, String)> = source_keys.into_iter().collect(); - let mut executable = PyFrozenSet::new(py, cache.keys(&explicit_sources)?)?.into_any(); - if !auto_load { - executable = PyFrozenSet::empty(py)?.into_any(); - } + let source_keys = + source_selectors(&sent, names, &selected, true).map_err(|error| carrier.error(error))?; + let named_sources = + source_selectors(&sent, names, &selected, false).map_err(|error| carrier.error(error))?; + let mut executable = if auto_load { + source_keys + } else { + HashSet::new() + }; if auto_load { - let loaded = managed_source_upstream_keys(&selected, &upstream, &project)?; - selected = selected.call_method1("union", (&loaded,))?; - executable = executable.call_method1("union", (&loaded,))?; + let mut managed: HashSet = HashSet::new(); + for source in project_reads::sources(&project)?.try_iter()? { + let source = source?; + if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { + managed.insert(source_reads::name(&source)?.extract()?); + } + } + let loaded = managed_sources(&selected, &upstream, &managed); + let mut seen: HashSet = selected.iter().cloned().collect(); + selected.extend( + loaded + .iter() + .filter(|key| seen.insert((*key).clone())) + .cloned(), + ); + executable.extend(loaded); } - executable = executable.call_method1("intersection", (&selected,))?; - if auto_load || keys(&selected)?.iter().any(|key| key.0 == SOURCE) { - let (expanded, deps) = - expand_selected_loader_dependencies(&project, &selected, &upstream, Some(&executable))?; - selected = expanded.into_any(); - upstream = deps.into_any(); - downstream = build_downstream_deps(&upstream)?.into_any(); + let selected_set: HashSet<_> = selected.iter().collect(); + executable.retain(|key| selected_set.contains(key)); + if auto_load || selected.iter().any(|key| key.0 == SOURCE) { + (selected, upstream) = + expand_loaders(&project_graph(&project)?, selected, upstream, &executable); } - let end_audit_sources: Vec<(String, String)> = keys(&selected)? - .into_iter() + let end_audit_sources: HashSet = selected + .iter() .filter(|key| key.0 == SOURCE && (!auto_load || named_sources.contains(key))) + .cloned() .collect(); - let end_audit_sources = PyFrozenSet::new(py, cache.keys(&end_audit_sources)?)?.into_any(); - let origins = build_execution_edge_origins(&project)?.into_any(); - let order = topologically_order_keys(&upstream, Some(origins))?.into_any(); - let models = PyDict::new(py); + let order = execution_order(py, &upstream, &execution_origins(&project)?)?; + let selected: HashSet<_> = selected.into_iter().collect(); + let mut models: std::collections::HashMap = + std::collections::HashMap::new(); + let mut model_order: Vec = Vec::new(); for model in project_reads::models(&project)?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + let name: String = model_reads::name_object(&model)?.extract()?; + if !models.contains_key(&name) { + model_order.push(name.clone()); + } + models.insert(name, ModelHandle::read(&model)); } let mut reads = args.value("python_read_source_names")?; if reads.is_none() { @@ -139,26 +126,21 @@ pub(crate) fn build_planner_scope<'py>( if !tests.is_truthy()? { tests = new_record(py, RecordId::SqlTestSelection, (), None)?; } - new_record_with( - py, - RecordId::PlannerScope, - &[ - ("upstream_deps", upstream), - ("downstream_deps", downstream), - ("all_keys", all_keys), - ("models_by_name", models.into_any()), - ("selected_keys", selected.clone()), - ("execution_order", order), - ("user_selected_keys", selected), - ("python_read_source_names", reads), - ("sql_test_selection", tests), - ( - "load_sources", - PyBool::new(py, auto_load).to_owned().into_any(), - ), - ("end_audit_source_keys", end_audit_sources), - ], - ) + ScopeData { + downstream: downstream_edges(&upstream), + upstream, + names: names.clone(), + models, + model_order, + user_selected: selected.clone(), + selected, + order, + python_reads: reads.extract()?, + tests: tests.extract()?, + load_sources: auto_load, + end_audit_sources, + } + .object(py) } fn default_tuple<'py>(value: &Bound<'py, PyAny>) -> Bound<'py, PyAny> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs index a1376c22a4..eb6e863918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs @@ -4,15 +4,18 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; +use crate::bindings::_helpers::planner_scope::values::{edges, keys}; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::PyDictMethods; use pyo3::types::{PyBool, PyDict, PyTuple}; -use pyo3::types::{PyDictMethods, PySetMethods}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_analysis::planner_scope::constants::SOURCE; #[pyfunction] @@ -22,20 +25,25 @@ pub(crate) fn build_source_load_map<'py>( selected_keys: &Bound<'py, PyAny>, upstream_deps: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let relevant = upstream_deps - .map(|edges| { - closure( - selected_keys.try_iter()?.collect::>>()?, - edges, - (true, None), - ) - }) - .transpose()?; + source_load_map( + project, + &keys(selected_keys)?, + upstream_deps.map(edges).transpose()?.as_deref(), + ) +} + +/// Select source entries from native reachability; the resulting entries cross the executor edge. +pub(crate) fn source_load_map<'py>( + project: &Bound<'py, PyAny>, + selected: &[ObjectKey], + upstream_deps: Option<&[(ObjectKey, Vec)]>, +) -> PyResult> { + let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); let result = PyDict::new(project.py()); for source in project_reads::sources(project)?.try_iter()? { let source = source?; if let Some(keys) = &relevant - && !keys.contains(source_reads::key(&source)?)? + && !keys.contains(&native_key(&source_reads::key(&source)?)?) { continue; } @@ -43,7 +51,7 @@ pub(crate) fn build_source_load_map<'py>( result.set_item(native_field(&entry, "name")?, entry)?; } for upstream in [false, true] { - for (key, entry) in intermediate_map(project, selected_keys, upstream)?.iter() { + for (key, entry) in intermediate_map(project, selected, upstream)?.iter() { result.set_item(key, entry)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 9fafbd54dd..52a86f9aed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -94,19 +94,25 @@ fn resolved<'py>( graph: &ProjectGraph, (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { + compiled_keys(select.py(), &resolve_graph_keys(graph, (select, exclude))?) +} + +/// Resolve graph keys for native planning without building a public frozenset. +pub(crate) fn resolve_graph_keys<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { let py: Python<'py> = select.py(); let select: Vec> = py_strings(select)?; let exclude: Vec> = py_strings(exclude)?; let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; - selected( - &carrier, - resolve_selectors( - graph, - &carrier.send_all(&select)?, - &carrier.send_all(&exclude)?, - ), + resolve_selectors( + graph, + &carrier.send_all(&select)?, + &carrier.send_all(&exclude)?, ) + .map_err(|error| carrier.error(error)) } fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 8876606089..901247329e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -1,21 +1,28 @@ //! Assemble audit plan dataclasses using native rendering and scheduling. +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::models::GraphIndexes; +use std::collections::HashSet; + use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::auditing::render::render_audit_sql; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::audit_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::views::project_lineage_views; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_scope::retained::scope_data; +use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::audits::{ - reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, + attachment, outside_target_lineage, resolve_effective_run_scope, }; -use crate::bindings::_helpers::records::construct::{enum_from_value, new_record, new_record_with}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use crate::bindings::_helpers::records::construct::new_record; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, -}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::python_calls::imported; @@ -87,8 +94,14 @@ pub(crate) fn plan_audit_of<'py>( source_map: request.get("source_map")?, adapter: request.get("adapter")?, model_materializations: request.get("model_materializations")?, - lineage_upstream_deps: request.get("lineage_upstream_deps")?, - lineage_downstream_deps: request.get("lineage_downstream_deps")?, + lineage: Py::new( + request.py(), + NativeProjectGraph::new(graph_from_indexes(GraphIndexes { + upstream: edges(&request.get("lineage_upstream_deps")?)?, + downstream: edges(&request.get("lineage_downstream_deps")?)?, + ..GraphIndexes::default() + })), + )?, }, ) } @@ -116,12 +129,15 @@ fn plan_audit<'py>( &format!("audit '{name}' planned evidence SQL"), )? }; - let attachment: Bound<'_, PyTuple> = - resolve_attachment_kind(&audit, &request.lineage_downstream_deps)?; - let kind = attachment.get_item(0)?; - let target = attachment.get_item(1)?; - let kind_text: String = kind.extract()?; - let materialization = if kind_text == MODEL_ATTACHMENT && !target.is_none() { + let lineage = request.lineage.bind(py).borrow(); + let (kind_text, target_name) = attachment(&audit, lineage.graph())?; + let kind = audit_enum( + PyString::new(py, &kind_text).into_any(), + "AuditAttachmentKind", + "end", + )?; + let target = target_name.as_deref().into_pyobject(py)?.into_any(); + let materialization = if kind_text == MODEL_ATTACHMENT && target_name.is_some() { mapping_get(&request.model_materializations, &target)? } else { py.None().into_bound(py) @@ -179,12 +195,9 @@ fn plan_audit<'py>( ("attached_target_name", target), ( "reads_outside_target_lineage", - PyBool::new( - py, - reads_outside_target_lineage(&audit, &request.lineage_upstream_deps)?, - ) - .to_owned() - .into_any(), + PyBool::new(py, outside_target_lineage(&audit, lineage.graph())?) + .to_owned() + .into_any(), ), ] { values.set_item(field, value)?; @@ -195,25 +208,21 @@ fn plan_audit<'py>( /// Select attached audits by their logical target and singular audits by dependency overlap. pub(crate) fn audit_is_selected( audit: &Bound<'_, PyAny>, - selected_keys: &Bound<'_, PyAny>, + selected_keys: &HashSet, ) -> PyResult { let py = audit.py(); let kind = audit_reads::attached_target_kind(audit)?; let name = audit_reads::attached_target_name(audit)?; if !kind.is_none() && !name.is_none() { - let target_value = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")? - .call1((kind,))? - .getattr("value")?; - let resource_type = enum_from_value(py, RecordId::CompiledResourceType, target_value)?; - let key = new_record_with( - py, - RecordId::CompiledObjectKey, - &[("resource_type", resource_type), ("name", name)], + let target_value = native_field( + &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, + "value", )?; - return selected_keys.contains(key); + let key = (target_value.extract()?, name.extract()?); + return Ok(selected_keys.contains(&key)); } for dependency in audit_reads::scope_deps(audit)?.try_iter()? { - if selected_keys.contains(dependency?)? { + if selected_keys.contains(&native_key(&dependency?)?) { return Ok(true); } } @@ -227,11 +236,11 @@ pub(crate) fn build_selected_audit_entries<'py>( let request = Request::new("build_selected_audit_entries", request)?; let py = request.py(); let project = request.get("project")?; - let selected_keys = native_field(&request.get("scope")?, "selected_keys")?; + let scope = scope_data(&request.get("scope")?)?; let mut selected: Vec> = Vec::new(); for audit in project_reads::audits(&project)?.try_iter()? { let audit = audit?; - if audit_is_selected(&audit, &selected_keys)? { + if audit_is_selected(&audit, &scope.selected)? { selected.push(audit); } } @@ -239,7 +248,6 @@ pub(crate) fn build_selected_audit_entries<'py>( if selected.is_empty() { return Ok(entries); } - let lineage = project_lineage_views(py, &project)?; let relations = request.get("relations")?; let planning = AuditPlanning { model_locations: native_field(&relations, "model_locations")?, @@ -247,8 +255,7 @@ pub(crate) fn build_selected_audit_entries<'py>( source_map: native_field(&relations, "source_read_map")?, adapter: request.get("adapter")?, model_materializations: request.get("model_materializations")?, - lineage_upstream_deps: native_field(&lineage, "upstream_deps")?, - lineage_downstream_deps: native_field(&lineage, "downstream_deps")?, + lineage: native_field(&project, "lineage_graph")?.extract()?, }; for audit in selected { entries.append(plan_audit(&audit, &planning)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 78c6ff857a..c1f063feba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -1,9 +1,15 @@ //! Audit attachment, target lineage and run scope. +use std::cell::RefCell; +use std::collections::HashMap; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PySet, PySetMethods, PyString, PyTuple}; +use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::main::reachable_graph_keys::reachable_graph_keys; +use sqlbuild_analysis::graph::models::{EdgeDirection, GraphIndexes, ProjectGraph}; use sqlbuild_analysis::planner_strategy::constants::{MODEL_REFERENCE_KIND, SOURCE_REFERENCE_KIND}; use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_run_scope; use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; @@ -11,32 +17,17 @@ use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunS use crate::bindings::_helpers::boundary::python_calls::imported; use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::graph::keys::native_key; +use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; -use crate::bindings::_helpers::records::construct::{enum_from_value, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::records::ids::ErrorId; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const MODEL_RESOURCE_TYPE: &str = "model"; const FINAL_RUN_SCOPE: &str = "final"; -/// `CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name)`. -fn object_key<'py>(py: Python<'py>, kind: &str, name: &str) -> PyResult> { - new_record_with( - py, - RecordId::CompiledObjectKey, - &[ - ( - "resource_type", - enum_from_value(py, RecordId::CompiledResourceType, kind)?, - ), - ("name", PyString::new(py, name).into_any()), - ], - ) -} - /// The names of an audit's references of one kind, in order. fn reference_names(audit: &Bound<'_, PyAny>, kind: &str) -> PyResult> { let mut names: Vec = Vec::new(); @@ -66,15 +57,41 @@ pub(crate) fn resolve_attachment_kind<'py>( downstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = audit.py(); - let attached_kind: Bound<'py, PyAny> = audit_reads::attached_target_kind(audit)?; + let graph = graph_from_indexes(GraphIndexes { + downstream: edges(downstream_deps)?, + ..GraphIndexes::default() + }); + let (kind, name) = attachment(audit, &graph)?; + PyTuple::new( + py, + [ + imported(py, AUDITING_TYPES_MODULE, "AuditAttachmentKind")?.call1((kind,))?, + name.into_pyobject(py)?.into_any(), + ], + ) +} + +/// Resolve an attachment against the existing native graph, without public key or tuple objects. +pub(crate) fn attachment( + audit: &Bound<'_, PyAny>, + graph: &ProjectGraph, +) -> PyResult<(String, Option)> { + let py = audit.py(); + let attached_kind = audit_reads::attached_target_kind(audit)?; let (kind, name): (String, Option) = if attached_kind.is_none() { + let reached = RefCell::new(HashMap::new()); let reaches = |from: &str, to: &str| -> PyResult { - let start: Bound<'py, PyAny> = object_key(py, MODEL_RESOURCE_TYPE, from)?; - closure(vec![start], downstream_deps, (false, None))?.contains(object_key( - py, - MODEL_RESOURCE_TYPE, - to, - )?) + Ok(reached + .borrow_mut() + .entry(from.to_owned()) + .or_insert_with(|| { + reachable_graph_keys( + &[(MODEL_RESOURCE_TYPE.to_owned(), from.to_owned())], + graph, + EdgeDirection::Downstream, + ) + }) + .contains(&(MODEL_RESOURCE_TYPE.to_owned(), to.to_owned()))) }; let (kind, name) = singular_attachment( &reference_names(audit, MODEL_REFERENCE_KIND)?, @@ -83,7 +100,7 @@ pub(crate) fn resolve_attachment_kind<'py>( )?; (attachment_value(kind).to_owned(), name) } else { - let name: Bound<'py, PyAny> = audit_reads::attached_target_name(audit)?; + let name = audit_reads::attached_target_name(audit)?; if name.is_none() { return Err(input_error( py, @@ -101,13 +118,7 @@ pub(crate) fn resolve_attachment_kind<'py>( )?; (kind, Some(text(&name)?)) }; - PyTuple::new( - py, - [ - imported(py, AUDITING_TYPES_MODULE, "AuditAttachmentKind")?.call1((kind,))?, - name.into_pyobject(py)?.into_any(), - ], - ) + Ok((kind, name)) } /// Whether an attached audit reads anything besides its target and the target's upstream. @@ -116,6 +127,18 @@ pub(crate) fn resolve_attachment_kind<'py>( pub(crate) fn reads_outside_target_lineage( audit: &Bound<'_, PyAny>, upstream_deps: &Bound<'_, PyAny>, +) -> PyResult { + let graph = graph_from_indexes(GraphIndexes { + upstream: edges(upstream_deps)?, + ..GraphIndexes::default() + }); + outside_target_lineage(audit, &graph) +} + +/// Check target lineage using native keys and the project's edge indexes. +pub(crate) fn outside_target_lineage( + audit: &Bound<'_, PyAny>, + graph: &ProjectGraph, ) -> PyResult { let py: Python<'_> = audit.py(); let kind: Bound<'_, PyAny> = audit_reads::attached_target_kind(audit)?; @@ -125,10 +148,10 @@ pub(crate) fn reads_outside_target_lineage( } let kind: String = text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; - let target: Bound<'_, PyAny> = object_key(py, &kind, &text(&name)?)?; - let lineage: Bound<'_, PySet> = closure(vec![target], upstream_deps, (true, None))?; + let target = (kind, text(&name)?); + let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); for dependency in audit_reads::scope_deps(audit)?.try_iter()? { - if !lineage.contains(dependency?)? { + if !lineage.contains(&native_key(&dependency?)?) { return Ok(true); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 3c4a567b6f..414a8697f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::model_config::typed_config::{ config_cursor_bound, config_text_object, config_value, model_values, }; use crate::bindings::_helpers::planner_identity::metadata::or_empty; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_strategy::constants::{ ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, @@ -64,7 +65,7 @@ fn lookups_of<'py>(request: &Request<'_, 'py>) -> PyResult = request.py(); let lookup = |name: &str| or_empty(request.optional(name)?, py); Ok(CursorInputLookups { - models_by_name: lookup("models_by_name")?, + models_by_name: lookup("models_by_name")?.into(), functions_by_name: lookup("functions_by_name")?, source_map: lookup("source_map")?, source_warehouse_columns: lookup("source_warehouse_columns")?, @@ -102,7 +103,7 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( } let model: &Bound<'py, PyAny> = model; let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(model)?; - let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let models_by_name = &lookups.models_by_name; let source_map: &Bound<'py, PyAny> = &lookups.source_map; let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; let model_name: String = model_reads::model_name(model)?; @@ -112,7 +113,7 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let reference: Bound<'py, PyAny> = resolve_lineage_reference( model, &input_name, - models_by_name.clone(), + models_by_name.clone_ref(py), lookups.functions_by_name.clone(), )?; let kind: String = text(&native_field(&reference, "ref_kind")?)?; @@ -125,7 +126,7 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( known_columns, }; if kind == MODEL_REFERENCE { - let upstream: Bound<'py, PyAny> = mapping_get(models_by_name, &name)?; + let upstream = models_by_name.get(py, &name)?; if upstream.is_none() || !config_value(py, &model_values(&upstream)?, "contract")? .eq(ENFORCED_CONTRACT)? @@ -258,7 +259,7 @@ pub(crate) fn build_cursor_input_relations<'py>( return Ok(PyTuple::empty(py)); } let model: Bound<'py, PyAny> = model.clone(); - let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let models_by_name = &lookups.models_by_name; let functions_by_name: &Bound<'py, PyAny> = &lookups.functions_by_name; let model_locations: &Bound<'py, PyAny> = &lookups.model_locations; let producers: &Bound<'py, PyAny> = &lookups.runtime_cursor_producer_names; @@ -275,7 +276,7 @@ pub(crate) fn build_cursor_input_relations<'py>( let reference: Bound<'py, PyAny> = resolve_lineage_reference( &model, &input_name, - models_by_name.clone(), + models_by_name.clone_ref(py), functions_by_name.clone(), )?; let kind: String = text(&native_field(&reference, "ref_kind")?)?; @@ -285,7 +286,7 @@ pub(crate) fn build_cursor_input_relations<'py>( continue; }; let producer: Option> = if kind == MODEL_REFERENCE { - Some(mapping_get(models_by_name, &name)?).filter(|model| !model.is_none()) + Some(models_by_name.get(py, &name_text)?).filter(|model| !model.is_none()) } else { None }; @@ -336,14 +337,13 @@ pub(crate) fn runtime_cursor_producer_names<'py>( source_map: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = scope.py(); - let mut names: Vec> = Vec::new(); - for key in native_field(scope, "selected_keys")?.try_iter()? { - let key: Bound<'py, PyAny> = key?; - let name: Bound<'py, PyAny> = native_field(&key, "name")?; - match text(&native_field(&key, "resource_type")?)?.as_str() { + let data = scope_data(scope)?; + let mut names: Vec<&str> = Vec::new(); + for (kind, name) in &data.selected { + match kind.as_str() { "model" | "seed" => names.push(name), "source" => { - let source: Bound<'py, PyAny> = mapping_get(source_map, &name)?; + let source: Bound<'py, PyAny> = mapping_get(source_map, name)?; if !source.is_none() && !native_field(&source, "loader")?.is_none() { names.push(name); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 9d5f12e701..539e5c1e98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::compiled_views::source_reads; +use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; use crate::bindings::_helpers::planner_strategy::constants::{ MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, }; @@ -52,22 +53,28 @@ pub(crate) fn planning_context( ) -> PyResult { let py = project.py(); work_counts(py)?.borrow_mut().context_builds += 1; - let model_map = match models_by_name { - Some(models) => models, + let model_map: HashMap = match models_by_name { + Some(models) => models + .iter() + .map(|(name, model)| Ok((name.extract()?, ModelHandle::read(&model)))) + .collect::>()?, None => { - let models = PyDict::new(py); + let mut models: HashMap = HashMap::new(); for model in project_reads::models(project)?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, &model)?; + models.insert( + model_reads::name_object(&model)?.extract()?, + ModelHandle::read(&model), + ); } models } }; let mut relations: HashMap = HashMap::new(); - for (name, model) in model_map.iter() { + for (name, model) in &model_map { relations.insert( - (MODEL_REFERENCE_KIND.to_owned(), name.extract()?), - model_fixture_relation(&model)?, + (MODEL_REFERENCE_KIND.to_owned(), name.clone()), + model_fixture_relation(&model.bind(py))?, ); } for (kind, source) in [(SOURCE_REFERENCE_KIND, true), (SEED_REFERENCE_KIND, false)] { @@ -99,14 +106,14 @@ pub(crate) fn planning_context( } } Ok(context_of( - model_map.unbind(), + model_map, relations, Py::new(py, FixtureColumnInferences::new())?.into_any(), )) } fn context_of( - models_by_name: Py, + models_by_name: HashMap, relations: HashMap, fixture_inferences: Py, ) -> RelationFixturePlanningContext { @@ -143,9 +150,10 @@ impl RelationFixturePlanningContext { if let Some(found) = self.facts_cache()?.get(name) { return Ok(Some(Arc::clone(found))); } - let Some(model) = self.models_by_name.bind(py).get_item(name)? else { + let Some(model) = self.models_by_name.get(name) else { return Ok(None); }; + let model = model.bind(py); let facts = Arc::new(ModelFixtureFacts { reads: lineage_reads(&model)?, projections: projections(&model)?, @@ -194,7 +202,10 @@ impl RelationFixturePlanningContext { ); } Ok(context_of( - models_by_name.unbind(), + models_by_name + .iter() + .map(|(name, model)| Ok((name.extract()?, ModelHandle::read(&model)))) + .collect::>()?, known, Py::new(py, FixtureColumnInferences::new())?.into_any(), )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index aacaa6ffab..965439ef0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -1,7 +1,7 @@ //! Boundary conversion for native fixture read analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDictMethods, PyFrozenSet}; +use pyo3::types::PyFrozenSet; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; @@ -52,12 +52,12 @@ pub(crate) fn required_fixture_columns( .call_method0("expression_inference_profile")? .getattr("sql_analysis_dialect")? .extract::>()?; - let models_by_name = context.models_by_name.bind(py); let mut models: Vec = Vec::new(); for name in ordered_model_names { - let Some(model) = models_by_name.get_item(name)? else { + let Some(model) = context.models_by_name.get(name) else { continue; }; + let model = model.bind(py); let Some(facts) = context.model_facts(py, name)? else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 07127e9bb1..553c9b1e73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_ use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::model_config::typed_config::{config_text_is, model_values}; +use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -25,7 +26,7 @@ const MICROBATCH_MODE: &str = "microbatch"; /// The project's models and SQL functions by name. struct ProjectLineage<'py> { - models_by_name: Bound<'py, PyAny>, + models_by_name: ModelLookup, functions_by_name: Bound<'py, PyAny>, } @@ -75,7 +76,11 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { } fn model_references(&self, name: &str) -> PyResult>>> { - mapped_references(&self.models_by_name, name) + let value = self.models_by_name.get(self.functions_by_name.py(), name)?; + if value.is_none() { + return Ok(None); + } + references(&value).map(Some) } fn function_references(&self, name: &str) -> PyResult>>> { @@ -83,7 +88,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { } fn producer_facts(&self, name: &str) -> PyResult { - let producer: Bound<'py, PyAny> = self.models_by_name.get_item(name)?; + let producer = self.models_by_name.get(self.functions_by_name.py(), name)?; let values: Bound<'py, PyAny> = native_field(&model_reads::config(&producer)?, "values")?; let capped: bool = match microbatch_limit_config(&values)?.1 { Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), @@ -96,7 +101,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { } fn watermark_inputs(&self, name: &str) -> PyResult> { - watermark_names(&self.models_by_name.get_item(name)?) + watermark_names(&self.models_by_name.get(self.functions_by_name.py(), name)?) } } @@ -117,7 +122,7 @@ fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { pub(crate) fn resolve_lineage_reference<'py>( model: &Bound<'py, PyAny>, input_name: &str, - models_by_name: Bound<'py, PyAny>, + models_by_name: ModelLookup, functions_by_name: Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = model.py(); @@ -133,7 +138,7 @@ pub(crate) fn resolve_lineage_reference<'py>( /// Reject capped producers feeding a microbatch model's watermark inputs. pub(crate) fn reject_capped_watermark_inputs<'py>( model: &Bound<'py, PyAny>, - models_by_name: Bound<'py, PyAny>, + models_by_name: ModelLookup, functions_by_name: Option>, ) -> PyResult<()> { let py: Python<'py> = model.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs index cc125e3fc3..4f5a155c6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -5,12 +5,15 @@ use std::sync::{Arc, Mutex}; use pyo3::prelude::{Bound, Py}; use pyo3::pyclass; -use pyo3::types::{PyAny, PyDict}; +use pyo3::types::PyAny; use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureReadCache, FixtureTypeProjection, }; use sqlbuild_analysis::required_columns::models::ColumnRead; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; +use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; /// Per-extension cache replacing the Python fixture resolver's LRU. @@ -42,7 +45,7 @@ pub(crate) struct ModelFixtureFacts { #[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] pub(crate) struct RelationFixturePlanningContext { - pub(crate) models_by_name: Py, + pub(crate) models_by_name: HashMap, pub(crate) relations: HashMap, pub(crate) authoritative_columns: HashMap>, pub(crate) expected_types: HashMap>, @@ -86,7 +89,7 @@ pub(crate) struct ModelEntryDecisions<'py> { /// The project lookups that resolve one model's cursor inputs. #[derive(Debug)] pub(crate) struct CursorInputLookups<'py> { - pub(crate) models_by_name: Bound<'py, PyAny>, + pub(crate) models_by_name: ModelLookup, pub(crate) functions_by_name: Bound<'py, PyAny>, pub(crate) source_map: Bound<'py, PyAny>, pub(crate) source_warehouse_columns: Bound<'py, PyAny>, @@ -103,6 +106,5 @@ pub(crate) struct AuditPlanning<'py> { pub(crate) source_map: Bound<'py, PyAny>, pub(crate) adapter: Bound<'py, PyAny>, pub(crate) model_materializations: Bound<'py, PyAny>, - pub(crate) lineage_upstream_deps: Bound<'py, PyAny>, - pub(crate) lineage_downstream_deps: Bound<'py, PyAny>, + pub(crate) lineage: Py, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index b2c7779c8b..32ff170938 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -272,7 +272,7 @@ fn lineage_reference<'py>( resolve_lineage_reference( model, &input_name.extract::()?, - maps.models.clone().into_any(), + maps.models.clone().into_any().into(), maps.functions.clone().into_any(), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index e72cafdd90..a4379597e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -10,10 +10,10 @@ use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; -use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; +use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; +use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ @@ -33,9 +33,10 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; @@ -104,8 +105,15 @@ fn relevant_state_keys<'py>( if selected.is_none() { return Ok(selected.clone()); } - let deps = build_execution_upstream_deps(project)?.into_any(); - object(py, transitive_closure_many(selected, &deps, true, None)?) + let deps = execution_deps(project)?; + let selected = selected + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>>()?; + let reachable = reachable_keys(&selected, &deps) + .into_iter() + .collect::>(); + Ok(PyFrozenSet::new(py, KeyCache::new(py)?.keys(&reachable)?)?.into_any()) } fn resolve_database<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index d2a86ae501..3ce4fbb65f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -8,19 +8,18 @@ use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_phases::objects::{ - model_key_names, native, replace, text, tuple, -}; +use crate::bindings::_helpers::planner_phases::objects::{native, replace, text, tuple}; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; use crate::bindings::_helpers::records::construct::enum_member; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use std::collections::BTreeSet; @@ -87,12 +86,14 @@ fn state_schemas( } } if schemas.is_empty() { - let models = native_field(scope, "models_by_name")?; - for name in model_key_names(&native_field(scope, "selected_keys")?)? { - let model = mapping_get(&models, name)?; - if model.is_none() { + let data = scope_data(scope)?; + for (kind, name) in &data.selected { + if kind != MODEL { continue; } + let Some(model) = data.model(scope.py(), name) else { + continue; + }; let schema = native_field(&model_reads::destination(&model)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 1970820ed6..80cde938c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; @@ -18,7 +19,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::PyString; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; use sqlbuild_analysis::planner_warehouse::models::MissingSourceTable; @@ -26,18 +28,21 @@ use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { let mut readers: HashMap> = HashMap::new(); - let deps = native_field(scope, "upstream_deps")?; - for key in native_field(scope, "selected_keys")?.try_iter()? { - let key = key?; - if native_field(&key, "resource_type")?.eq("source")? { + let scope = scope_data(scope)?; + let deps: HashMap<_, _> = scope + .upstream + .iter() + .map(|(key, deps)| (key, deps)) + .collect(); + for key in &scope.selected { + if key.0 == SOURCE { continue; } - let reader: String = native_field(&key, "name")?.extract()?; - for dep in mapping_get_or(&deps, &key, PyTuple::empty(key.py()))?.try_iter()? { - let dep = dep?; - if native_field(&dep, "resource_type")?.eq("source")? { + let reader = &key.1; + for dep in deps.get(key).into_iter().flat_map(|deps| deps.iter()) { + if dep.0 == SOURCE { readers - .entry(native_field(&dep, "name")?.extract()?) + .entry(dep.1.clone()) .or_default() .push(reader.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index 9279669fe2..be5971425d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -255,7 +255,7 @@ pub(crate) fn declared_relations<'py>( Some(contract) => contract.eq(ENFORCED_CONTRACT)?, None => false, }; - if enforced && !columns.is_empty() && !schema.dynamic_columns.is_truthy()? { + if enforced && !columns.is_empty() && schema.dynamic_columns.is_empty() { declared.complete(&maps, &name, (&columns, false))?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 7d87753fd1..653e77639e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; @@ -12,6 +13,9 @@ use crate::bindings::_helpers::records::fields::field as native_field; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; +/// Native SQL and dynamic family specs indexed by their model's position. +type PivotModelRows = Vec<(usize, (String, Vec))>; + fn config_value<'py>(model_input: &Bound<'py, PyAny>, key: &str) -> PyResult> { native_field(&native_field(model_input, "config")?, "values")?.call_method1("get", (key,)) } @@ -110,13 +114,27 @@ pub(crate) fn standalone_pivot_models<'py>( ) -> PyResult> { let py: Python<'py> = model_inputs.py(); let models: Bound<'py, PyDict> = PyDict::new(py); + for (index, (sql, _)) in standalone_pivot_rows(model_inputs, analysed)? { + let model = model_inputs.get_item(index)?; + let entry = native_field(&model, "schema_entry")?; + models.set_item(index, (sql, native_field(&entry, "dynamic_columns")?))?; + } + Ok(models) +} + +/// Native-only pivot requests; the public wrapper alone projects contract records. +pub(crate) fn standalone_pivot_rows( + model_inputs: &Bound<'_, PyAny>, + analysed: &Bound<'_, PyAny>, +) -> PyResult { + let mut models: PivotModelRows = Vec::new(); for (index, model_input) in model_inputs.try_iter()?.enumerate() { - let model_input: Bound<'py, PyAny> = model_input?; + let model_input = model_input?; let Some(schema) = attached_schema(&model_input)? else { continue; }; - let families: Bound<'py, PyAny> = schema.dynamic_columns; - if !families.is_truthy()? || analysed.contains(model_name(&model_input)?)? { + let families = schema.dynamic_columns; + if families.is_empty() || analysed.contains(model_name(&model_input)?)? { continue; } let query_sql: String = analysis_sql(&model_input)?.extract()?; @@ -130,7 +148,7 @@ pub(crate) fn standalone_pivot_models<'py>( } _ => query_sql, }; - models.set_item(index, (sql, families))?; + models.push((index, (sql, families))); } Ok(models) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 6251d5cf49..422162374f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -4,22 +4,23 @@ use std::sync::Arc; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, PyTupleMethods, -}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use sqlbuild_analysis::semantic_checks::models::{ AuditValue, MetadataAudit, MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, MetadataSource, MetadataSqlTest, ModelMetadataFindings, }; use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_views::model_reads::binding_validated; +use crate::bindings::_helpers::model_config::config_nodes::PyConfigNode; +use crate::bindings::_helpers::model_config::header_audits::AuditInstance; +use crate::bindings::_helpers::model_config::header_metadata::HeaderValue; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; -use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::inputs::{ @@ -29,6 +30,7 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; +use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_project::models::ModelMetadataTexts; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. @@ -509,7 +511,7 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = model.py(); - let mut attached: Vec<(Option, Bound<'_, PyAny>)> = Vec::new(); + let mut attached: Vec<(Option, AuditInstance<'_>)> = Vec::new(); match model.downcast::() { Ok(view) => { let Some((entry_audits, columns, audited)) = @@ -517,12 +519,12 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { else { return Ok(Vec::new()); }; - for audit in entry_audits.bind(py).iter() { + for audit in entry_audits.instances(py) { attached.push((None, audit)); } for (index, name) in audited { - for audit in columns.specs[index].field(py, "audits")?.try_iter()? { - attached.push((Some(name.clone()), audit?)); + for audit in columns.specs[index].audit_instances(py)? { + attached.push((Some(name.clone()), audit)); } } } @@ -532,13 +534,13 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { return Ok(Vec::new()); } for audit in native_field(&entry, "audits")?.try_iter()? { - attached.push((None, audit?)); + attached.push((None, AuditInstance::Python(audit?))); } for column in native_field(&entry, "columns")?.try_iter()? { let column: Bound<'_, PyAny> = column?; let name: String = native_field(&column, "name")?.extract()?; for audit in native_field(&column, "audits")?.try_iter()? { - attached.push((Some(name.clone()), audit?)); + attached.push((Some(name.clone()), AuditInstance::Python(audit?))); } } } @@ -546,25 +548,23 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { let empty: Bound<'_, PyAny> = PyString::new(py, "").into_any(); let mut audits: Vec = Vec::with_capacity(attached.len()); for (column, audit) in attached { - let arguments: Bound<'_, PyAny> = native_field(&audit, "arguments")?; - let values: Bound<'_, PyAny> = mapping_get(&arguments, "values")?; - let values: Option> = - if values.is_instance_of::() || values.is_instance_of::() { - Some( - values - .try_iter()? - .map(|value| audit_value(&value?)) - .collect::>()?, - ) - } else { - None - }; + let values = audit_values(py, &audit)?; audits.push(MetadataAudit { column, - definition: native_field(&audit, "definition_name")?.extract()?, - expression: text(&mapping_get(&arguments, "expression")?)?, - target: mapping_get_or(&arguments, "to", &empty)?.str()?.extract()?, - field: mapping_get_or(&arguments, "field", &empty)? + definition: audit.field("definition_name")?.extract()?, + expression: text( + &audit + .argument("expression")? + .unwrap_or_else(|| py.None().into_bound(py)), + )?, + target: audit + .argument("to")? + .unwrap_or_else(|| empty.clone()) + .str()? + .extract()?, + field: audit + .argument("field")? + .unwrap_or_else(|| empty.clone()) .str()? .extract()?, values, @@ -573,6 +573,52 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { Ok(audits) } +fn audit_values(py: Python<'_>, audit: &AuditInstance<'_>) -> PyResult>> { + if let AuditInstance::Native(spec, _) = audit { + for (key, value) in &spec.arguments { + match key { + ConfigNode::Str(key) if key == ACCEPTED_VALUES_ARGUMENT => {} + ConfigNode::Host(_) => break, + _ => continue, + } + match value { + ConfigNode::List(values) | ConfigNode::Tuple(values) => { + return values + .iter() + .map(|value| config_audit_value(py, value)) + .collect::>() + .map(Some); + } + ConfigNode::Host(_) => break, + _ => return Ok(None), + } + } + } + let Some(values) = audit.argument("values")? else { + return Ok(None); + }; + if values.is_instance_of::() || values.is_instance_of::() { + values + .try_iter()? + .map(|value| audit_value(&value?)) + .collect::>() + .map(Some) + } else { + Ok(None) + } +} + +fn config_audit_value(py: Python<'_>, value: &PyConfigNode) -> PyResult { + Ok(match value { + ConfigNode::Null => AuditValue::Null, + ConfigNode::Bool(value) => AuditValue::Bool(*value), + ConfigNode::Int(value) => AuditValue::Number(value.to_string()), + ConfigNode::Str(value) => AuditValue::Text(value.clone()), + ConfigNode::Float(_) | ConfigNode::Host(_) => return audit_value(&value.object(py)?), + _ => AuditValue::Other, + }) +} + /// An accepted value by Python type: null, bool, number, text or other. fn audit_value(value: &Bound<'_, PyAny>) -> PyResult { if value.is_none() { diff --git a/fensu.toml b/fensu.toml index bf9498738f..ea0561e0ee 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1773,7 +1773,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", @@ -2019,15 +2019,15 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs=1", @@ -2039,7 +2039,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", @@ -2055,12 +2055,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs=3", @@ -2077,12 +2077,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=2", @@ -2271,7 +2270,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/fields.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/authored_values/sql_values.rs=6", @@ -2312,7 +2311,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=3", @@ -2332,7 +2331,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=9", @@ -2388,12 +2387,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs=6", @@ -2431,7 +2430,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=57", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=56", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", @@ -2451,8 +2450,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=34", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", @@ -2641,6 +2640,7 @@ module_layers = [ "microbatches=plan", "model_config=front", "native_host=infra", + "native_host/model_lookup=project", "native_threads=infra", "output_capture=output", "plan_command=commands", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 61328c8c02..d3700a511d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -14,6 +14,7 @@ InferredColumn, InvalidQualifiedIdentityError, SchemaColumn, + SchemaDynamicColumnFamily, SchemaModelEntry, build_compiled_project, discover_project_inputs, @@ -98,6 +99,22 @@ def counted(self: SchemaColumn, *args: Any, **kwargs: Any) -> None: assert model.schema_entry is entry assert model.schema_entry.columns is columns assert pickle.loads(pickle.dumps(columns)) == columns + assert pickle.loads(pickle.dumps(entry.audits)) == entry.audits + pivot_entry: SchemaModelEntry | None = models["category_amounts"].schema_entry + assert pivot_entry is not None + families: tuple[SchemaDynamicColumnFamily, ...] = pivot_entry.dynamic_columns + assert len(families) == 1 + assert ( + families[0].name, + families[0].pivot_column, + families[0].value_column, + families[0].aggregate, + families[0].type, + families[0].name_pattern, + ) == ("category_amounts", "category", "amount", "MAX", "INTEGER", "^[a-z]+$") + assert models["category_amounts"].schema_entry is pivot_entry + assert pivot_entry.dynamic_columns is families + assert pickle.loads(pickle.dumps(families)) == families @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index b574754880..be015ece62 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -17,6 +17,15 @@ class HeaderMetadataParityTestCase: expected_maximum_later_validated: int +@dataclass(frozen=True) +class HeaderAuditArgumentsTestCase: + """Python-supplied audit arguments and the compiled audit targets they produce.""" + + description: str + header: dict[object, object] + expected_audits: list[tuple[str, str | None]] + + @dataclass(frozen=True) class ModelConfigTierTestCase: """How often the model loop calls each native model config entry.""" diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index bb6dc9cca1..7e9c4d9e70 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -4,22 +4,31 @@ import random from pathlib import Path +from typing import TYPE_CHECKING import pytest +from sqlbuild._native import assemble_project +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( ColumnMetadataExpansionTestCase, + HeaderAuditArgumentsTestCase, HeaderMetadataParityTestCase, ModelConfigTierTestCase, ) from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( HeaderMetadataParity, column_metadata_outcome, + config_value_inputs, generated_header_metadata, header_metadata_parity, model_config_outcome, ) +if TYPE_CHECKING: + from sqlbuild._native import CompileInputsView + @pytest.mark.parametrize( "test_case", @@ -51,6 +60,36 @@ def test_given_generated_header_metadata_when_attaching_then_model_loop_matches_ ) == ([], True, True, True), (parity.parsed, parity.rejected, parity.later_validated) +@pytest.mark.parametrize( + "test_case", + [ + HeaderAuditArgumentsTestCase( + description="numeric keys on model and column audits are ignored during compilation", + header={ + "description": "One row per order.", + "columns": {"order_id": {"audits": [{"unique": {1: "ignored"}}]}}, + "audits": [{"unique": {2: "also ignored", "column": "order_id"}}], + }, + expected_audits=[("unique", "order_id"), ("unique", "order_id")], + ), + ], + ids=lambda case: case.description, +) +def test_given_non_string_audit_arguments_when_attaching_then_keys_remain_ignored( + test_case: HeaderAuditArgumentsTestCase, + tmp_path: Path, +) -> None: + inputs: CompileInputsView = config_value_inputs(project_dir=tmp_path, header=test_case.header) + project: CompiledProjectView = assemble_project( + inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb") + ) + + assert [ + (audit.definition_name, audit.attached_column_name) for audit in project.audits + ] == test_case.expected_audits + assert project.diagnostics == () + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index f6d1084adf..10f54ca9bd 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -47,6 +47,7 @@ Fingerprint, FunctionArgument, PlannerChangeResults, + PlannerScope, PlanOutput, PlanReason, ScenarioArtifactIdentity, @@ -1395,8 +1396,13 @@ def detect_function_reason( sql_analysis_dialect=None, hook_functions=(), ) - scope: Mock = Mock( - selected_keys=frozenset({function.key}), execution_order=(), models_by_name={} + scope: PlannerScope = PlannerScope( + upstream_deps={}, + downstream_deps={}, + all_keys={}, + selected_keys=frozenset({function.key}), + execution_order=(), + models_by_name={}, ) snapshot: Mock = Mock(fingerprints=SimpleNamespace(functions=fingerprints), column_dialect=None) changes: PlannerChangeResults = detect_changes( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py index c2047ede65..471fbbc6ed 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py @@ -1,9 +1,13 @@ from __future__ import annotations +import pickle +from dataclasses import replace + import pytest from sqlbuild._native import ( PlannerPolicies, + PlannerScope, PlannerScopeResolution, PlannerSelection, resolve_planner_scopes, @@ -84,3 +88,10 @@ def test_given_selection_when_resolving_scopes_then_stale_warning_scope_is_upstr assert stale_scope_names == test_case.expected_stale_warning_names assert tuple(sorted(stale_order_names)) == test_case.expected_stale_warning_names assert resolution.inspection_scope is resolution.selected_scope + assert isinstance(resolution.selected_scope, PlannerScope) + assert tuple(resolution.selected_scope.models_by_name) == tuple(test_case.model_deps) + assert set(resolution.stale_warning_scope.models_by_name) == set( + test_case.expected_stale_warning_names + ) + assert replace(resolution.selected_scope, load_sources=True).load_sources + assert pickle.loads(pickle.dumps(resolution.selected_scope)) == resolution.selected_scope From 7945e66c79df584ee9433bc9f7ba1235a1f04641 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:18:00 +0000 Subject: [PATCH 2/3] fix: keep migration planning on native scope data Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .../edge_cases/rpt_edge_planner_fan_in.sql | 46 +++++++++++++++++++ .../test_rpt_edge_planner_fan_in.sql | 29 ++++++++++++ .../planner_migrations/column_candidates.rs | 18 ++++---- .../planner_migrations/discovery_models.rs | 24 ++++------ .../planner_migrations/discovery_requests.rs | 14 +++++- .../planner_migrations/model_planning.rs | 16 ++++--- .../planner_migrations/old_name_planning.rs | 13 +++--- fensu.toml | 6 +-- 8 files changed, 123 insertions(+), 43 deletions(-) create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_planner_fan_in.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_planner_fan_in.sql diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_planner_fan_in.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_planner_fan_in.sql new file mode 100644 index 0000000000..a8e233280c --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_planner_fan_in.sql @@ -0,0 +1,46 @@ +MODEL ( + description "Ledger report whose branches share an upstream model during migration planning", + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + channel (type VARCHAR), + unit_amount (type NUMBER(38, 10)), + ), +); + +WITH ledger AS ( + SELECT entry_id + FROM __ref("stg_edge_ledger_entries") +), + +amounts AS ( + SELECT + entry_id, + unit_amount + FROM __ref("int_edge_line_amounts") +), + +channels AS ( + SELECT + entry_id, + channel + FROM __ref("int_edge_orders_all_channels") +), + +final AS ( + SELECT + l.entry_id, + c.channel, + a.unit_amount + FROM ledger AS l + INNER JOIN amounts AS a + ON l.entry_id = a.entry_id + INNER JOIN channels AS c + ON l.entry_id = c.entry_id +) + +SELECT + entry_id, + channel, + unit_amount +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_planner_fan_in.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_planner_fan_in.sql new file mode 100644 index 0000000000..3ef4d46e1f --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_planner_fan_in.sql @@ -0,0 +1,29 @@ +TEST (name "rpt_edge_planner_fan_in__keeps_only_entries_in_every_branch"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT v.entry_id + FROM VALUES ('LE-01'), ('LE-02'), ('LE-03') AS v(entry_id) +), + +__ref__int_edge_line_amounts AS ( + SELECT + v.entry_id, + v.unit_amount::DECIMAL(38, 10) AS unit_amount + FROM VALUES ('LE-01', 2.50), ('LE-02', 7.25), ('LE-04', 9.50) AS v(entry_id, unit_amount) +), + +__ref__int_edge_orders_all_channels AS ( + SELECT + v.entry_id, + v.channel + FROM VALUES ('LE-01', 'web'), ('LE-03', 'app'), ('LE-04', 'partner') AS v(entry_id, channel) +), + +__expected__rpt_edge_planner_fan_in AS ( + SELECT + 'LE-01' AS entry_id, + 'web' AS channel, + 2.50::DECIMAL(38, 10) AS unit_amount +) + +SELECT 1 diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index 6cca98b4c1..03ca747dea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -8,27 +8,27 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnCandidate, ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; pub(super) fn candidates<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), dialect: &Bound<'py, PyAny>, ) -> PyResult>> { - let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); - let py = scope.py(); + let py = phase.scope.py(); + let scope = scope_data(&phase.scope)?; let mut result: Vec> = Vec::new(); - for key in native_field(&scope, "execution_order")?.try_iter()? { - let key = key?; - if !native_field(&scope, "selected_keys")?.contains(&key)? - || !native_field(&key, "resource_type")?.eq("model")? - { + for key in &scope.order { + if !scope.selected.contains(key) || key.0 != MODEL_RESOURCE { continue; } - let model = native_field(&scope, "models_by_name")? - .call_method1("get", (native_field(&key, "name")?,))?; + let Some(model) = scope.model(py, &key.1) else { + continue; + }; if model.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 38899cfdf7..5cca26ed8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -4,18 +4,19 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; +use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_order; -use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { @@ -48,22 +49,13 @@ pub(super) fn triggered( snapshot: &Bound<'_, PyAny>, declared: &Bound<'_, PyDict>, ) -> PyResult { - let py = scope.py(); - let selected: Vec> = native_field(scope, "selected_keys")? - .try_iter()? - .collect::>()?; - let mut relevant = PyNodeSet(PySet::new(py, &selected)?); - breadth_first_reach(selected, None, &mut relevant, |key| { - native_field(scope, "upstream_deps")? - .call_method1("get", (key, pyo3::types::PyTuple::empty(py)))? - .try_iter()? - .collect::>>>() - })?; - let relevant = relevant.0; + let scope = scope_data(scope)?; + let selected: Vec<_> = scope.selected.iter().cloned().collect(); + let relevant = reachable_keys(&selected, &scope.upstream); let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?; for item in models { let name = model_reads::name_object(item)?; - if relevant.contains(model_reads::model_key(item)?)? + if relevant.contains(&native_key(&model_reads::model_key(item)?)?) && !fingerprints.contains(&name)? && !declared.contains(&name)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index b0dd9cf867..2f101d65ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ @@ -10,6 +11,7 @@ use crate::bindings::_helpers::planner_migrations::discovery_events::{ use crate::bindings::_helpers::planner_migrations::models::{ ModelMigrationDeclaration, ModelMigrationRequest, }; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::construct::enum_member; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -113,9 +115,13 @@ pub(super) fn automatic<'py>( matches: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), ) -> PyResult>> { let py = runtime.py(); + let scope = scope_data(scope)?; let mut requests: Vec> = Vec::new(); for item in models { - if !native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { + if !scope + .selected + .contains(&native_key(&model_reads::model_key(item)?)?) + { continue; } let name = model_reads::name_object(item)?; @@ -161,9 +167,13 @@ pub(super) fn warnings<'py>( ambiguous: &Bound<'py, PyAny>, ) -> PyResult> { let py = scope.py(); + let scope = scope_data(scope)?; let selected = PySet::empty(py)?; for item in models { - if native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { + if scope + .selected + .contains(&native_key(&model_reads::model_key(item)?)?) + { selected.add(model_reads::name_object(item)?)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 2a90e97ead..61a2374d02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -19,12 +19,14 @@ use crate::bindings::_helpers::planner_migrations::planning_origin::{ }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::construct::{enum_member, new_record}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; +use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_model_config::config_values::models::ConfigNode; const MIGRATE_FROM_KEY: &str = "migrate_from"; @@ -55,16 +57,16 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult( scope: &Bound<'py, PyAny>, ) -> PyResult>> { + let py = scope.py(); + let scope = scope_data(scope)?; let mut requests: Vec> = Vec::new(); - for key in native_field(scope, "execution_order")?.try_iter()? { - let key = key?; - if !native_field(scope, "selected_keys")?.contains(&key)? - || !native_field(&key, "resource_type")?.eq("model")? - { + for key in &scope.order { + if !scope.selected.contains(key) || key.0 != MODEL_RESOURCE { continue; } - let item = native_field(scope, "models_by_name")? - .call_method1("get", (native_field(&key, "name")?,))?; + let Some(item) = scope.model(py, &key.1) else { + continue; + }; if !item.is_none() && let Some(request) = declared(&item)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index a7eb581cc9..b85d9f1923 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -13,9 +13,11 @@ use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_err use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; +use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; fn schemas<'py>( (phase, migration_entries): (&MigrationPhase<'py>, &Bound<'py, PyAny>), @@ -75,7 +77,7 @@ pub(crate) fn plan_old_name_views<'py>( ) -> PyResult> { let py = phase.runtime.py(); let runtime = phase.runtime.clone(); - let scope = phase.scope.clone(); + let scope = scope_data(&phase.scope)?; let project = runtime_field(&runtime, |runtime| &runtime.project)?; let models: Vec> = project_reads::models(&project)? .try_iter()? @@ -85,11 +87,10 @@ pub(crate) fn plan_old_name_views<'py>( by_name.set_item(model_reads::name_object(item)?, item)?; } let mut selected: Vec> = Vec::new(); - for key in native_field(&scope, "execution_order")?.try_iter()? { - let key = key?; - if native_field(&scope, "selected_keys")?.contains(&key)? - && native_field(&key, "resource_type")?.eq("model")? - && let Some(item) = by_name.get_item(native_field(&key, "name")?)? + for key in &scope.order { + if scope.selected.contains(key) + && key.0 == MODEL_RESOURCE + && let Some(item) = by_name.get_item(&key.1)? { selected.push(item); } diff --git a/fensu.toml b/fensu.toml index ea0561e0ee..f7a2915372 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2028,7 +2028,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs=8", @@ -2037,14 +2037,14 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs=4", From a0c071dcf548f7cbee67f18e9f963625dbc865fa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:18:01 +0000 Subject: [PATCH 3/3] chore: keep orb bootstrap in private project settings Amp-Thread-ID: https://ampcode.com/threads/T-01a18715-4e00-708e-a87d-6bd109748d66 --- .agents/setup | 52 --------------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100755 .agents/setup diff --git a/.agents/setup b/.agents/setup deleted file mode 100755 index 68571288bd..0000000000 --- a/.agents/setup +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Prepares a fresh Amp orb: Rust stable, Python 3.12 and the locked uv environment with the -# native extension built. Idempotent; a warm run on a restored snapshot only re-verifies. -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo_root" - -step() { - local name="$1" - shift - local start=$SECONDS - echo "==> ${name}" - "$@" - echo "<== ${name} ($((SECONDS - start))s)" -} - -# One persistent Cargo target directory per worktree, outside the repository (see AGENTS.md). -target_dir="$HOME/.cache/cargo-target/$(basename "$repo_root")" -export CARGO_TARGET_DIR="$target_dir" -export PATH="$HOME/.cargo/bin:$PATH" - -install_rust() { - if ! command -v rustup >/dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable \ - --component clippy --component rustfmt - else - # CI lints with the latest stable toolchain. - rustup update stable --no-self-update - rustup component add clippy rustfmt --toolchain stable - fi -} - -persist_env() { - local marker="# sqlbuild orb environment (${repo_root})" - if ! grep -Fqx "$marker" "$HOME/.bash_profile" 2>/dev/null; then - cat >>"$HOME/.bash_profile" <