Skip to content

Commit 4d07a41

Browse files
authored
feat(native): analyse models natively behind the preview engine (#1045)
* feat(native): analyse models natively behind the preview engine Run the uncached model analysis as a native session: compact batches, dataflow waves, shape publication, deferred binding validation and expression-source shapes. Legacy analyses and input enrichment defer to Python per model and are recorded. * test(native): count only lineage deferrals in the fast lineage engine test * test(native): use the native attachment entry field after #1043 and #1046 * fix(native): compare catalog shapes on cold compiles and record deferrals after the session Keep the binding catalog's schemas and analysis shapes in captures and ignore them only on warm stages, where Python's analysis cache skips catalog preparation. Record per-model deferrals once the session completes, and fall back to Python when provide or finish panics.
1 parent 0ce6787 commit 4d07a41

57 files changed

Lines changed: 4426 additions & 31 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
//! A session copy of the compile's binding catalog that records what Python's would gain.
2+
3+
use std::collections::{HashMap, HashSet};
4+
5+
use crate::assembly::analysis_session::_helpers::mappings::{
6+
ShapeTable, catalog_columns, catalog_relations, same_mapping,
7+
};
8+
use crate::assembly::analysis_session::types::{Pairs, Shapes};
9+
use crate::semantic_validation::models::{Columns, ProjectCatalog};
10+
use crate::semantic_validation::types::{BindingRequest, Relations};
11+
12+
/// Python's `BindingCatalog` state that model analysis reads and changes.
13+
#[derive(Debug)]
14+
pub(crate) struct SessionCatalog {
15+
pub(crate) native: ProjectCatalog,
16+
schemas: HashMap<String, Pairs>,
17+
additions: Shapes,
18+
analysis_names: Vec<String>,
19+
recorded_analysis: HashSet<String>,
20+
}
21+
22+
impl SessionCatalog {
23+
/// Copy `native`, whose Python wrapper holds `schemas`.
24+
pub(crate) fn new(native: &ProjectCatalog, schemas: &Shapes) -> Self {
25+
Self {
26+
native: native.with_relations(Relations::new()),
27+
schemas: schemas.iter().cloned().collect(),
28+
additions: Vec::new(),
29+
analysis_names: Vec::new(),
30+
recorded_analysis: HashSet::new(),
31+
}
32+
}
33+
34+
/// Python's `BindingCatalog.prepare`: add unknown closed relations, then override the rest.
35+
pub(crate) fn prepare(&mut self, requests: &[(&str, &Shapes)]) -> Vec<BindingRequest> {
36+
let mut additions: Shapes = Vec::new();
37+
for (_, schema) in requests {
38+
for (name, columns) in schema.iter() {
39+
if !columns.is_empty() && !self.schemas.contains_key(name) {
40+
self.schemas.insert(name.clone(), columns.clone());
41+
additions.push((name.clone(), columns.clone()));
42+
}
43+
}
44+
}
45+
if !additions.is_empty() {
46+
self.native.update_relations(catalog_relations(&additions));
47+
self.additions.extend(additions);
48+
}
49+
requests
50+
.iter()
51+
.map(|(sql, schema)| self.binding_request(sql, schema))
52+
.collect()
53+
}
54+
55+
fn binding_request(&self, sql: &str, schema: &Shapes) -> BindingRequest {
56+
let mut overrides: Relations = Relations::new();
57+
let mut references: Vec<(String, bool)> = Vec::with_capacity(schema.len());
58+
for (name, columns) in schema {
59+
references.push((name.clone(), !columns.is_empty()));
60+
let known: bool = self
61+
.schemas
62+
.get(name)
63+
.is_some_and(|known| same_mapping(known, columns));
64+
if !columns.is_empty() && !known {
65+
overrides.insert(name.clone(), catalog_columns(columns));
66+
}
67+
}
68+
references.sort();
69+
(sql.to_owned(), references, overrides)
70+
}
71+
72+
/// Python's `BindingCatalog.prepare_analysis` for the relations `names` a batch reads.
73+
pub(crate) fn prepare_analysis(
74+
&mut self,
75+
names: &[String],
76+
types: &ShapeTable,
77+
nullability: &ShapeTable,
78+
) {
79+
let mut updates: HashMap<String, (Columns, Columns)> = HashMap::new();
80+
for name in names {
81+
let Some(columns) = nullability.get(name) else {
82+
continue;
83+
};
84+
let column_types: Pairs = types.get(name).cloned().unwrap_or_default();
85+
updates.insert(
86+
name.clone(),
87+
(catalog_columns(&column_types), catalog_columns(columns)),
88+
);
89+
if self.recorded_analysis.insert(name.clone()) {
90+
self.analysis_names.push(name.clone());
91+
}
92+
}
93+
if !updates.is_empty() {
94+
self.native.update_analysis(updates);
95+
}
96+
}
97+
98+
pub(crate) fn register_override(&mut self, overrides: Relations) -> usize {
99+
self.native.register_override(overrides)
100+
}
101+
102+
/// The `schemas` additions and analysis-shape names Python's catalog records.
103+
pub(crate) fn into_changes(self) -> (Shapes, Vec<String>) {
104+
(self.additions, self.analysis_names)
105+
}
106+
}

0 commit comments

Comments
 (0)