refactor: read native record fields directly across the bindings (#1123)

* refactor(rules): read SQL rule model inputs from native facts

SQL rules read each compiled model's relative path, sql_analysis opt-out,
authored and expanded SQL and dynamic pivot proof from the native facts
and analysis instead of Python attributes. The proof's cache-key text is
encoded natively as the same sorted compact JSON json.dumps produced, so
SQL rule cache keys are unchanged; hand-built models keep reading their
record fields.

* refactor(rules): read SQL test and fixture schema inputs in place

Native records now keep their class layout, so records::fields::field
reads a declared field straight from the record's values without the
Python attribute protocol, falling back to getattr for other objects.
SQL test and scenario facts and the fixture-format schemas read their
inputs this way, and discovered model files' paths, header values and
column locations through the native model file record.

* refactor(rules): run rule planning and dispatch on native record reads

Rule planning, dispatch, catalogue selection, finalization, results and
format --fix now read native record fields in place, rebuild records with
records::fields::replaced instead of dataclasses.replace, and call the
rule helpers (plan, SQL rules, finish, finalize, selection, catalogue,
compiled project builds) directly instead of through module attributes.
Rules configurations and custom payloads encode natively as the JSON the
selection and request paths parsed, falling back to Python's encoder for
values only it spells (non-finite or wide numbers, non-text keys).
Duplicate-literal hints read finding files natively, and project views
answer settings, scenarios and scope index reads from stored fields.

* refactor(rules): read lint, guidance and SQL rule inputs from native records

SQL rule runs, lint settings and compiled expansions, rule guidance,
early SQL lint planning and fix-verification facts read native record
fields in place, and project views answer diagnostics reads from their
stored fields.

* refactor: read native record fields directly instead of through the Python attribute protocol

Author: kvlonge

crates/sqlbuild-analysis/src/assembly/analysis_session/main/contract_proof_json.rs

@@ -0,0 +1,64 @@
+//! A dynamic pivot proof as `json.dumps(asdict(proof), sort_keys=True, separators=(",", ":"))`.
+
+use sqlbuild_core::json::main::dumps::dumps;
+use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions};
+
+use crate::assembly::analysis_session::models::ContractProof;
+
+/// The proof's canonical JSON text, as the SQL rule cache keys a model's contract.
+pub fn contract_proof_json(proof: &ContractProof) -> String {
+    let text = |value: &str| JsonValue::String(value.to_owned());
+    let optional = |value: Option<&String>| value.map_or(JsonValue::Null, |value| text(value));
+    let fixed: Vec<JsonValue> = proof
+        .fixed_columns
+        .iter()
+        .map(|column| {
+            JsonValue::Object(vec![
+                ("name".to_owned(), text(&column.name)),
+                ("nullability".to_owned(), text(&column.nullability)),
+                ("type".to_owned(), optional(column.data_type.as_ref())),
+            ])
+        })
+        .collect();
+    let families: Vec<JsonValue> = proof
+        .families
+        .iter()
+        .map(|(name, inferred_type)| {
+            JsonValue::Object(vec![
+                ("inferred_type".to_owned(), optional(inferred_type.as_ref())),
+                ("name".to_owned(), text(name)),
+            ])
+        })
+        .collect();
+    let value: JsonValue = JsonValue::Object(vec![
+        (
+            "bare_dynamic_pivot".to_owned(),
+            JsonValue::Bool(proof.bare_dynamic_pivot),
+        ),
+        (
+            "failure_reason".to_owned(),
+            optional(proof.failure_reason.as_ref()),
+        ),
+        ("families".to_owned(), JsonValue::Array(families)),
+        ("fixed_columns".to_owned(), JsonValue::Array(fixed)),
+        (
+            "input_relations".to_owned(),
+            JsonValue::Array(
+                proof
+                    .input_relations
+                    .iter()
+                    .map(|name| text(name))
+                    .collect(),
+            ),
+        ),
+        (
+            "output_proven".to_owned(),
+            JsonValue::Bool(proof.output_proven),
+        ),
+    ]);
+    let options: StdlibJsonOptions = StdlibJsonOptions {
+        sort_keys: true,
+        ..StdlibJsonOptions::new(None).with_separators(",", ":")
+    };
+    dumps(&value, &JsonDialect::Stdlib(options)).unwrap_or_default()
+}

crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs

@@ -1,4 +1,5 @@
 pub mod attach_analysis_cache;
+pub mod contract_proof_json;
 pub mod expression_shapes;
 pub mod finish_analysis_session;
 pub mod finished_fact_models;

crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs

@@ -2,6 +2,8 @@
 mod analysis_cache;
 #[path = "tests/test_analysis_session.rs"]
 mod analysis_session;
+#[path = "tests/test_contract_proof_json.rs"]
+mod contract_proof_json;
 #[path = "tests/test_expression_shapes.rs"]
 mod expression_shapes;
 #[path = "tests/helpers.rs"]

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_contract_proof_json.rs

@@ -0,0 +1,53 @@
+use crate::assembly::analysis_session::main::contract_proof_json::contract_proof_json;
+use crate::assembly::analysis_session::tests::test_types::ContractProofJsonTestCase;
+
+use crate::assembly::analysis_session::models::{ColumnFact, ContractProof};
+
+#[test]
+fn given_contract_proof_when_encoding_then_matches_python_sorted_compact_json() {
+    let test_cases = [
+        ContractProofJsonTestCase {
+            description: "a proven pivot escapes non-ASCII text as Python does",
+            proof: ContractProof {
+                output_proven: true,
+                fixed_columns: vec![ColumnFact {
+                    name: "a\u{e9}\"".to_owned(),
+                    data_type: None,
+                    nullability: "non_null".to_owned(),
+                }],
+                families: vec![("f".to_owned(), Some("INT".to_owned()))],
+                input_relations: vec!["x".to_owned()],
+                failure_reason: None,
+                bare_dynamic_pivot: false,
+            },
+            expected_json: concat!(
+                r#"{"bare_dynamic_pivot":false,"failure_reason":null,"#,
+                r#""families":[{"inferred_type":"INT","name":"f"}],"#,
+                r#""fixed_columns":[{"name":"a\u00e9\"","nullability":"non_null","type":null}],"#,
+                r#""input_relations":["x"],"output_proven":true}"#
+            ),
+        },
+        ContractProofJsonTestCase {
+            description: "a failed proof keeps its reason and empty collections",
+            proof: ContractProof {
+                output_proven: false,
+                fixed_columns: Vec::new(),
+                families: vec![("g".to_owned(), None)],
+                input_relations: Vec::new(),
+                failure_reason: Some("no schema".to_owned()),
+                bare_dynamic_pivot: true,
+            },
+            expected_json: concat!(
+                r#"{"bare_dynamic_pivot":true,"failure_reason":"no schema","#,
+                r#""families":[{"inferred_type":null,"name":"g"}],"fixed_columns":[],"#,
+                r#""input_relations":[],"output_proven":false}"#
+            ),
+        },
+    ];
+
+    for test_case in test_cases {
+        let json: String = contract_proof_json(&test_case.proof);
+
+        assert_eq!(json, test_case.expected_json, "{}", test_case.description);
+    }
+}

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs

@@ -1,4 +1,4 @@
-use crate::assembly::analysis_session::models::PivotOutcome;
+use crate::assembly::analysis_session::models::{ContractProof, PivotOutcome};
 
 /// A model as `(name, sql, source names, model refs)`.
 pub(crate) type ModelSpec = (
@@ -183,3 +183,9 @@ pub(crate) struct NativeDialectTestCase {
     pub(crate) dialect: Option<&'static str>,
     pub(crate) expected_dialect: Option<&'static str>,
 }
+
+pub(crate) struct ContractProofJsonTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) proof: ContractProof,
+    pub(crate) expected_json: &'static str,
+}

crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs

@@ -11,6 +11,7 @@ use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRo
 
 use crate::bindings::_helpers::analysis::conversions;
 use crate::bindings::_helpers::boundary::panics::compiler_error;
+use crate::bindings::_helpers::records::fields::field as native_field;
 use crate::bindings::_helpers::records::ids::RecordId;
 use crate::bindings::_helpers::records::registry;
 use crate::bindings::models::{CatalogInput, ProjectCatalog};
@@ -88,7 +89,7 @@ impl BindingCatalog {
         let schemas: &Bound<'_, PyDict> = self.schemas.bind(py);
         let additions: Bound<'_, PyDict> = PyDict::new(py);
         for request in requests {
-            let schema: Bound<'_, PyAny> = request.getattr("schema")?;
+            let schema: Bound<'_, PyAny> = native_field(request, "schema")?;
             for item in schema.call_method0("items")?.try_iter()? {
                 let (name, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?;
                 if columns.is_truthy()? && !schemas.contains(&name)? {
@@ -103,7 +104,7 @@ impl BindingCatalog {
         }
         let mut prepared: Vec<BindingRequest> = Vec::with_capacity(requests.len());
         for request in requests {
-            let schema: Bound<'_, PyAny> = request.getattr("schema")?;
+            let schema: Bound<'_, PyAny> = native_field(request, "schema")?;
             let mut names: Vec<String> = Vec::new();
             for name in schema.try_iter()? {
                 names.push(name?.extract()?);
@@ -123,7 +124,7 @@ impl BindingCatalog {
                 }
                 closed.push((name, truthy));
             }
-            let sql: String = request.getattr("sql")?.extract()?;
+            let sql: String = native_field(request, "sql")?.extract()?;
             prepared.push((sql, closed, conversions::relations(overrides.as_any())?));
         }
         Ok(prepared)
@@ -293,14 +294,14 @@ pub(crate) fn schema_validations<'py>(
     let mut groups: Vec<(Py<BindingCatalog>, bool, Vec<usize>)> = Vec::new();
     for (index, request) in requests.iter().enumerate() {
         let key: CatalogKey = (
-            native_dialect(&request.getattr("dialect")?)?,
+            native_dialect(&native_field(request, "dialect")?)?,
             request
                 .getattr("quoted_identifiers_ignore_case")?
                 .extract()?,
-            request.getattr("known_functions")?.extract()?,
-            request.getattr("known_types")?.extract()?,
+            native_field(request, "known_functions")?.extract()?,
+            native_field(request, "known_types")?.extract()?,
         );
-        let explicit: Bound<'py, PyAny> = request.getattr("catalog")?;
+        let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?;
         let own: bool = !explicit.is_none();
         let catalog: Py<BindingCatalog> = if own {
             explicit.extract()?
@@ -309,7 +310,7 @@ pub(crate) fn schema_validations<'py>(
         } else {
             let catalog: Py<BindingCatalog> = Py::new(
                 py,
-                BindingCatalog::create(py, key.clone(), &request.getattr("schema")?)?,
+                BindingCatalog::create(py, key.clone(), &native_field(request, "schema")?)?,
             )?;
             created.push((key, catalog.clone_ref(py)));
             catalog

crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs

@@ -28,6 +28,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_error;
 use crate::bindings::_helpers::compile_driver::python_calls::imported;
 use crate::bindings::_helpers::native_host::host;
 use crate::bindings::_helpers::native_host::user_code::UserCodeHost;
+use crate::bindings::_helpers::records::fields::field as native_field;
 use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId};
 use crate::bindings::_helpers::records::registry;
 use crate::bindings::types::CompilerDetach;
@@ -65,8 +66,8 @@ impl<'py> PythonSqlOwner<'py> {
         Ok(Self {
             label,
             kind,
-            function: node.getattr("function")?,
-            relative_path: node.getattr("relative_path")?,
+            function: native_field(node, "function")?,
+            relative_path: native_field(node, "relative_path")?,
             own_refs: Vec::new(),
             upstream_loader_by_source: Vec::new(),
         })
@@ -117,7 +118,7 @@ fn explicit_reference_report<'py>(
     }
     let request = ReferenceRequest {
         enforce_explicit,
-        dialect: project.getattr("sql_analysis_dialect")?.extract()?,
+        dialect: native_field(project, "sql_analysis_dialect")?.extract()?,
         explicit_help: if enforce_explicit {
             references_help(HARD_CODED_ALLOWED)
         } else {
@@ -158,7 +159,7 @@ fn python_sql_owners<'py>(
     for (label, attribute) in NODE_OWNERS {
         for node in discovered_inputs.getattr(attribute)?.try_iter()? {
             let node: Bound<'py, PyAny> = node?;
-            let name: String = node.getattr("name")?.extract()?;
+            let name: String = native_field(&node, "name")?.extract()?;
             owners.push(PythonSqlOwner::new(
                 format!("{label}:{name}"),
                 NODE_KIND,
@@ -169,21 +170,21 @@ fn python_sql_owners<'py>(
     owners.extend(loader_owners(project, discovered_inputs)?);
     let hook_names = registry::native(py, NativeAttr::ModelPythonHookNames)?;
     let mut attached: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
-    for model in project.getattr("models")?.try_iter()? {
+    for model in native_field(project, "models")?.try_iter()? {
         let model: Bound<'py, PyAny> = model?;
         let keywords: Bound<'py, PyDict> = PyDict::new(py);
         keywords.set_item("model", &model)?;
-        let model_name: String = model.getattr("name")?.extract()?;
+        let model_name: String = native_field(&model, "name")?.extract()?;
         for hook_name in hook_names.call((), Some(&keywords))?.try_iter()? {
             attached
                 .entry(hook_name?.extract()?)
                 .or_default()
                 .insert(model_name.clone());
         }
     }
-    for hook in project.getattr("hook_functions")?.try_iter()? {
+    for hook in native_field(project, "hook_functions")?.try_iter()? {
         let hook: Bound<'py, PyAny> = hook?;
-        let name: String = hook.getattr("name")?.extract()?;
+        let name: String = native_field(&hook, "name")?.extract()?;
         let mut owner = PythonSqlOwner::new(format!("hook:{name}"), HOOK_KIND, &hook)?;
         owner.own_refs = attached
             .get(&name)
@@ -203,7 +204,7 @@ fn loader_owners<'py>(
 ) -> PyResult<Vec<PythonSqlOwner<'py>>> {
     let py: Python<'py> = project.py();
     let mut sources_by_loader: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
-    for source in project.getattr("sources")?.try_iter()? {
+    for source in native_field(project, "sources")?.try_iter()? {
         let source: Bound<'py, PyAny> = source?;
         let loader: Option<String> = source
             .getattr("source_entry")?
@@ -213,16 +214,16 @@ fn loader_owners<'py>(
             sources_by_loader
                 .entry(loader)
                 .or_default()
-                .insert(source.getattr("name")?.extract()?);
+                .insert(native_field(&source, "name")?.extract()?);
         }
     }
     let read_definition = host::entry(py, UserCodeHost::ReadLoaderDefinition)?;
     let mut owners: Vec<PythonSqlOwner<'py>> = Vec::new();
-    for loader in discovered_inputs.getattr("loader_functions")?.try_iter()? {
+    for loader in native_field(discovered_inputs, "loader_functions")?.try_iter()? {
         let loader: Bound<'py, PyAny> = loader?;
-        let name: String = loader.getattr("name")?.extract()?;
+        let name: String = native_field(&loader, "name")?.extract()?;
         let mut upstream: Vec<(String, String)> = Vec::new();
-        for dependency in loader.getattr("depends_on")?.try_iter()? {
+        for dependency in native_field(&loader, "depends_on")?.try_iter()? {
             let dependency: Bound<'py, PyAny> = dependency?;
             if !dependency.is_callable() {
                 continue;
@@ -231,7 +232,7 @@ fn loader_owners<'py>(
             if definition.is_none() {
                 continue;
             }
-            let definition_name: String = definition.getattr("name")?.extract()?;
+            let definition_name: String = native_field(&definition, "name")?.extract()?;
             let shown: String = match dependency.getattr_opt("__name__")? {
                 Some(shown) => shown.extract()?,
                 None => definition_name.clone(),
@@ -341,7 +342,7 @@ fn python_report<'py>(
     }
     let literal_relation: Bound<'py, PyAny> =
         registry::class(reference_models.py(), RecordId::LiteralSqlRelation)?;
-    let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?;
+    let relation_name: Bound<'py, PyAny> = native_field(&reference_models, "RelationName")?;
     let mut unmatched: Vec<Bound<'py, PyAny>> = Vec::with_capacity(report.unmatched.len());
     for row in &report.unmatched {
         let owner: &PythonSqlOwner<'py> = &owners[row.owner];
@@ -392,8 +393,8 @@ pub(crate) fn hard_coded_relation_remedy(
     upstream_loader_by_source: Option<&Bound<'_, PyAny>>,
 ) -> PyResult<String> {
     let target: ResourceKey = (
-        r#ref.getattr("kind")?.getattr("value")?.extract()?,
-        r#ref.getattr("name")?.extract()?,
+        native_field(r#ref, "kind")?.getattr("value")?.extract()?,
+        native_field(r#ref, "name")?.extract()?,
     );
     let mut upstream_loaders: Vec<(String, String)> = Vec::new();
     if let Some(loaders) = upstream_loader_by_source.filter(|loaders| !loaders.is_none()) {