refactor: move compile assembly, lint and planner state to native data

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs

@@ -2,6 +2,7 @@
 
 use std::collections::{HashMap, HashSet, VecDeque};
 
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
 
 use crate::compiler::_helpers::sql_tests::authored_macro_calls::calls_macro;
@@ -15,6 +16,10 @@ use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_p
 use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end;
 use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed};
 
+/// Batches smaller than this are extracted on the calling thread.
+const PARALLEL_EXTRACTION_MINIMUM: usize = 64;
+const EXTRACTION_WORKERS: usize = 4;
+const EXTRACTION_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024;
 const MODEL_PREFIXES: [&str; 8] = [
     "__macro__",
     "__ref__",
@@ -134,10 +139,29 @@ pub(crate) fn extract_tests(
     tests: &[SqlTestRequest],
     syntax: &LexicalSyntax,
 ) -> Result<Vec<SqlTestClassification>, (usize, Failure)> {
-    tests
-        .iter()
+    let extracted: Vec<Result<SqlTestClassification, Failure>> =
+        if tests.len() < PARALLEL_EXTRACTION_MINIMUM {
+            tests
+                .iter()
+                .map(|test| extract_test(test, syntax))
+                .collect()
+        } else {
+            let pool: rayon::ThreadPool = rayon::ThreadPoolBuilder::new()
+                .num_threads(EXTRACTION_WORKERS)
+                .stack_size(EXTRACTION_WORKER_STACK_BYTES)
+                .build()
+                .map_err(|error| (0, Failure::from(error.to_string())))?;
+            pool.install(|| {
+                tests
+                    .par_iter()
+                    .map(|test| extract_test(test, syntax))
+                    .collect()
+            })
+        };
+    extracted
+        .into_iter()
         .enumerate()
-        .map(|(index, test)| extract_test(test, syntax).map_err(|failure| (index, failure)))
+        .map(|(index, result)| result.map_err(|failure| (index, failure)))
         .collect()
 }
 

crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs

@@ -13,6 +13,9 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_s
 use crate::bindings::_helpers::boundary::keyword_request::Request;
 use crate::bindings::_helpers::cache::native_store::current_code_identity;
 use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics;
+use crate::bindings::_helpers::compiled_project::model_input_view::{
+    AttachedSchema, attached_schema,
+};
 use crate::bindings::_helpers::native_host::executor::ExecutorHost;
 use crate::bindings::_helpers::native_host::host;
 use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId};
@@ -299,9 +302,11 @@ fn model_rows<'py>(
                 row.get_item(1)?.extract()?,
             ));
         }
-        let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?;
+        let schema: Option<AttachedSchema<'py>> = attached_schema(model_input)?;
         let recovers: bool = get("contract")?.eq(&enforced)?
-            || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?);
+            || schema
+                .as_ref()
+                .is_some_and(|schema| schema.type_enforcement);
         let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? {
             Some((
                 or_default(&get("valid_from_column")?, VALID_FROM_DEFAULT)?,
@@ -310,10 +315,9 @@ fn model_rows<'py>(
         } else {
             None
         };
-        let families = if entry.is_none() {
-            Vec::new()
-        } else {
-            family_rows(&entry.getattr("dynamic_columns")?)?
+        let families = match &schema {
+            Some(schema) => family_rows(&schema.dynamic_columns)?,
+            None => Vec::new(),
         };
         let row: Vec<Bound<'py, PyAny>> = vec![
             model_input

crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs

@@ -8,11 +8,11 @@ use sqlbuild_analysis::auditing::main::evaluate_measurement::evaluate_measuremen
 use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds};
 
 use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds;
+use crate::bindings::_helpers::native_host::facade::FacadeHost;
+use crate::bindings::_helpers::native_host::host;
 use crate::bindings::_helpers::records::ids::ErrorId;
 use crate::bindings::_helpers::records::registry;
 
-const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types";
-
 fn measurement_error(py: Python<'_>, message: &str) -> PyErr {
     match registry::error(py, ErrorId::MeasurementAudit).and_then(|class| class.call1((message,))) {
         Ok(error) => PyErr::from_value(error),
@@ -40,9 +40,7 @@ fn evaluate_measurement<'py>(
         MeasurementOutcome::Error => "ERROR",
         MeasurementOutcome::Insufficient => "INSUFFICIENT",
     };
-    py.import(AUDITING_TYPES_MODULE)?
-        .getattr("AuditOutcome")?
-        .getattr(member)
+    host::entry(py, FacadeHost::AuditOutcome)?.getattr(member)
 }
 
 pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {

crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs

@@ -1,24 +1,21 @@
 //! Project variable values rendered as the text SQL and config interpolation inserts.
 
+use crate::bindings::_helpers::native_host::facade::FacadeHost;
+use crate::bindings::_helpers::native_host::host;
 use pyo3::prelude::{
     Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python,
 };
 use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods};
 use pyo3::{pyfunction, wrap_pyfunction};
 
-const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions";
 const JSON_MODULE: &str = "json";
 /// Characters of a structured value's JSON preview before it is cut short.
 const PREVIEW_LIMIT: usize = 120;
 const PREVIEW_ELLIPSIS: &str = "...";
 
 /// `SharedInputError(message)`.
 fn shared_input_error(py: Python<'_>, message: String) -> PyErr {
-    match py
-        .import(ERRORS_MODULE)
-        .and_then(|module| module.getattr("SharedInputError"))
-        .and_then(|class| class.call1((message,)))
-    {
+    match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) {
         Ok(error) => PyErr::from_value(error),
         Err(error) => error,
     }

crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs

@@ -13,11 +13,10 @@ use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_
 
 use crate::bindings::_helpers::boundary::keyword_request::Request;
 use crate::bindings::_helpers::compile_driver::python_calls::imported;
+use crate::bindings::_helpers::native_host::facade::FacadeHost;
 use crate::bindings::_helpers::native_host::host;
 use crate::bindings::_helpers::native_host::user_code::UserCodeHost;
 
-const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions";
-const SHARED_INPUT_ERROR: &str = "SharedInputError";
 const RANDOM_MODULE: &str = "random";
 const MAX_ATTEMPTS_TOO_LOW: &str = "RetryPolicy max_attempts must be at least 1";
 const INITIAL_DELAY_NEGATIVE: &str = "RetryPolicy initial_delay_seconds must be non-negative";
@@ -36,8 +35,7 @@ const HOOK_READS_INVALID: &str =
 
 /// `SharedInputError(message)`.
 fn shared_input_error(py: Python<'_>, message: &str) -> PyErr {
-    match imported(py, ERRORS_MODULE, SHARED_INPUT_ERROR).and_then(|class| class.call1((message,)))
-    {
+    match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) {
         Ok(error) => PyErr::from_value(error),
         Err(error) => error,
     }

crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs

@@ -5,6 +5,7 @@ use pyo3::pyfunction;
 use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString};
 
 use crate::bindings::_helpers::compile_driver::python_calls::call_with;
+use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows;
 use crate::bindings::_helpers::records::ids::RecordId;
 use crate::bindings::_helpers::records::registry;
 
@@ -84,8 +85,8 @@ pub(crate) fn resolve_cursor_input_roles<'py>(
         Err(_) => {
             let cursor: Bound<'py, PyAny> = values.call_method1("get", ("cursor",))?;
             if cursor.is_instance_of::<PyString>() {
-                for reference in model.getattr("references")?.try_iter()? {
-                    filter.set_item(reference?.getattr("ref_name")?, &cursor)?;
+                for (_, name, _, _) in input_reference_rows(model)? {
+                    filter.set_item(name, &cursor)?;
                 }
             }
             if !rolling {

crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs

@@ -2,7 +2,7 @@
 
 use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
 use pyo3::pyfunction;
-use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple};
+use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple};
 use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help;
 
 use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers;
@@ -11,9 +11,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{
     KeywordArguments, call_with, imported,
 };
 use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes;
+use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput;
+use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop;
+use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject;
+use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile;
 use crate::bindings::_helpers::model_config::contract_classes::contract_classes;
 use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment;
-use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql;
+use crate::bindings::_helpers::render::declaration_values::ConstantRendering;
 use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes;
 
 use crate::bindings::_helpers::native_host::host;
@@ -51,31 +55,15 @@ fn opt_out_help(py: Python<'_>, model_owned: bool) -> PyResult<Bound<'_, PyAny>>
     Ok(PyString::new(py, &sql_analysis_opt_out_help(model_owned)).into_any())
 }
 
-/// The host the loop calls back into: `constant_text(*, declaration, file_path)`.
-fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
-    let py: Python<'py> = context.py();
-    let context: Py<PyAny> = context.clone().unbind();
-    let constant_text = move |arguments: &Bound<'_, PyTuple>,
-                              keywords: Option<&Bound<'_, PyDict>>|
-          -> PyResult<Py<PyAny>> {
-        let py: Python<'_> = arguments.py();
-        let context: &Bound<'_, PyAny> = context.bind(py);
-        let passed: Bound<'_, PyDict> =
-            keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone());
-        let file_path: Bound<'_, PyAny> = passed.as_any().get_item("file_path")?;
-        let rendered: String = constant_declaration_sql(
-            &passed.as_any().get_item("declaration")?,
-            &context.getattr("value_renderer")?,
-            (&context.getattr("collection_rendering")?, Some(&file_path)),
-        )?;
-        Ok(PyString::new(py, &rendered).into_any().unbind())
-    };
-    let keywords: Bound<'py, PyDict> = PyDict::new(py);
-    keywords.set_item(
-        "constant_text",
-        PyCFunction::new_closure(py, None, None, constant_text)?,
-    )?;
-    imported(py, "types", "SimpleNamespace")?.call((), Some(&keywords))
+/// The adapter's constant rendering every model, source and hook expansion shares.
+fn constant_rendering<'py>(context: &Bound<'py, PyAny>) -> PyResult<Bound<'py, ConstantRendering>> {
+    Bound::new(
+        context.py(),
+        ConstantRendering::new(
+            context.getattr("value_renderer")?.unbind(),
+            context.getattr("collection_rendering")?.unbind(),
+        ),
+    )
 }
 
 fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyDict>> {
@@ -283,14 +271,16 @@ pub(crate) fn build_model_inputs<'py>(
                 project_config_path.str()?,
             ),
         ))?;
-    let files: Bound<'py, PyList> = PyList::new(
-        py,
-        discovered_inputs
-            .getattr("model_files")?
-            .try_iter()?
-            .collect::<PyResult<Vec<_>>>()?,
+    let files: Vec<Py<NativeModelFile>> = discovered_inputs
+        .getattr("model_files")?
+        .try_iter()?
+        .map(|file| file?.extract())
+        .collect::<PyResult<_>>()?;
+    let inputs: Vec<Bound<'py, NativeModelInput>> = NativeModelLoop::run(
+        native_loop.cast::<NativeModelLoop>()?,
+        native_project.cast::<NativeCompiledProject>()?.clone(),
+        constant_rendering(context)?,
+        files,
     )?;
-    let inputs: Bound<'py, PyAny> =
-        native_loop.call_method1("run", (native_project, loop_host(context)?, files))?;
-    PyTuple::new(py, inputs.try_iter()?.collect::<PyResult<Vec<_>>>()?)
+    PyTuple::new(py, inputs)
 }