refactor(native): layer sqlbuild-python modules and add the sqlbuild-host crate

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "crates/sqlbuild-cache",
     "crates/sqlbuild-sqltext",
     "crates/sqlbuild-config",
+    "crates/sqlbuild-host",
     "crates/sqlbuild-recall",
     "crates/sqlbuild-discovery",
     "crates/sqlbuild-scopes",
@@ -33,6 +34,7 @@ order = [
     "sqlbuild-cache",
     "sqlbuild-sqltext",
     "sqlbuild-config",
+    "sqlbuild-host",
     "sqlbuild-recall",
     "sqlbuild-model-config",
     "sqlbuild-discovery",
@@ -105,6 +107,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" }
 sqlbuild-core = { path = "crates/sqlbuild-core" }
 sqlbuild-cursor = { path = "crates/sqlbuild-cursor" }
 sqlbuild-discovery = { path = "crates/sqlbuild-discovery" }
+sqlbuild-host = { path = "crates/sqlbuild-host" }
 sqlbuild-render = { path = "crates/sqlbuild-render" }
 sqlbuild-refactor = { path = "crates/sqlbuild-refactor" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }

crates/README.md

@@ -13,6 +13,7 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. |
 | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. |
 | `sqlbuild-config` | Configuration file reading without Python: TOML, YAML 1.2 with the core schema and merge keys, and typed project and local config readers for the fields discovery needs. It also writes the workspace files `sqb playground` and `sqb skills` create from packaged templates and skills. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. |
+| `sqlbuild-host` | The contract of the permanent Python edges as plain Rust: the catalogue of `sqlbuild._native_host` entries and the trait that resolves them, the per-thread compile timing collector with its process clock, the compile diagnostic sink, and the executor result types the Python result dataclasses and enums mirror. `sqlbuild-python` implements the traits. |
 | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. |
 | `sqlbuild-scopes` | Declaration scopes, visibility, relationship grants, placement, report queries and the offline scope cache, including dialect-aware SQL-test and scenario scans. |
 | `sqlbuild-model-config` | MODEL configuration, columns and audits, template expansion over caller values, and dynamic-column name patterns using Rust regex syntax. |
@@ -31,8 +32,8 @@ Dependencies point one way, from the top of this graph to the bottom:
 ```text
 sqlbuild-python
   -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
-  -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config
-  -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
+  -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
+  -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
 sqlbuild-cli -> sqlbuild-cache, sqlbuild-core
 ```
 
@@ -58,6 +59,14 @@ version other than its manifest's, because the version is part of the build iden
 `fensu check` applies the Rust structure rules inside each crate: other crates use items through
 `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`.
 
+Inside `sqlbuild-python`, the custom rule XSB075 orders the `bindings/_helpers` modules in layers
+that follow the planned crate split: shared Python plumbing (`infra`), then the glue of the lower
+crates (`front`), the compiled project (`project`), planning and compilation as siblings (`plan`,
+`compile`), command output (`output`), command handlers (`commands`) and registration
+(`registry`). A module may import only from lower layers; `plan` and `compile` may not import each
+other. `module_layers` in `fensu.toml` assigns every helper module a layer. `pending_budgets` is
+empty: every import points down, and a new upward import fails the check.
+
 ## Python module paths
 
 `sqlbuild-python` and `sqlbuild-cli` import Python by module path strings. The repository test

crates/sqlbuild-host/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "sqlbuild-host"
+version.workspace = true
+description = "The contract of the permanent Python edges: host entries and traits, compile timings, diagnostics and executor results."
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+sqlbuild-core.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-host/src/compile_context/_helpers/active.rs

@@ -0,0 +1,21 @@
+//! The thread's active collector slot.
+
+use std::cell::RefCell;
+
+use crate::compile_context::types::CompileTimingsHandle;
+
+thread_local! {
+    static ACTIVE: RefCell<Option<CompileTimingsHandle>> = const { RefCell::new(None) };
+}
+
+/// Make `collector` the thread's active collector; return the one it replaces.
+pub(crate) fn replace_active(
+    collector: Option<CompileTimingsHandle>,
+) -> Option<CompileTimingsHandle> {
+    ACTIVE.with(|active| active.replace(collector))
+}
+
+/// The thread's active collector, if a compile is collecting.
+pub(crate) fn active() -> Option<CompileTimingsHandle> {
+    ACTIVE.with(|active| active.borrow().clone())
+}

crates/sqlbuild-host/src/compile_context/_helpers/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod active;

crates/sqlbuild-host/src/compile_context/main/add_compile_timing.rs

@@ -0,0 +1,14 @@
+//! Add elapsed time to one compile phase.
+
+use crate::compile_context::_helpers::active::active;
+
+/// Add elapsed nanoseconds to a phase when a collector is active.
+pub fn add_compile_timing(phase: &str, elapsed_ns: u64) {
+    if let Some(collector) = active() {
+        *collector
+            .borrow_mut()
+            .elapsed_ns
+            .entry(phase.to_owned())
+            .or_default() += elapsed_ns;
+    }
+}

crates/sqlbuild-host/src/compile_context/main/collect_compile_timings.rs

@@ -0,0 +1,17 @@
+//! Collect the timings one compile records on this thread.
+
+use std::rc::Rc;
+
+use crate::compile_context::_helpers::active::replace_active;
+use crate::compile_context::types::CompileTimingsHandle;
+
+/// Run `body` with a fresh collector active on this thread, restoring the previous one after.
+pub fn collect_compile_timings<T, E>(
+    body: impl FnOnce(&CompileTimingsHandle) -> Result<T, E>,
+) -> Result<T, E> {
+    let collector: CompileTimingsHandle = Rc::default();
+    let previous: Option<CompileTimingsHandle> = replace_active(Some(Rc::clone(&collector)));
+    let result: Result<T, E> = body(&collector);
+    let _ = replace_active(previous);
+    result
+}

crates/sqlbuild-host/src/compile_context/main/mod.rs

@@ -0,0 +1,6 @@
+pub mod add_compile_timing;
+pub mod collect_compile_timings;
+pub mod record_compile_cpu_timing;
+pub mod record_compile_metric;
+pub mod record_compile_timing;
+pub mod timing_milliseconds;

crates/sqlbuild-host/src/compile_context/main/record_compile_cpu_timing.rs

@@ -0,0 +1,21 @@
+//! Record the process CPU time of one compile phase.
+
+use crate::compile_context::_helpers::active::active;
+use crate::compile_context::main::add_compile_timing::add_compile_timing;
+use crate::compile_context::types::ProcessClock;
+
+/// Record the process CPU time `body` spans, including its worker threads, under `phase`.
+pub fn record_compile_cpu_timing<C: ProcessClock, T>(
+    clock: &C,
+    phase: &str,
+    body: impl FnOnce() -> Result<T, C::Error>,
+) -> Result<T, C::Error> {
+    if active().is_none() {
+        return body();
+    }
+    let started: u64 = clock.process_time_ns()?;
+    let result: Result<T, C::Error> = body();
+    let ended: u64 = clock.process_time_ns()?;
+    add_compile_timing(phase, ended.saturating_sub(started));
+    result
+}

crates/sqlbuild-host/src/compile_context/main/record_compile_metric.rs

@@ -0,0 +1,14 @@
+//! Add to one compile metric.
+
+use crate::compile_context::_helpers::active::active;
+
+/// Add a non-negative metric value when a collector is active.
+pub fn record_compile_metric(metric: &str, value: i64) {
+    if let Some(collector) = active() {
+        *collector
+            .borrow_mut()
+            .metrics
+            .entry(metric.to_owned())
+            .or_default() += u64::try_from(value).unwrap_or(0);
+    }
+}

crates/sqlbuild-host/src/compile_context/main/record_compile_timing.rs

@@ -0,0 +1,23 @@
+//! Record the wall time of one compile phase.
+
+use std::time::Instant;
+
+use crate::compile_context::_helpers::active::active;
+use crate::compile_context::main::add_compile_timing::add_compile_timing;
+
+/// Record the wall time `body` spans under `phase` when a collector is active.
+pub fn record_compile_timing<T, E>(
+    phase: &str,
+    body: impl FnOnce() -> Result<T, E>,
+) -> Result<T, E> {
+    if active().is_none() {
+        return body();
+    }
+    let started: Instant = Instant::now();
+    let result: Result<T, E> = body();
+    add_compile_timing(
+        phase,
+        u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX),
+    );
+    result
+}

crates/sqlbuild-host/src/compile_context/main/timing_milliseconds.rs

@@ -0,0 +1,11 @@
+//! A collector's timings as `--json` reports them.
+
+use sqlbuild_core::compile_timings::main::compile_timing_milliseconds::compile_timing_milliseconds;
+
+use crate::compile_context::types::CompileTimingsHandle;
+
+/// Every reported phase in milliseconds, then every metric.
+#[must_use]
+pub fn timing_milliseconds(collector: &CompileTimingsHandle) -> Vec<(&'static str, u64)> {
+    compile_timing_milliseconds(&collector.borrow())
+}

crates/sqlbuild-host/src/compile_context/mod.rs

@@ -0,0 +1,8 @@
+//! Per-compile host state: the thread's timing collector and the diagnostic sink producers report into.
+
+pub(crate) mod _helpers;
+pub mod main;
+pub mod types;
+
+#[cfg(test)]
+mod tests;

crates/sqlbuild-host/src/compile_context/tests.rs

@@ -0,0 +1,6 @@
+#[path = "tests/test_collect_compile_timings.rs"]
+mod collect_compile_timings;
+#[path = "tests/helpers.rs"]
+mod helpers;
+#[path = "tests/test_types.rs"]
+mod test_types;

crates/sqlbuild-host/src/compile_context/tests/helpers.rs

@@ -0,0 +1,45 @@
+use sqlbuild_core::compile_timings::models::CompileTimings;
+
+use crate::compile_context::main::add_compile_timing::add_compile_timing;
+use crate::compile_context::main::collect_compile_timings::collect_compile_timings;
+use crate::compile_context::main::record_compile_cpu_timing::record_compile_cpu_timing;
+use crate::compile_context::main::record_compile_metric::record_compile_metric;
+use crate::compile_context::main::record_compile_timing::record_compile_timing;
+use crate::compile_context::main::timing_milliseconds::timing_milliseconds;
+use crate::compile_context::tests::test_types::SteppingClock;
+
+/// Record one timing, one CPU timing, two metric values and one wall timing.
+fn record_all(clock: &SteppingClock) -> Result<(), ()> {
+    add_compile_timing("attachment_ms", 2_000_000);
+    record_compile_cpu_timing(clock, "model_analysis_cpu_ms", || Ok(()))?;
+    record_compile_metric("analysis_cache_misses", 3);
+    record_compile_metric("analysis_cache_misses", -1);
+    record_compile_timing("test_planning_ms", || Ok(()))
+}
+
+/// The timings `record_all` leaves in a collection around it.
+pub(super) fn collected(clock: &SteppingClock) -> Option<CompileTimings> {
+    collect_compile_timings(|collector| {
+        record_all(clock)?;
+        Ok::<_, ()>(collector.borrow().clone())
+    })
+    .ok()
+}
+
+/// No timings: `record_all` runs outside any collection.
+pub(super) fn uncollected(clock: &SteppingClock) -> Option<CompileTimings> {
+    record_all(clock).ok().and(None)
+}
+
+/// The outer collection's first phase in milliseconds after a nested collection ends.
+pub(super) fn outer_attachment_ms(inner_ns: u64, outer_ns: u64) -> Option<u64> {
+    collect_compile_timings(|outer| {
+        collect_compile_timings(|_inner| {
+            add_compile_timing("attachment_ms", inner_ns);
+            Ok::<(), ()>(())
+        })?;
+        add_compile_timing("attachment_ms", outer_ns);
+        Ok::<_, ()>(timing_milliseconds(outer)[0].1)
+    })
+    .ok()
+}

crates/sqlbuild-host/src/compile_context/tests/test_collect_compile_timings.rs

@@ -0,0 +1,84 @@
+use std::cell::Cell;
+
+use crate::compile_context::tests::helpers::{collected, outer_attachment_ms, uncollected};
+use crate::compile_context::tests::test_types::{
+    NestingTestCase, RecordingTestCase, SteppingClock,
+};
+
+#[test]
+fn given_recorders_when_a_compile_collects_then_only_the_active_collector_receives_them() {
+    let test_cases = [
+        RecordingTestCase {
+            description: "a collecting compile receives every recording",
+            run: collected,
+            expected_elapsed_ns: Some(2_000_000),
+            expected_cpu_ns: Some(5),
+            expected_metric: Some(3),
+            expected_wall_recorded: true,
+        },
+        RecordingTestCase {
+            description: "recordings outside a collection are dropped",
+            run: uncollected,
+            expected_elapsed_ns: None,
+            expected_cpu_ns: None,
+            expected_metric: None,
+            expected_wall_recorded: false,
+        },
+    ];
+    for test_case in test_cases {
+        let clock = SteppingClock {
+            now: Cell::new(0),
+            step: 5,
+        };
+        let timings = (test_case.run)(&clock);
+        let phase = |name: &str| {
+            timings
+                .as_ref()
+                .and_then(|t| t.elapsed_ns.get(name).copied())
+        };
+        assert_eq!(
+            phase("attachment_ms"),
+            test_case.expected_elapsed_ns,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            phase("model_analysis_cpu_ms"),
+            test_case.expected_cpu_ns,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            timings
+                .as_ref()
+                .and_then(|t| t.metrics.get("analysis_cache_misses").copied()),
+            test_case.expected_metric,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            phase("test_planning_ms").is_some(),
+            test_case.expected_wall_recorded,
+            "{}",
+            test_case.description
+        );
+    }
+}
+
+#[test]
+fn given_a_nested_collection_when_it_ends_then_the_outer_collector_is_active_again() {
+    let test_cases = [NestingTestCase {
+        description: "the inner collection's timing stays out of the outer one",
+        inner_ns: 9_000_000,
+        outer_ns: 1_000_000,
+        expected_outer_ms: 1,
+    }];
+    for test_case in test_cases {
+        assert_eq!(
+            outer_attachment_ms(test_case.inner_ns, test_case.outer_ns),
+            Some(test_case.expected_outer_ms),
+            "{}",
+            test_case.description
+        );
+    }
+}