refactor(compile): drop fine-grained edit reuse in favour of coarse caches

.github/workflows/verify.yml

@@ -561,11 +561,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # Temporary: a one-model edit recompiles the edited model uncached until native
+        # rendering becomes the default; tighten edit_max_ratio back to max_ratio then.
         include:
           - kind: dense
             max_ratio: "1.10"
+            edit_max_ratio: "1.25"
           - kind: fresh
             max_ratio: "1.10"
+            edit_max_ratio: "1.20"
     steps:
       - uses: actions/checkout@v4
       - name: Check out the comparison base
@@ -596,11 +600,13 @@ jobs:
         env:
           KIND: ${{ matrix.kind }}
           MAX_RATIO: ${{ matrix.max_ratio }}
+          EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }}
         run: |
           uv run python -m scripts.compare_compile_performance \
             --kind "${KIND}" \
             --modes cold warm edit \
             --max-ratio "${MAX_RATIO}" \
+            --edit-max-ratio "${EDIT_MAX_RATIO}" \
             --models 3000 \
             --base-root base \
             --base-python base/.venv/bin/python \

crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs

@@ -1 +1,3 @@
 pub(crate) mod native_store;
+pub(crate) mod registration;
+pub(crate) mod sql_test_scan_store;

crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs

@@ -1,4 +1,4 @@
-//! Digests and project fingerprints of the shared native store, with the GIL released.
+//! Digests, project fingerprints and store files of the shared native store, with the GIL released.
 
 use pyo3::exceptions::{PyOSError, PyRuntimeError};
 use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python};
@@ -8,8 +8,10 @@ use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each;
 use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint;
 use sqlbuild_cache::digest::main::hex_digest::hex_digest;
 use sqlbuild_cache::digest::types::ContentDigest;
+use sqlbuild_cache::store::main::open_native_store::open_native_store;
+use sqlbuild_cache::store::models::NativeStore;
 use std::collections::HashSet;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use crate::bindings::types::CompilerDetach;
 
@@ -50,6 +52,34 @@ fn readable_digest(digest: &std::io::Result<ContentDigest>) -> Option<String> {
     }
 }
 
+/// Open one cache kind's store with the GIL released; a failed read is an `OSError`.
+pub(crate) fn open_store(
+    py: Python<'_>,
+    path: &Path,
+    kind: &str,
+    environment: &str,
+) -> PyResult<NativeStore> {
+    py.compiler_detach(|| Ok(open_native_store(path, kind, environment)))
+        .map_err(PyRuntimeError::new_err)?
+        .map_err(|error| PyOSError::new_err(error.to_string()))
+}
+
+/// Atomically save a store holding unsaved values with the GIL released; returns entries written.
+pub(crate) fn save_store(
+    py: Python<'_>,
+    store: &NativeStore,
+    path: &Path,
+    metadata: &[u8],
+) -> PyResult<Option<usize>> {
+    if !store.needs_save() {
+        return Ok(None);
+    }
+    py.compiler_detach(|| Ok(store.save(path, metadata)))
+        .map_err(PyRuntimeError::new_err)?
+        .map(Some)
+        .map_err(|error| PyOSError::new_err(error.to_string()))
+}
+
 pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(content_digest, module)?)?;
     module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?;

crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs

@@ -0,0 +1,11 @@
+//! Register the shared native store functions and classes.
+
+use pyo3::prelude::{Bound, PyModule, PyResult};
+
+use crate::bindings::_helpers::cache::{native_store, sql_test_scan_store};
+
+pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    native_store::register(module)?;
+    sql_test_scan_store::register(module)?;
+    Ok(())
+}

crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs

@@ -0,0 +1,67 @@
+//! Whole per-file SQL-test scan results kept as one kind of the shared native store.
+
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python};
+use pyo3::{pyclass, pymethods};
+use sqlbuild_cache::digest::main::content_digest::content_digest;
+use sqlbuild_cache::digest::types::ContentDigest;
+use sqlbuild_cache::store::models::NativeStore;
+use std::path::PathBuf;
+use std::sync::Mutex;
+
+use crate::bindings::_helpers::cache::native_store::{open_store, save_store};
+
+/// The native store kind holding SQL-test scan results.
+const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans";
+
+/// One compile's view of the stored scan results, keyed by the digest of the caller's key parts.
+#[pyclass(module = "sqlbuild._native", frozen)]
+#[derive(Debug)]
+pub(crate) struct SqlTestScanStore {
+    store: Mutex<NativeStore>,
+}
+
+fn poisoned<T>(_: T) -> pyo3::PyErr {
+    PyValueError::new_err("SQL test scan store is poisoned")
+}
+
+#[pymethods]
+impl SqlTestScanStore {
+    /// Open the store at `path` for `environment`; missing, damaged or foreign files open empty.
+    #[new]
+    fn new(py: Python<'_>, path: PathBuf, environment: &str) -> PyResult<Self> {
+        let store: NativeStore = open_store(py, &path, SQL_TEST_SCAN_STORE_KIND, environment)?;
+        Ok(Self {
+            store: Mutex::new(store),
+        })
+    }
+
+    /// The value stored under the digest of `key_parts`, or `None`.
+    fn get(&self, key_parts: Vec<String>) -> PyResult<Option<Vec<u8>>> {
+        let key: ContentDigest = content_digest(&key_parts);
+        Ok(self
+            .store
+            .lock()
+            .map_err(poisoned)?
+            .get(&key)
+            .map(<[u8]>::to_vec))
+    }
+
+    /// Store `value` under the digest of `key_parts`.
+    fn put(&self, key_parts: Vec<String>, value: Vec<u8>) -> PyResult<()> {
+        let key: ContentDigest = content_digest(&key_parts);
+        self.store.lock().map_err(poisoned)?.put(key, value);
+        Ok(())
+    }
+
+    /// Atomically save the store when this compile stored a value; returns entries written.
+    fn save(&self, py: Python<'_>, path: PathBuf) -> PyResult<Option<usize>> {
+        let store = self.store.lock().map_err(poisoned)?;
+        save_store(py, &store, &path, &[])
+    }
+}
+
+pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add_class::<SqlTestScanStore>()?;
+    Ok(())
+}

crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs

@@ -1,15 +1,14 @@
 //! The macro call memo class: one compile's recorded calls and its optional native store.
 
-use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError};
+use pyo3::exceptions::PyValueError;
 use pyo3::prelude::Python;
 use pyo3::{PyResult, pyclass, pymethods};
-use sqlbuild_cache::store::main::open_native_store::open_native_store;
 use sqlbuild_cache::store::models::NativeStore;
 use sqlbuild_render::macro_calls::models::MacroCallEntry;
 use std::path::PathBuf;
 use std::sync::Mutex;
 
-use crate::bindings::types::CompilerDetach;
+use crate::bindings::_helpers::cache::native_store::{open_store, save_store};
 
 use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row};
 
@@ -80,10 +79,7 @@ impl MacroCallMemo {
         path: PathBuf,
         environment: &str,
     ) -> PyResult<(Vec<u8>, usize)> {
-        let store: NativeStore = py
-            .compiler_detach(|| Ok(open_native_store(&path, MACRO_CALL_STORE_KIND, environment)))
-            .map_err(PyRuntimeError::new_err)?
-            .map_err(|error| PyOSError::new_err(error.to_string()))?;
+        let store: NativeStore = open_store(py, &path, MACRO_CALL_STORE_KIND, environment)?;
         let opened = (store.metadata().to_vec(), store.loaded_entries());
         self.inner.lock().map_err(poisoned)?.attach_store(store);
         Ok(opened)
@@ -119,13 +115,9 @@ impl MacroCallMemo {
         metadata: &[u8],
     ) -> PyResult<Option<usize>> {
         let mut memo = self.inner.lock().map_err(poisoned)?;
-        let Some(store) = memo.store_mut().filter(|store| store.needs_save()) else {
+        let Some(store) = memo.store_mut() else {
             return Ok(None);
         };
-        let store: &NativeStore = store;
-        py.compiler_detach(|| Ok(store.save(&path, metadata)))
-            .map_err(PyRuntimeError::new_err)?
-            .map(Some)
-            .map_err(|error| PyOSError::new_err(error.to_string()))
+        save_store(py, store, &path, metadata)
     }
 }

crates/sqlbuild-python/src/bindings/main/register.rs

@@ -32,7 +32,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> {
     scopes::registration::register(module)?;
     model_config::registration::register(module)?;
     render::macro_calls::register(module)?;
-    cache::native_store::register(module)?;
+    cache::registration::register(module)?;
     attachments::registration::register(module)?;
     register_analysis_stages(module)?;
     module.add("API_VERSION", API_VERSION)?;

fensu.toml

@@ -177,12 +177,6 @@ path = "src/sqlbuild/runtime/event_exporting/main/command_output_sink.py"
 symbols = ["command_output_sink"]
 reason = "Bare command-output sink decorators are evaluated as decorator(function), requiring positional function calling."
 
-[[targets.python.rule_exceptions]]
-rule = "FFS120"
-path = "src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py"
-symbols = ["_RestrictedUnpickler.find_class"]
-reason = "This override preserves pickle.Unpickler's positional find_class(module, name) protocol."
-
 [[targets.python.rule_exceptions]]
 rule = "FFR402"
 path = "src/sqlbuild/integrations/dagster/__init__.py"

scripts/compile_performance_ratio/_helpers/report.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import platform
+from collections.abc import Mapping
 from pathlib import Path
 
 from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES
@@ -16,6 +17,7 @@ def comparison_markdown(
     max_ratio: float,
     per_side_projects: bool,
     failures: tuple[str, ...],
+    mode_max_ratios: Mapping[str, float] | None = None,
 ) -> str:
     """Summarize medians, ratios and per-phase timings of every mode as Markdown tables."""
 
@@ -31,7 +33,8 @@ def comparison_markdown(
                 f"### {comparison.kind} {comparison.models} models, "
                 f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)",
                 "",
-                f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
+                f"Runner CPU: {cpu_model()}; {runs} alternating runs each; "
+                f"limit {(mode_max_ratios or {}).get(comparison.mode, max_ratio):.2f}x.",
                 "",
                 generation,
                 "",

scripts/compile_performance_ratio/_helpers/verdict.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import math
+from collections.abc import Mapping
 
 from scripts.compile_performance_ratio.models import CompileComparison
 
@@ -13,6 +14,7 @@ def ratio_failures(
     modes: tuple[str, ...],
     max_ratio: float,
     noise_floor_seconds: float,
+    mode_max_ratios: Mapping[str, float] | None = None,
 ) -> tuple[str, ...]:
     """Describe every requested mode that is unmeasured or slower than ratio plus floor allow."""
 
@@ -26,6 +28,7 @@ def ratio_failures(
             failures.append(f"{mode}: no measurement")
             continue
         prefix: str = f"{comparison.kind} {comparison.models} {mode}"
+        limit: float = (mode_max_ratios or {}).get(mode, max_ratio)
         for metric, base, head, ratio in (
             (
                 "wall",
@@ -39,8 +42,8 @@ def ratio_failures(
                 failures.append(
                     f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)"
                 )
-            elif head > base * max_ratio + noise_floor_seconds:
-                failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}")
+            elif head > base * limit + noise_floor_seconds:
+                failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {limit:.2f}")
     return tuple(failures)
 
 

scripts/compile_performance_ratio/main/compare.py

@@ -19,6 +19,7 @@
     COMPILE_MODES,
     DEFAULT_MAX_RATIO,
     DEFAULT_RUNS,
+    EDIT_MODE,
     NOISE_FLOOR_SECONDS,
     PROJECT_KINDS,
 )
@@ -59,18 +60,23 @@ def compare_compile_performance(argv: list[str] | None = None) -> int:
             runs=args.runs,
             modes=modes,
         )
+    mode_max_ratios: dict[str, float] = (
+        {} if args.edit_max_ratio is None else {EDIT_MODE: args.edit_max_ratio}
+    )
     failures: tuple[str, ...] = ratio_failures(
         comparisons=comparisons,
         modes=modes,
         max_ratio=args.max_ratio,
         noise_floor_seconds=args.noise_floor_seconds,
+        mode_max_ratios=mode_max_ratios,
     )
     markdown: str = comparison_markdown(
         comparisons=comparisons,
         runs=args.runs,
         max_ratio=args.max_ratio,
         per_side_projects=per_side_projects,
         failures=failures,
+        mode_max_ratios=mode_max_ratios,
     )
     print(markdown)
     _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown)
@@ -104,6 +110,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     )
     parser.add_argument("--runs", type=int, default=DEFAULT_RUNS)
     parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO)
+    parser.add_argument(
+        "--edit-max-ratio",
+        type=float,
+        default=None,
+        help="Ratio limit of the one-model edit mode only; defaults to --max-ratio.",
+    )
     parser.add_argument(
         "--noise-floor-seconds",
         type=float,

scripts/compiler_differential/constants.py

@@ -432,7 +432,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]:
         "external_sql_reference_resolver",
         "scope_index",
         "declaration_scope",
-        "analysis_reuse",
     }
 )
 RENDER_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS