perf(compile): reuse rule results on warm and one-edit compiles (#915)

* perf(compile): reuse custom-rule results whose recorded fact reads are unchanged

The isolated custom-rule host now records which compiler facts each rule
invocation reads through its context. Results are cached per rule and subject
and reused only while every recorded fact digest, the rule implementation and
import closure, the rules configuration, the installed code identity, and the
dialect are unchanged. Rules whose reachable module state changes between
invocations, or that touch private view state, are treated as reading every
fact.

Custom rules no longer serialise the whole compiled project for a cache key:
the project is pickled only when a host run is needed, and built-in rules key
their project-level identity on request facts instead of walking project files.

Refs CHI-443, CHI-444

* perf(compile): reuse the built-in rules response for an identical native request

Warm compiles re-sent an unchanged 60 MB request to the native engine,
which spent about a second re-validating per-model cache entries. The
complete response is now memoized under a digest of the serialized request
and the installed code identity, so any fact, scope, config or code change
re-evaluates through the native per-model cache.

Refs CHI-445

* perf(compile): resolve macro context injection once per loaded macro

Every macro call rebuilt the callable's inspect signature to decide whether
to inject the macro context. The decision is now a cached property of the
loaded macro, removing about 3,600 signature constructions from a dense
3,000-model compile.

Refs CHI-442

* perf(compile): guard dense warm and one-edit compile budgets and oracles

Adds 3,000- and 5,000-model dense guards that compile cold, warm, and after
a single model edit, then compare the edit with a from-scratch uncached
oracle. They assert exact fingerprints, complete warm cache reuse, and that
the edit re-evaluates only the edited model's built-in rules while the
project-wide custom rule is reused. The dense job timeout grows to cover the
extra compiles.

Refs CHI-441

* fix(rules): verify filesystem reads and conservatively key untracked custom-rule results

- Record the digest of each project-tree value a rule actually read in the
  host and refuse to store a result whose filesystem facts changed before
  the parent recorded the fingerprint.
- Key untracked rules on their recorded reads plus every observable fact,
  so files outside the discovered tree (hidden directories) still
  invalidate them.
- Never cache rules that touch private or unknown fact-view attributes.
- Include the thread id in the custom-rule cache temporary file name.

Author: kvlonge

.github/workflows/verify.yml

@@ -365,7 +365,7 @@ jobs:
     needs: [changes, lockfile]
     if: needs.changes.outputs.performance == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:

crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs

@@ -73,6 +73,12 @@ fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
         .map_err(value_error)
 }
 
+#[pyfunction]
+fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult<String> {
+    py.compiler_detach(|| crate::engine::main::custom_host::run_custom_host_json(spec_json))
+        .map_err(value_error)
+}
+
 #[pyfunction]
 fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
     py.compiler_detach(|| crate::sql_lint::main::engine::lint_json(request_json))
@@ -422,6 +428,7 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String {
 pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(evaluate_json, module)?)?;
     module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?;
+    module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?;
     module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?;
     module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?;
     module.add(

crates/sqlbuild-rules-native/src/constants.rs

@@ -13,7 +13,6 @@ pub(crate) const GENERIC_BASIC_NAME: &str = "basic";
 pub(crate) const GENERIC_SCENARIO_NAME: &str = "scenario";
 pub(crate) const GENERIC_CASE_NAME: &str = "case";
 pub(crate) const CUSTOM_RULE_NAMESPACE: &str = "XSQBR";
-pub(crate) const TARGET_DIRECTORY: &str = "target";
 pub(crate) const REFERENCE_KIND: &str = "ref";
 pub(crate) const MODEL_SCHEMA_CONFIG_KEY: &str = "schema";
 pub(crate) const STAGING_LAYER_DIRECTORY: &str = "staging";
@@ -33,11 +32,6 @@ pub(crate) const NEGATION_OPERATOR: &str = "-";
 pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3;
 pub(crate) const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2;
 pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3;
-pub(crate) const CUSTOM_HOST_REQUIRED_ERROR: &str = "selected custom rules require a custom host";
-pub(crate) const LOGS_DIRECTORY: &str = "logs";
-pub(crate) const GIT_DIRECTORY: &str = ".git";
-pub(crate) const RULES_DIRECTORY: &str = "rules";
-pub(crate) const PYTHON_EXTENSION: &str = "py";
 pub(crate) const SQL_WILDCARD: &str = "*";
 pub(crate) const UNKNOWN_SQL_TYPE: &str = "UNKNOWN";
 pub(crate) const VARCHAR_SQL_TYPE: &str = "VARCHAR";

crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs

@@ -1,6 +1,4 @@
 use crate::models::Fault;
-use fensu_policy::lifecycle::models::CacheRead;
-use fensu_policy::{read_cache, write_cache};
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
@@ -28,36 +26,6 @@ impl Cache {
         })
     }
 
-    pub(crate) fn get(&self, path: &str, fingerprint: &str) -> Result<Option<Vec<Fault>>, String> {
-        match read_cache::<Vec<Fault>>(&self.root, path, fingerprint)
-            .map_err(|error| format!("could not read rules cache: {error}"))?
-        {
-            CacheRead::Hit(faults) => Ok(Some(faults)),
-            CacheRead::Miss | CacheRead::Invalidated => Ok(None),
-        }
-    }
-
-    pub(crate) fn put(
-        &self,
-        path: &str,
-        fingerprint: &str,
-        faults: &[Fault],
-    ) -> Result<(), String> {
-        write_cache(&self.root, path, fingerprint, &faults)
-            .map_err(|error| format!("could not write rules cache: {error}"))
-    }
-
-    pub(crate) fn custom_rule_bucket(
-        &self,
-        code: &str,
-        rule_identity: &str,
-    ) -> Result<RuleCacheBucket, String> {
-        read_bucket(
-            &self.root.join("bulk").join(format!("custom-{code}.json")),
-            rule_identity,
-        )
-    }
-
     pub(crate) fn native_rules_bucket(
         &self,
         ruleset_identity: &str,
@@ -76,19 +44,6 @@ impl Cache {
             bucket,
         )
     }
-
-    pub(crate) fn put_custom_rule_bucket(
-        &self,
-        code: &str,
-        rule_identity: &str,
-        bucket: &RuleCacheBucket,
-    ) -> Result<(), String> {
-        write_bucket(
-            &self.root.join("bulk").join(format!("custom-{code}.json")),
-            rule_identity,
-            bucket,
-        )
-    }
 }
 
 fn read_bucket(path: &Path, fingerprint: &str) -> Result<RuleCacheBucket, String> {

crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs

@@ -0,0 +1,29 @@
+use crate::models::CustomHostSpec;
+use fensu_policy::lifecycle::models::{
+    CustomHostInvocation, CustomHostOutputLimits, CustomHostRequest,
+};
+use fensu_policy::run_custom_host;
+use std::path::Path;
+use std::time::Duration;
+
+pub(crate) fn run_custom_host_json(spec_json: &str) -> Result<String, String> {
+    let spec: CustomHostSpec = serde_json::from_str(spec_json)
+        .map_err(|error| format!("invalid custom host request: {error}"))?;
+    let request = CustomHostRequest {
+        protocol: fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION,
+        runtime_version: spec.runtime_version.clone(),
+        payload: spec.payload,
+    };
+    let response = run_custom_host::<_, serde_json::Value>(CustomHostInvocation {
+        program: Path::new(&spec.program),
+        arguments: &spec.arguments,
+        timeout: Duration::from_millis(spec.timeout_millis),
+        output_limits: CustomHostOutputLimits::default(),
+        request: &request,
+    })
+    .map_err(|error| format!("invalid rules lifecycle: {error}"))?;
+    let payload = response.payload.ok_or_else(|| {
+        "custom rule host returned no payload after successful validation".to_owned()
+    })?;
+    serde_json::to_string(&payload).map_err(|error| error.to_string())
+}