perf(compile): plan SQL tests in the background in every mode (#926)

* perf(compile): collect SQL-test relation markers without a JSON tree

Native SQL-test planning serialized each parsed model statement to a
serde_json Value only to find its FROM and JOIN marker calls. A streaming
serializer now reproduces that walk, including key order, repeated-key
replacement and to_value failures, cutting native planning CPU by about
40 percent. A JSON-walk oracle in the tests pins the equivalence.

* perf(compile): plan SQL tests in the background in every compile mode

SQL-test planning only ran in the background for uncached projects of
128 to 5000 models with rules selected, so cache-on and large compiles
planned on the critical path in the write phase. Prepared artifacts now
fall back to planning SQL tests alone in the background, against the
published target and its artifact cache, and the write phase consumes
that plan at the point it would otherwise plan inline. Results and
errors are identical to inline planning.

* style: reword a docstring the setting-help scanner misread

Author: kvlonge

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/errors.rs

@@ -0,0 +1,23 @@
+//! Error types owned by SQL-test planning helpers.
+
+use std::fmt::Display;
+
+use serde::ser;
+
+/// A parsed statement that `serde_json::to_value` could not represent.
+#[derive(Debug)]
+pub(crate) struct RelationMarkerError;
+
+impl Display for RelationMarkerError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str("value cannot be represented as JSON")
+    }
+}
+
+impl std::error::Error for RelationMarkerError {}
+
+impl ser::Error for RelationMarkerError {
+    fn custom<M: Display>(_message: M) -> Self {
+        RelationMarkerError
+    }
+}

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs

@@ -1,9 +1,11 @@
 pub(crate) mod cte_rename;
 pub(crate) mod cte_slices;
 pub(crate) mod cte_sql;
+pub(crate) mod errors;
 pub(crate) mod expected_columns;
 pub(crate) mod extraction;
 pub(crate) mod helper_scope;
 pub(crate) mod markers;
 pub(crate) mod planning;
+pub(crate) mod relation_markers;
 pub(crate) mod rendering;

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs

@@ -8,12 +8,12 @@ use polyglot_sql::{Dialect, Expression};
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
 
 use crate::compiler::_helpers::sql_tests::cte_slices::SliceDialect;
 use crate::compiler::_helpers::sql_tests::cte_sql::{
     cte_definition_sql, leading_with_prefix_end, with_leading_ctes, with_unique_ctes,
 };
+use crate::compiler::_helpers::sql_tests::errors::RelationMarkerError;
 use crate::compiler::_helpers::sql_tests::expected_columns::expected_columns;
 use crate::compiler::_helpers::sql_tests::helper_scope::{
     ScopeGraph, helper_scope_ctes, merged_scoped_ctes,
@@ -22,6 +22,7 @@ use crate::compiler::_helpers::sql_tests::markers::{
     ProtectedRanges, marker_names, marker_names_in, replace_callable_markers,
     replace_dbt_ref_markers, replace_named_markers,
 };
+use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls;
 use crate::compiler::_helpers::sql_tests::rendering::{
     AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect,
     rendered_chain_steps,
@@ -49,7 +50,7 @@ const ASSERT_PREFIX: &str = "__assert__";
 const REF_FUNCTION: &str = "__ref";
 const SOURCE_FUNCTION: &str = "__source";
 const SEED_FUNCTION: &str = "__seed";
-const DBT_REF_FUNCTION: &str = "__dbt_ref";
+pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref";
 const UDF_FUNCTION: &str = "__udf";
 const TABLE_FUNCTION: &str = "__table_fn";
 
@@ -1304,13 +1305,13 @@ fn analysis_template(
                 return None;
             }
             let expression = statements.remove(0);
-            let value = match serde_json::to_value(&expression) {
-                Ok(value) => value,
-                Err(_) => return None,
+            let marker_calls = match relation_marker_calls(&expression) {
+                Ok(marker_calls) => marker_calls,
+                Err(RelationMarkerError) => return None,
             };
             Some(AnalysisTemplate {
                 existing_cte_names: top_level_cte_names(&expression),
-                marker_calls: relation_marker_calls(&value),
+                marker_calls,
             })
         })
     })
@@ -1484,73 +1485,6 @@ fn resolve_function_calls(
     Ok(result)
 }
 
-fn relation_marker_calls(value: &Value) -> Vec<(String, String)> {
-    fn marker(expression: &Value) -> Option<(String, String)> {
-        let object = expression.as_object()?;
-        if let Some(alias) = object.get("alias") {
-            return marker(alias.get("this")?);
-        }
-        let function = object.get("function")?.as_object()?;
-        let function_name = function.get("name")?.as_str()?.to_ascii_lowercase();
-        let args = function.get("args")?.as_array()?;
-        let arg_name = |arg: &Value| {
-            arg.get("column")?
-                .get("name")?
-                .get("name")?
-                .as_str()
-                .map(str::to_string)
-        };
-        let referenced_name = if function_name == DBT_REF_FUNCTION {
-            match args.as_slice() {
-                [first] => arg_name(first)?,
-                [first, second] => format!("{}__{}", arg_name(first)?, arg_name(second)?),
-                _ => return None,
-            }
-        } else {
-            let [first] = args.as_slice() else {
-                return None;
-            };
-            arg_name(first)?
-        };
-        Some((function_name, referenced_name))
-    }
-
-    fn walk(value: &Value, calls: &mut Vec<(String, String)>) {
-        match value {
-            Value::Object(object) => {
-                if let Some(expressions) = object
-                    .get("from")
-                    .and_then(|from| from.get("expressions"))
-                    .and_then(Value::as_array)
-                {
-                    calls.extend(expressions.iter().filter_map(marker));
-                }
-                if let Some(joins) = object.get("joins").and_then(Value::as_array) {
-                    calls.extend(
-                        joins
-                            .iter()
-                            .filter_map(|join| join.get("this"))
-                            .filter_map(marker),
-                    );
-                }
-                for child in object.values() {
-                    walk(child, calls);
-                }
-            }
-            Value::Array(values) => {
-                for child in values {
-                    walk(child, calls);
-                }
-            }
-            _ => {}
-        }
-    }
-
-    let mut calls: Vec<(String, String)> = Vec::new();
-    walk(value, &mut calls);
-    calls
-}
-
 fn top_level_cte_names(expression: &Expression) -> HashSet<String> {
     let Expression::Select(select) = expression else {
         return HashSet::new();