perf(compile): faster cold compiles

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs

@@ -19,15 +19,15 @@ pub(crate) fn replace_callable_markers<F>(
 where
     F: FnMut(&str, &str) -> Option<(String, bool)>,
 {
-    let protected = protected_ranges(syntax, sql);
+    let mut protected = ProtectedRanges::new(syntax, sql);
     let mut output = String::with_capacity(sql.len());
     let mut reached: HashSet<String> = HashSet::new();
     let mut cursor = 0;
     for captures in pattern.captures_iter(sql) {
         let Some(full) = captures.get(0) else {
             continue;
         };
-        if full.start() < cursor || in_protected_range(full.start(), &protected) {
+        if full.start() < cursor || protected.contains(full.start()) {
             continue;
         }
         let suffix_start = skip_whitespace(sql, full.end());
@@ -107,14 +107,14 @@ fn replace_marker_captures<F>(
 where
     F: FnMut(&Captures<'_>) -> Option<String>,
 {
-    let protected = protected_ranges(syntax, sql);
+    let mut protected = ProtectedRanges::new(syntax, sql);
     let mut output = String::with_capacity(sql.len());
     let mut cursor = 0;
     for captures in pattern.captures_iter(sql) {
         let Some(full) = captures.get(0) else {
             continue;
         };
-        if in_protected_range(full.start(), &protected) {
+        if protected.contains(full.start()) {
             continue;
         }
         let Some(value) = replacement(&captures) else {
@@ -129,13 +129,18 @@ where
 }
 
 pub(crate) fn marker_names(pattern: &Regex, syntax: &LexicalSyntax, sql: &str) -> Vec<String> {
-    let protected = protected_ranges(syntax, sql);
+    marker_names_in(pattern, &mut ProtectedRanges::new(syntax, sql))
+}
+
+/// Marker names outside the comments and quoted text of the SQL that `protected` scans.
+pub(crate) fn marker_names_in(pattern: &Regex, protected: &mut ProtectedRanges<'_>) -> Vec<String> {
+    let sql = protected.sql;
     let mut names: Vec<String> = Vec::new();
     for captures in pattern.captures_iter(sql) {
         let Some(full) = captures.get(0) else {
             continue;
         };
-        if in_protected_range(full.start(), &protected) {
+        if protected.contains(full.start()) {
             continue;
         }
         if let Some(name) = captures.get(1) {
@@ -149,8 +154,32 @@ pub(crate) fn protected_ranges(syntax: &LexicalSyntax, sql: &str) -> Vec<(usize,
     dialect_non_code_ranges(sql, syntax)
 }
 
+/// Whether `index` falls inside one of the sorted, non-overlapping `ranges`.
 pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool {
-    ranges
-        .iter()
-        .any(|(start, end)| index >= *start && index < *end)
+    let following = ranges.partition_point(|(start, _)| *start <= index);
+    following > 0 && index < ranges[following - 1].1
+}
+
+/// Comment and quoted-text ranges of one SQL text, scanned only when a marker needs them.
+pub(crate) struct ProtectedRanges<'a> {
+    sql: &'a str,
+    syntax: &'a LexicalSyntax,
+    ranges: Option<Vec<(usize, usize)>>,
+}
+
+impl<'a> ProtectedRanges<'a> {
+    pub(crate) fn new(syntax: &'a LexicalSyntax, sql: &'a str) -> Self {
+        Self {
+            sql,
+            syntax,
+            ranges: None,
+        }
+    }
+
+    pub(crate) fn contains(&mut self, index: usize) -> bool {
+        let ranges = self
+            .ranges
+            .get_or_insert_with(|| protected_ranges(self.syntax, self.sql));
+        in_protected_range(index, ranges)
+    }
 }

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs

@@ -19,7 +19,7 @@ use crate::compiler::_helpers::sql_tests::helper_scope::{
     ScopeGraph, helper_scope_ctes, merged_scoped_ctes,
 };
 use crate::compiler::_helpers::sql_tests::markers::{
-    in_protected_range, marker_names, protected_ranges, replace_callable_markers,
+    ProtectedRanges, marker_names, marker_names_in, replace_callable_markers,
     replace_dbt_ref_markers, replace_named_markers,
 };
 use crate::compiler::_helpers::sql_tests::rendering::{
@@ -1620,7 +1620,8 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec
             });
         }
     };
-    for name in marker_names(&patterns.reference, &patterns.lexical, sql) {
+    let mut protected = ProtectedRanges::new(&patterns.lexical, sql);
+    for name in marker_names_in(&patterns.reference, &mut protected) {
         let message = format!(
             "test '{test_name}': model '{model_name}' references __ref('{name}') which has no mock and is not in the expected chain"
         );
@@ -1630,19 +1631,18 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec
         (&patterns.source, SOURCE_FUNCTION),
         (&patterns.seed, SEED_FUNCTION),
     ] {
-        for name in marker_names(pattern, &patterns.lexical, sql) {
+        for name in marker_names_in(pattern, &mut protected) {
             let message = format!(
                 "test '{test_name}': model '{model_name}' references {function_name}('{name}') which has no mock"
             );
             warn(function_name, name, message);
         }
     }
-    let protected = protected_ranges(&patterns.lexical, sql);
     for captures in patterns.dbt_reference.captures_iter(sql) {
         let Some(full) = captures.get(0) else {
             continue;
         };
-        if in_protected_range(full.start(), &protected) {
+        if protected.contains(full.start()) {
             continue;
         }
         let Some(first) = captures.get(1).map(|value| value.as_str()) else {

crates/sqlbuild-rules-native/src/compiler/tests.rs

@@ -14,6 +14,8 @@ mod sql_test_cte_slices;
 mod sql_test_expected_columns;
 #[path = "tests/test_sql_test_extraction.rs"]
 mod sql_test_extraction;
+#[path = "tests/test_sql_test_markers.rs"]
+mod sql_test_markers;
 #[path = "tests/test_sql_test_planning.rs"]
 mod sql_test_planning;
 #[path = "tests/test_sql_test_rendering.rs"]

crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_markers.rs

@@ -0,0 +1,72 @@
+use regex::Regex;
+
+use crate::compiler::_helpers::sql_tests::markers::{
+    in_protected_range, marker_names, replace_named_markers,
+};
+use crate::compiler::tests::test_types::{MarkerNamesTestCase, ProtectedRangeTestCase};
+use crate::sql_scan::models::LexicalSyntax;
+
+#[test]
+fn given_markers_around_comments_and_strings_when_scanning_then_only_code_markers_are_used() {
+    let test_cases = [
+        MarkerNamesTestCase {
+            description: "no markers leaves the SQL unchanged",
+            sql: "SELECT 'a' AS label -- note\nFROM orders",
+            expected_names: &[],
+            expected_replaced_sql: "SELECT 'a' AS label -- note\nFROM orders",
+        },
+        MarkerNamesTestCase {
+            description: "markers between many comments and strings",
+            sql: "SELECT '__ref(\"quoted\")' AS a, /* __ref(\"hidden\") */ b\nFROM __ref(\"orders\") -- __ref(\"commented\")\nJOIN __ref(\"customers\") ON 'x' = 'y'\nWHERE c = '__ref(\"text\")' AND d IN (SELECT id FROM __ref(\"products\"))",
+            expected_names: &["orders", "customers", "products"],
+            expected_replaced_sql: "SELECT '__ref(\"quoted\")' AS a, /* __ref(\"hidden\") */ b\nFROM mock_orders -- __ref(\"commented\")\nJOIN mock_customers ON 'x' = 'y'\nWHERE c = '__ref(\"text\")' AND d IN (SELECT id FROM mock_products)",
+        },
+    ];
+    let pattern = Regex::new(r#"(?i)__ref\("([^"]+)"\)"#).expect("valid marker pattern");
+    let syntax = LexicalSyntax {
+        line_comment_prefixes: vec!["--".to_string()],
+        ..LexicalSyntax::default()
+    };
+    for test_case in test_cases {
+        assert_eq!(
+            marker_names(&pattern, &syntax, test_case.sql),
+            test_case.expected_names,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            replace_named_markers(test_case.sql, &pattern, &syntax, |name| Some(format!(
+                "mock_{name}"
+            ))),
+            test_case.expected_replaced_sql,
+            "{}",
+            test_case.description
+        );
+    }
+}
+
+#[test]
+fn given_sorted_ranges_when_checking_offsets_then_only_offsets_inside_a_range_are_protected() {
+    let test_cases = [
+        ProtectedRangeTestCase {
+            description: "no ranges protect nothing",
+            ranges: &[],
+            expected_protected: &[],
+        },
+        ProtectedRangeTestCase {
+            description: "starts are inclusive and ends exclusive across several ranges",
+            ranges: &[(2, 4), (6, 7), (10, 15)],
+            expected_protected: &[2, 3, 6, 10, 11, 12, 13, 14],
+        },
+    ];
+    for test_case in test_cases {
+        let protected: Vec<usize> = (0..17)
+            .filter(|index| in_protected_range(*index, test_case.ranges))
+            .collect();
+        assert_eq!(
+            protected, test_case.expected_protected,
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs

@@ -60,3 +60,16 @@ pub(crate) struct CteRenameTestCase {
     pub(crate) renamed_index: usize,
     pub(crate) expected_sql: Option<&'static str>,
 }
+
+pub(crate) struct MarkerNamesTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) sql: &'static str,
+    pub(crate) expected_names: &'static [&'static str],
+    pub(crate) expected_replaced_sql: &'static str,
+}
+
+pub(crate) struct ProtectedRangeTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) ranges: &'static [(usize, usize)],
+    pub(crate) expected_protected: &'static [usize],
+}

src/sqlbuild/cli/commands/_helpers/compile/collection.py

@@ -0,0 +1,20 @@
+"""Pause cyclic garbage collection while one compile builds its long-lived project state."""
+
+from __future__ import annotations
+
+import gc
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+
+@contextmanager
+def paused_cyclic_collection() -> Iterator[None]:
+    """Skip full-heap cycle scans of long-lived compile state, then restore the prior state."""
+
+    was_enabled: bool = gc.isenabled()
+    gc.disable()
+    try:
+        yield
+    finally:
+        if was_enabled:
+            gc.enable()

src/sqlbuild/cli/commands/_helpers/compile/pipeline.py

@@ -206,7 +206,9 @@ def _analyze_compile_project(
             and rules_config.select
             and not any(item.is_error for item in core_diagnostics)
         ):
-            prepared_artifacts.start(project=graph.project, adapter=adapter)
+            prepared_artifacts.start(
+                project=graph.project, adapter=adapter, target_dir=project_dir / "target"
+            )
         _ = start_compile_phase(status=status, message="Evaluating built-in and custom rules...")
         rules_start: float = time.monotonic()
         rules_result = run_rules(