perf(compile): lower rules-phase peak memory

src/sqlbuild/lint/_helpers/project_files.py

@@ -4,6 +4,7 @@
 
 from pathlib import Path
 
+from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
 from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES
 
 
@@ -12,15 +13,19 @@ def collect_project_files(
     project_dir: Path,
     selected_paths: frozenset[Path] | None = None,
     source_files: dict[Path, str] | None = None,
+    discovered_inputs: DiscoveredProjectInputs | None = None,
 ) -> dict[Path, str]:
-    """Return authored SQL files keyed by path, reusing contents a caller already read."""
+    """Return authored SQL files keyed by path, sharing identical text discovery already holds."""
 
     if source_files is not None:
         return {
             file_path: contents
             for file_path, contents in source_files.items()
             if selected_paths is None or file_path.resolve() in selected_paths
         }
+    discovered: dict[Path, str] = (
+        {} if discovered_inputs is None else discovered_sql_contents(discovered_inputs)
+    )
     files: dict[Path, str] = {}
     directory_name: str
     for directory_name in LINT_DIRECTORY_NAMES:
@@ -32,10 +37,31 @@ def collect_project_files(
             if selected_paths is not None and file_path.resolve() not in selected_paths:
                 continue
             with file_path.open("r", encoding="utf-8", newline="") as handle:
-                files[file_path] = handle.read()
+                contents: str = handle.read()
+            known: str | None = discovered.get(file_path)
+            files[file_path] = known if known is not None and known == contents else contents
     return files
 
 
+def discovered_sql_contents(discovered_inputs: DiscoveredProjectInputs) -> dict[Path, str]:
+    """Return the text discovery read for every SQL file it retains, keyed by path."""
+
+    contents: dict[Path, str] = {}
+    for group in (
+        discovered_inputs.model_files,
+        discovered_inputs.enum_files,
+        discovered_inputs.constant_files,
+        discovered_inputs.model_schema_files,
+        discovered_inputs.sql_function_files,
+        discovered_inputs.sql_hook_files,
+        discovered_inputs.test_files,
+        discovered_inputs.scenario_files,
+        discovered_inputs.audit_files,
+    ):
+        contents.update({item.file_path: item.contents for item in group})
+    return contents
+
+
 def sort_violations(violations: list) -> tuple:
     """Return violations sorted by file path then position."""
 

src/sqlbuild/lint/main/collect_project_files.py

@@ -2,12 +2,20 @@
 
 from pathlib import Path
 
+from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
 from sqlbuild.lint._helpers.project_files import collect_project_files as _collect_project_files
 
 
 def collect_project_files(
-    *, project_dir: Path, selected_paths: frozenset[Path] | None
+    *,
+    project_dir: Path,
+    selected_paths: frozenset[Path] | None,
+    discovered_inputs: DiscoveredProjectInputs | None = None,
 ) -> dict[Path, str]:
-    """Collect supported SQL-analysis inputs for a project or selected path set."""
+    """Collect supported SQL-analysis inputs, sharing text discovery already holds."""
 
-    return _collect_project_files(project_dir=project_dir, selected_paths=selected_paths)
+    return _collect_project_files(
+        project_dir=project_dir,
+        selected_paths=selected_paths,
+        discovered_inputs=discovered_inputs,
+    )

src/sqlbuild/lint/main/run_lint.py

@@ -50,7 +50,10 @@ def run_lint(
     """Lint all DSL files without modifying anything; optionally skip unexpandable SQL."""
 
     files: dict[Path, str] = collect_project_files(
-        project_dir=project_dir, selected_paths=selected_paths, source_files=source_files
+        project_dir=project_dir,
+        selected_paths=selected_paths,
+        source_files=source_files,
+        discovered_inputs=discovered_inputs,
     )
     violations: list[LintViolation] = []
     context: SqlExpansionContext | None = expansion_context or _expansion_context(

src/sqlbuild/rule_engine/_helpers/run/memory.py

@@ -0,0 +1,30 @@
+"""Return memory the C allocator keeps after earlier compile phases freed it."""
+
+from __future__ import annotations
+
+import ctypes
+import sys
+from collections.abc import Callable
+from functools import cache
+from typing import Any
+
+
+def release_freed_memory() -> None:
+    """Hand freed glibc heap pages back to the OS; a no-op on other platforms and allocators."""
+
+    trim: Callable[[int], int] | None = _malloc_trim()
+    if trim is not None:
+        _ = trim(0)
+
+
+@cache
+def _malloc_trim() -> Callable[[int], int] | None:
+    if not sys.platform.startswith("linux"):
+        return None
+    try:
+        trim: Any = ctypes.CDLL(None).malloc_trim
+    except (OSError, AttributeError):
+        return None
+    trim.argtypes = (ctypes.c_size_t,)
+    trim.restype = ctypes.c_int
+    return trim

src/sqlbuild/rule_engine/_helpers/run/rules.py

@@ -60,6 +60,7 @@
 )
 from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings
 from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints
+from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory
 from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES
 from sqlbuild.rule_engine.exceptions import RulesError
 from sqlbuild.rule_engine.main.load_config import load_rules_config
@@ -106,6 +107,7 @@ def evaluate_rules(
     no_sql_analysis: bool = False,
 ) -> RulesRunResult:
     """Evaluate independent rule phases concurrently, then finalize their combined findings."""
+    release_freed_memory()
     effective_config: RulesConfig = resolve_rule_ignore_selectors(
         config=config, project=graph.project
     )
@@ -185,6 +187,7 @@ def evaluate_rules(
             dialect=_lint_identity(lint_config),
             project_dir=resolved_project_dir,
             collect_files=prepared_sql is None and model_paths is None,
+            discovered_inputs=discovered_inputs,
         )
         sql_started: float = time.monotonic()
         sql_result: _SqlRulesEvaluation = _run_sql_rules(
@@ -239,6 +242,7 @@ def _prepare_sql_rule_inputs(
     dialect: str,
     project_dir: Path,
     collect_files: bool,
+    discovered_inputs: DiscoveredProjectInputs,
 ) -> _SqlRuleInputs:
     """Hash SQL and read the project files on the caller while built-in rules evaluate natively."""
 
@@ -248,7 +252,9 @@ def _prepare_sql_rule_inputs(
     return _SqlRuleInputs(
         identities=identities,
         project_files=(
-            collect_project_files(project_dir=project_dir, selected_paths=None)
+            collect_project_files(
+                project_dir=project_dir, selected_paths=None, discovered_inputs=discovered_inputs
+            )
             if identities and collect_files
             else None
         ),
@@ -529,7 +535,11 @@ def _run_sql_rules(
             project_files = (
                 prepared_inputs.project_files
                 if prepared_inputs is not None and prepared_inputs.project_files is not None
-                else collect_project_files(project_dir=project_dir, selected_paths=None)
+                else collect_project_files(
+                    project_dir=project_dir,
+                    selected_paths=None,
+                    discovered_inputs=discovered_inputs,
+                )
             )
         file_path: Path
         contents: str

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py

@@ -62,7 +62,7 @@
             "dense_models_10000_all_rules",
             10000,
             145.0,
-            7 * _GIB // 2,
+            13 * _GIB // 4,
             "913a08c2277304efacf75e293728767a3fa416f6b32f5488dca6ab5baea80668",
         ),
     ),

tests/integration/src/sqlbuild/cli/commands/main/test_rules.py

@@ -909,10 +909,17 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_
     original_collect: Callable[..., dict[Path, str]] = rules_module.collect_project_files
 
     def counting_collect(
-        *, project_dir: Path, selected_paths: frozenset[Path] | None
+        *,
+        project_dir: Path,
+        selected_paths: frozenset[Path] | None,
+        discovered_inputs: DiscoveredProjectInputs | None = None,
     ) -> dict[Path, str]:
         collections.append(selected_paths)
-        return original_collect(project_dir=project_dir, selected_paths=selected_paths)
+        return original_collect(
+            project_dir=project_dir,
+            selected_paths=selected_paths,
+            discovered_inputs=discovered_inputs,
+        )
 
     monkeypatch.setattr(rules_module, "collect_project_files", counting_collect)
 

tests/unit/src/sqlbuild/lint/_helpers/_test_types.py

@@ -234,3 +234,12 @@ class BatchedPreparationFailureTestCase:
     unexpandable_file: str
     skip_unexpandable: bool
     expected_error: type[Exception] | None
+
+
+@dataclass(frozen=True)
+class CollectProjectFilesTestCase:
+    """Test case for sharing discovered SQL text when collecting lint inputs."""
+
+    description: str
+    model_bytes: bytes
+    expected_shared: bool

tests/unit/src/sqlbuild/lint/_helpers/helpers.py

@@ -91,3 +91,24 @@ def refuse_preparation(request: NativeLintPreparationRequest) -> NativePreparedS
     """Fail the per-body native preparation of any body."""
 
     raise ValueError(f"cannot prepare {request['dialect']} body")
+
+
+def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path:
+    """Write a DuckDB project with one model and one SQL test; return the model path."""
+
+    _ = (project_dir / "sqlbuild_project.toml").write_text(
+        'name = "orders"\nadapter = "duckdb"\n', encoding="utf-8"
+    )
+    model_path: Path = project_dir / "models" / "orders.sql"
+    model_path.parent.mkdir(parents=True)
+    _ = model_path.write_bytes(model_bytes)
+    test_path: Path = project_dir / "tests" / "test_orders.sql"
+    test_path.parent.mkdir(parents=True)
+    _ = test_path.write_text(
+        "TEST (name orders_keep_ids);\n"
+        "WITH orders AS (SELECT 1 AS order_id),\n"
+        "expected AS (SELECT 1 AS order_id)\n"
+        "SELECT * FROM orders\n",
+        encoding="utf-8",
+    )
+    return model_path

tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py

@@ -0,0 +1,58 @@
+"""Unit tests for collecting authored SQL files for lint runs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from sqlbuild.compiler.discovery.main.discover import discover_project_inputs
+from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
+from sqlbuild.lint._helpers.project_files import collect_project_files
+from sqlbuild.lint.main.run_lint import run_lint
+from sqlbuild.lint.models import LintConfig, LintRunResult
+from tests.unit.src.sqlbuild.lint._helpers._test_types import CollectProjectFilesTestCase
+from tests.unit.src.sqlbuild.lint._helpers.helpers import write_orders_lint_project
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        CollectProjectFilesTestCase(
+            description="identical text shares the discovered string",
+            model_bytes=b"MODEL (materialized table);\nSELECT 1 AS order_id\n",
+            expected_shared=True,
+        ),
+        CollectProjectFilesTestCase(
+            description="carriage returns keep the exact authored text",
+            model_bytes=b"MODEL (materialized table);\r\nSELECT 1 AS order_id\r\n",
+            expected_shared=False,
+        ),
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_discovered_project_when_collecting_files_then_returns_exact_authored_text(
+    test_case: CollectProjectFilesTestCase, tmp_path: Path
+) -> None:
+    model_path: Path = write_orders_lint_project(
+        project_dir=tmp_path, model_bytes=test_case.model_bytes
+    )
+    inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path)
+    discovered: dict[Path, str] = {item.file_path: item.contents for item in inputs.model_files}
+
+    files: dict[Path, str] = collect_project_files(project_dir=tmp_path, discovered_inputs=inputs)
+    shared: LintRunResult = run_lint(
+        project_dir=tmp_path, config=LintConfig(), discovered_inputs=inputs
+    )
+    fresh: LintRunResult = run_lint(project_dir=tmp_path, config=LintConfig())
+
+    assert files == collect_project_files(project_dir=tmp_path)
+    assert files[model_path] == test_case.model_bytes.decode("utf-8")
+    assert (files[model_path] is discovered[model_path]) is test_case.expected_shared
+    assert shared.source_texts == fresh.source_texts
+    assert shared.violations == fresh.violations
+    assert shared.files_checked == fresh.files_checked == 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"]))