perf(rules): run custom rules across host processes

crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs

@@ -117,6 +117,31 @@ fn prepare_lint_sql(
     .map_err(value_error)
 }
 
+/// Prepare many bodies in one detached call; `false` marks a body to prepare again individually.
+#[pyfunction]
+fn prepare_lint_sql_batch(
+    py: Python<'_>,
+    requests: Vec<LintPreparationRequest>,
+) -> PyResult<Vec<(bool, Option<crate::sql_lint::types::PreparedSql>)>> {
+    py.compiler_detach(|| {
+        Ok(requests
+            .iter()
+            .map(|request| {
+                crate::bindings::_helpers::panics::catch_compiler_panic(|| {
+                    crate::sql_lint::main::preparation::prepare(
+                        &request.expanded,
+                        &request.before_expansion,
+                        &request.prior_sites,
+                        &request.dialect,
+                    )
+                })
+                .map_or((false, None), |prepared| (true, prepared))
+            })
+            .collect())
+    })
+    .map_err(value_error)
+}
+
 #[pyfunction]
 fn lint_backtick_identifiers(dialect: &str) -> PyResult<bool> {
     compiler_guard(|| {
@@ -431,6 +456,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?;
     module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?;
     module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?;
+    module.add_function(wrap_pyfunction!(prepare_lint_sql_batch, module)?)?;
     module.add(
         "NativeCompilerError",
         module

src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md

@@ -6,6 +6,14 @@
 
 Online: https://sqlbuild.com/docs/concepts/rules/custom-rules/testing-and-determinism/
 
+## Contents
+
+- Test Rules
+- Parallel evaluation
+- Helpers
+- Deterministic inputs
+- Cache granularity
+
 ## Test Rules
 
 Use the public harness to exercise discovery, compilation, and Rules evaluation:
@@ -39,6 +47,19 @@ the next. Compute findings from the subject and `ctx` alone.
 
 Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory.
 
+## Parallel evaluation
+
+On large projects, SQLBuild splits custom Rule evaluation across several host processes, at most
+one per available CPU and never more than eight. Each process evaluates a share of the subjects,
+and SQLBuild merges the findings back into the order a single process reports them.
+
+A Rule that changes module-level state while it runs, for example by adding each model to a set of
+already-seen names, could report different findings when its subjects are split. SQLBuild detects
+the state change and evaluates that Rule again in one process over all of its subjects, so its
+findings and cache entries match a single-process run. The result stays correct, but the Rule no
+longer runs in parallel and costs an extra host process. Keep each subject's findings independent
+of the others so Rules can run in parallel.
+
 ## Helpers
 
 Custom Rules can import repository-owned helpers under `rules/` and these deterministic standard

src/sqlbuild/_native.pyi

@@ -23,6 +23,9 @@ def lint_sql_batch_json(request_json: str) -> str: ...
 def prepare_lint_sql(
     request: LintPreparationRequest,
 ) -> tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None: ...
+def prepare_lint_sql_batch(
+    requests: list[LintPreparationRequest],
+) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ...
 def lint_backtick_identifiers(dialect: str) -> bool: ...
 def load_config_json(project_dir: str) -> str: ...
 def selected_codes_json(request_json: str) -> str: ...

src/sqlbuild/lint/_helpers/expansion.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import re
+from dataclasses import replace
 from pathlib import Path
 
 import sqlbuild._native as _native
@@ -23,15 +24,32 @@
 from sqlbuild.compiler.discovery.main.discover import discover_project_inputs
 from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs
 from sqlbuild.compiler.scopes.types import DeclarationKind
+from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role
+from sqlbuild.lint._helpers.native import external_identifiers_for_headers
+from sqlbuild.lint._helpers.native_sql import check_lint_not_stopped
 from sqlbuild.lint._helpers.sqlbuild_tokens import (
     neutralize_context_interpolation,
     neutralize_generic_audit_parameters,
     neutralize_interpolation,
     sentinel_spans,
 )
-from sqlbuild.lint.constants import RELATION_IDENTITY_TEMPLATE, TEMPLATE_INTERPOLATION_START
+from sqlbuild.lint.constants import (
+    HEADER_KIND_SCENARIO,
+    HEADER_KIND_TEST,
+    RELATION_IDENTITY_TEMPLATE,
+    TEMPLATE_INTERPOLATION_START,
+)
 from sqlbuild.lint.exceptions import ProjectCompileError
-from sqlbuild.lint.models import InterpolationSite, LintBody, LintFileRole
+from sqlbuild.lint.models import (
+    ExpandedLintFile,
+    HeaderSpan,
+    InterpolationSite,
+    LintBody,
+    LintFileRole,
+    PendingLintBody,
+    PreparedLintFiles,
+)
+from sqlbuild.lint.types import NativePreparedSql
 from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import (
     resolve_effective_adapter_name,
 )
@@ -146,6 +164,38 @@ def prepare_lint_body(
 ) -> LintBody:
     """Expand one authored body and neutralize whatever interpolation remains."""
 
+    pending: PendingLintBody = expand_lint_body(
+        role=role,
+        file_path=file_path,
+        contents=contents,
+        body_range=body_range,
+        context=context,
+        dialect=dialect,
+        external_identifiers=external_identifiers,
+        allows_ceremonial_select=allows_ceremonial_select,
+        allows_dynamic_output_star=allows_dynamic_output_star,
+        compiled_expansion=compiled_expansion,
+    )
+    return finish_lint_body(
+        pending=pending, prepared=_native.prepare_lint_sql(pending.request), dialect=dialect
+    )
+
+
+def expand_lint_body(
+    *,
+    role: LintFileRole,
+    file_path: Path,
+    contents: str,
+    body_range: tuple[int, int],
+    context: SqlExpansionContext,
+    dialect: str,
+    external_identifiers: tuple[str, ...] = (),
+    allows_ceremonial_select: bool = False,
+    allows_dynamic_output_star: bool = False,
+    compiled_expansion: CompiledSqlExpansion | None = None,
+) -> PendingLintBody:
+    """Expand one authored body like compile does, leaving its native preparation pending."""
+
     body_start: int
     body_end: int
     body_start, body_end = body_range
@@ -184,50 +234,84 @@ def prepare_lint_body(
         raise ProjectCompileError(
             f"{file_path} could not be expanded, so its SQL cannot be linted: {error}"
         ) from error
+    return PendingLintBody(
+        file_path=file_path,
+        body_start=body_start,
+        body_end=body_end,
+        expansion_input=expansion_input,
+        expanded=expanded,
+        expansion_passes=expansion_passes,
+        pre_expansion_sites=pre_expansion_sites,
+        external_identifiers=external_identifiers,
+        allows_ceremonial_select=allows_ceremonial_select,
+        allows_dynamic_output_star=allows_dynamic_output_star,
+        allows_empty_fixture_star=False,
+        request={
+            "expanded": expanded,
+            "before_expansion": expansion_input,
+            "prior_sites": [site.neutralized_start for site in pre_expansion_sites],
+            "dialect": dialect,
+        },
+    )
+
+
+def prepare_lint_sql_batch(
+    pending: tuple[PendingLintBody, ...],
+) -> tuple[NativePreparedSql | None, ...]:
+    """Prepare pending bodies in one GIL-free native call; failures re-raise per body, in order."""
+
+    if not pending:
+        return ()
+    results: list[tuple[bool, NativePreparedSql | None]] = _native.prepare_lint_sql_batch(
+        [body.request for body in pending]
+    )
+    return tuple(
+        prepared if succeeded else _native.prepare_lint_sql(body.request)
+        for body, (succeeded, prepared) in zip(pending, results, strict=True)
+    )
+
+
+def finish_lint_body(
+    *, pending: PendingLintBody, prepared: NativePreparedSql | None, dialect: str
+) -> LintBody:
+    """Build the lint body from one body's native preparation, or the Python fallback."""
+
+    expanded: str = pending.expanded
     neutralized: str
     sites: tuple[InterpolationSite, ...]
-    native_prepared: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = (
-        _native.prepare_lint_sql(
-            {
-                "expanded": expanded,
-                "before_expansion": expansion_input,
-                "prior_sites": [site.neutralized_start for site in pre_expansion_sites],
-                "dialect": dialect,
-            }
-        )
-    )
-    if native_prepared is None:
+    if prepared is None:
         neutralized, sites = neutralize_interpolation(body=expanded, dialect=dialect)
         externally_referenced_ctes: tuple[str, ...] = _externally_referenced_ctes(
             expanded=expanded,
             interpolation_sites=sites,
-            pre_expansion_body=expansion_input,
-            pre_expansion_sites=pre_expansion_sites,
+            pre_expansion_body=pending.expansion_input,
+            pre_expansion_sites=pending.pre_expansion_sites,
         )
     else:
-        neutralized = native_prepared[0]
-        sites = tuple(InterpolationSite(*site) for site in native_prepared[1])
-        externally_referenced_ctes = tuple(native_prepared[2])
+        neutralized = prepared[0]
+        sites = tuple(InterpolationSite(*site) for site in prepared[1])
+        externally_referenced_ctes = tuple(prepared[2])
     dependency_identifiers: tuple[str, ...] = tuple(
         site.sentinel
         for site in sites
         if _DEPENDENCY_INTRINSIC_PATTERN.match(site.original_text) is not None
     )
     return LintBody(
-        file_path=file_path,
-        body_start=body_start,
-        body_end=body_end,
+        file_path=pending.file_path,
+        body_start=pending.body_start,
+        body_end=pending.body_end,
         lint_text=neutralized,
         passes=(
-            sentinel_spans(sites=pre_expansion_sites),
-            *expansion_passes,
+            sentinel_spans(sites=pending.pre_expansion_sites),
+            *pending.expansion_passes,
             sentinel_spans(sites=sites),
         ),
-        external_identifiers=external_identifiers,
+        external_identifiers=pending.external_identifiers,
         dependency_identifiers=dependency_identifiers,
         externally_referenced_ctes=externally_referenced_ctes,
-        allows_ceremonial_select=allows_ceremonial_select,
-        allows_dynamic_output_star=allows_dynamic_output_star,
+        allows_ceremonial_select=pending.allows_ceremonial_select,
+        allows_dynamic_output_star=pending.allows_dynamic_output_star,
+        allows_empty_fixture_star=pending.allows_empty_fixture_star,
         dependency_relations=tuple(
             (
                 site.sentinel,
@@ -249,3 +333,96 @@ def _is_sql_hook_path(*, role: LintFileRole) -> bool:
     return role.in_hook_directory or (
         role.in_project and role.declaration_kind is DeclarationKind.SQL_HOOK
     )
+
+
+def expand_file_bodies(
+    *,
+    file_path: Path,
+    contents: str,
+    headers: tuple[HeaderSpan, ...],
+    context: SqlExpansionContext,
+    dialect: str,
+    project_dir: Path,
+    relative_path: Path,
+    dynamic_output_paths: frozenset[Path],
+    compiled_expansions: dict[Path, CompiledSqlExpansion] | None,
+) -> ExpandedLintFile:
+    """Expand one file's bodies up to the first body that cannot be expanded."""
+
+    check_lint_not_stopped()
+    bodies: list[PendingLintBody] = []
+    role: LintFileRole = lint_file_role(
+        file_path=file_path, project_dir=project_dir, relative_path=relative_path
+    )
+    allows_dynamic_output_star: bool = (
+        bool(dynamic_output_paths) and file_path.resolve() in dynamic_output_paths
+    )
+    compiled_expansion: CompiledSqlExpansion | None = (compiled_expansions or {}).get(file_path)
+    external_identifiers: tuple[str, ...] = external_identifiers_for_headers(
+        contents=contents, headers=headers
+    )
+    allows_ceremonial_select: bool = any(
+        header.kind in {HEADER_KIND_TEST, HEADER_KIND_SCENARIO} for header in headers
+    )
+    allows_empty_fixture_star: bool = any(header.kind == HEADER_KIND_TEST for header in headers)
+    body_start: int
+    body_end: int
+    try:
+        for body_start, body_end in lint_body_ranges(
+            contents=contents,
+            headers=headers,
+            file_path=file_path,
+            project_dir=project_dir,
+            role=role,
+        ):
+            bodies.append(
+                replace(
+                    expand_lint_body(
+                        role=role,
+                        file_path=file_path,
+                        contents=contents,
+                        body_range=(body_start, body_end),
+                        context=context,
+                        dialect=dialect,
+                        external_identifiers=external_identifiers,
+                        allows_ceremonial_select=allows_ceremonial_select,
+                        allows_dynamic_output_star=allows_dynamic_output_star,
+                        compiled_expansion=compiled_expansion,
+                    ),
+                    allows_empty_fixture_star=allows_empty_fixture_star,
+                )
+            )
+    except ProjectCompileError as error:
+        return ExpandedLintFile(file_path=file_path, bodies=tuple(bodies), failure=error)
+    return ExpandedLintFile(file_path=file_path, bodies=tuple(bodies), failure=None)
+
+
+def prepare_expanded_files(
+    *, expanded_files: tuple[ExpandedLintFile, ...], dialect: str, skip_unexpandable: bool
+) -> PreparedLintFiles:
+    """Prepare every expanded body natively at once, then report failures in file order."""
+
+    check_lint_not_stopped()
+    pending: list[PendingLintBody] = []
+    for expanded in expanded_files:
+        pending.extend(expanded.bodies)
+    prepared: tuple[NativePreparedSql | None, ...] = prepare_lint_sql_batch(tuple(pending))
+    bodies: list[LintBody] = []
+    unexpandable: dict[Path, str] = {}
+    position: int = 0
+    for expanded in expanded_files:
+        file_bodies: list[LintBody] = []
+        for body in expanded.bodies:
+            file_bodies.append(
+                finish_lint_body(pending=body, prepared=prepared[position], dialect=dialect)
+            )
+            position += 1
+        if expanded.failure is None:
+            bodies.extend(file_bodies)
+        elif skip_unexpandable:
+            unexpandable[expanded.file_path] = str(expanded.failure).removeprefix(
+                f"{expanded.file_path} "
+            )
+        else:
+            raise expanded.failure
+    return PreparedLintFiles(bodies=tuple(bodies), unexpandable=unexpandable)