From 87aca22cabcff0d520155790f285e5a0e94d86db Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:23:42 +0100 Subject: [PATCH 1/6] perf(lint): classify lint paths once and scan headers without per-character loops Compute each file's project-relative role once instead of repeating pathlib containment checks per body, skip resolving paths when no dynamic-output proofs exist, and jump between header delimiters with a compiled pattern. Header spans, body ranges and findings are unchanged. --- src/sqlbuild/lint/_helpers/expansion.py | 32 ++++------ src/sqlbuild/lint/_helpers/headers.py | 60 +++++++++++++------ src/sqlbuild/lint/main/run_lint.py | 29 +++++++-- src/sqlbuild/lint/models.py | 10 ++++ .../src/sqlbuild/lint/test_expanded_lint.py | 3 +- 5 files changed, 88 insertions(+), 46 deletions(-) diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index ce8e023be4..9a74de19e5 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -8,10 +8,7 @@ import sqlbuild._native as _native from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter -from sqlbuild.compiler.compile.constants import ( - HOOK_DIRECTORY_NAME, - MACRO_TOKEN, -) +from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans from sqlbuild.compiler.compile.main.sql_expansion_context import build_sql_expansion_context @@ -24,9 +21,6 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.main.named_declaration_role_kind import ( - named_declaration_role_kind, -) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.sqlbuild_tokens import ( @@ -37,7 +31,7 @@ ) from sqlbuild.lint.constants import TEMPLATE_INTERPOLATION_START from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.models import InterpolationSite, LintBody +from sqlbuild.lint.models import InterpolationSite, LintBody, LintFileRole from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -134,7 +128,7 @@ def _resolve_value_renderer( def prepare_lint_body( *, - project_dir: Path, + role: LintFileRole, file_path: Path, contents: str, body_range: tuple[int, int], @@ -153,11 +147,11 @@ def prepare_lint_body( authored_body: str = contents[body_start:body_end] pre_expansion_sites: tuple[InterpolationSite, ...] = () expansion_input: str = authored_body - if _is_generic_audit_path(file_path=file_path, project_dir=project_dir): + if _is_generic_audit_path(role=role): expansion_input, pre_expansion_sites = neutralize_generic_audit_parameters( body=authored_body ) - elif _is_sql_hook_path(file_path=file_path, project_dir=project_dir): + elif _is_sql_hook_path(role=role): expansion_input, pre_expansion_sites = neutralize_context_interpolation(body=authored_body) expanded: str expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] @@ -232,19 +226,13 @@ def prepare_lint_body( ) -def _is_generic_audit_path(*, file_path: Path, project_dir: Path) -> bool: +def _is_generic_audit_path(*, role: LintFileRole) -> bool: """Return whether a lint input is an authored generic-audit definition.""" - return ( - file_path.is_relative_to(project_dir) - and named_declaration_role_kind(relative_path=file_path.relative_to(project_dir)) - is DeclarationKind.AUDIT - ) + return role.in_project and role.declaration_kind is DeclarationKind.AUDIT -def _is_sql_hook_path(*, file_path: Path, project_dir: Path) -> bool: - return file_path.is_relative_to(project_dir / HOOK_DIRECTORY_NAME) or ( - file_path.is_relative_to(project_dir) - and named_declaration_role_kind(relative_path=file_path.relative_to(project_dir)) - is DeclarationKind.SQL_HOOK +def _is_sql_hook_path(*, role: LintFileRole) -> bool: + return role.in_hook_directory or ( + role.in_project and role.declaration_kind is DeclarationKind.SQL_HOOK ) diff --git a/src/sqlbuild/lint/_helpers/headers.py b/src/sqlbuild/lint/_helpers/headers.py index 84131fe69c..9dc2ccb033 100644 --- a/src/sqlbuild/lint/_helpers/headers.py +++ b/src/sqlbuild/lint/_helpers/headers.py @@ -4,20 +4,25 @@ import re from functools import cache -from pathlib import Path +from pathlib import Path, PurePath +from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME from sqlbuild.compiler.discovery.main.named_declaration_role_kind import ( named_declaration_role_kind, ) from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint.constants import DSL_HEADER_KINDS -from sqlbuild.lint.models import HeaderSpan +from sqlbuild.lint.models import HeaderSpan, LintFileRole _QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) _ESCAPE_CHARACTER: str = "\\" _OPEN_PAREN: str = "(" _CLOSE_PAREN: str = ")" _STATEMENT_TERMINATOR: str = ";" +_UNQUOTED_HEADER_TOKEN: re.Pattern[str] = re.compile(r"""[()'"]""") +_QUOTED_HEADER_TOKENS: dict[str, re.Pattern[str]] = { + quote: re.compile(rf"[\\{quote}]") for quote in _QUOTE_CHARACTERS +} def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan, ...]: @@ -26,6 +31,22 @@ def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan return _scan_delimited_regions(contents=contents, kinds=DSL_HEADER_KINDS, first_only=first_only) +def lint_file_role( + *, file_path: Path, project_dir: Path, relative_path: PurePath | None = None +) -> LintFileRole: + """Return the path facts lint needs, reusing a caller's project-relative path.""" + + if relative_path is None: + if not file_path.is_relative_to(project_dir): + return LintFileRole() + relative_path = file_path.relative_to(project_dir) + return LintFileRole( + declaration_kind=named_declaration_role_kind(relative_path=relative_path), + in_project=True, + in_hook_directory=relative_path.is_relative_to(HOOK_DIRECTORY_NAME), + ) + + def measurement_body_ranges(*, contents: str) -> tuple[tuple[int, int], ...]: """Return authored query ranges inside MEASURE and EVIDENCE blocks.""" @@ -50,12 +71,17 @@ def lint_body_ranges( headers: tuple[HeaderSpan, ...], file_path: Path, project_dir: Path, + role: LintFileRole | None = None, ) -> tuple[tuple[int, int], ...]: """Return lintable authored bodies using resource-specific DSL boundaries.""" - if file_path.is_relative_to(project_dir) and named_declaration_role_kind( - relative_path=file_path.relative_to(project_dir) - ) in {DeclarationKind.AUDIT, DeclarationKind.SINGULAR_AUDIT}: + file_role: LintFileRole = ( + role if role is not None else lint_file_role(file_path=file_path, project_dir=project_dir) + ) + if file_role.in_project and file_role.declaration_kind in { + DeclarationKind.AUDIT, + DeclarationKind.SINGULAR_AUDIT, + }: measurement_ranges: tuple[tuple[int, int], ...] = measurement_body_ranges(contents=contents) if measurement_ranges: return measurement_ranges @@ -132,24 +158,25 @@ def _match_header_span( return None depth: int = 0 in_quote: str | None = None - length: int = len(contents) - while index < length: + match: re.Match[str] | None + while True: + match = ( + _UNQUOTED_HEADER_TOKEN if in_quote is None else _QUOTED_HEADER_TOKENS[in_quote] + ).search(contents, index) + if match is None: + return None + index = match.start() character: str = contents[index] if in_quote is not None: if character == _ESCAPE_CHARACTER: index += 2 continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _QUOTE_CHARACTERS: + in_quote = None + elif character in _QUOTE_CHARACTERS: in_quote = character - index += 1 - continue - if character == _OPEN_PAREN: + elif character == _OPEN_PAREN: depth += 1 - elif character == _CLOSE_PAREN: + else: depth -= 1 if depth == 0: terminator_index: int = _skip_whitespace(contents=contents, index=index + 1) @@ -161,7 +188,6 @@ def _match_header_span( span_end = terminator_index + 1 return HeaderSpan(kind=kind, start=keyword_start, end=span_end) index += 1 - return None def _skip_whitespace(*, contents: str, index: int) -> int: diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index a79b2d25b2..606e1395a8 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -15,13 +15,20 @@ LOCAL_DECLARATION_DIRECTORIES, ) from sqlbuild.lint._helpers.expansion import build_lint_expansion_context, prepare_lint_body -from sqlbuild.lint._helpers.headers import lint_body_ranges, scan_headers +from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role, scan_headers from sqlbuild.lint._helpers.native import external_identifiers_for_headers, lint_native_headers from sqlbuild.lint._helpers.native_sql import run_native_sql_lint from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions from sqlbuild.lint.constants import HEADER_KIND_SCENARIO, HEADER_KIND_TEST -from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation +from sqlbuild.lint.models import ( + HeaderSpan, + LintBody, + LintConfig, + LintFileRole, + LintRunResult, + LintViolation, +) def run_lint( @@ -52,7 +59,8 @@ def run_lint( file_path: Path contents: str for file_path, contents in sorted(files.items()): - relative_parts: tuple[str, ...] = file_path.relative_to(project_dir).parts + relative_path: Path = file_path.relative_to(project_dir) + relative_parts: tuple[str, ...] = relative_path.parts declaration_directories: frozenset[str] = ( INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES @@ -83,7 +91,8 @@ def run_lint( context=context, dialect=config.dialect, project_dir=project_dir, - allows_dynamic_output_star=file_path.resolve() in dynamic_output_paths, + relative_path=relative_path, + dynamic_output_paths=dynamic_output_paths, compiled_expansions=compiled_expansions, ) ) @@ -130,10 +139,17 @@ def _prepared_bodies( context: SqlExpansionContext, dialect: str, project_dir: Path, - allows_dynamic_output_star: bool, + relative_path: Path, + dynamic_output_paths: frozenset[Path], compiled_expansions: dict[Path, CompiledSqlExpansion] | None, ) -> tuple[LintBody, ...]: bodies: list[LintBody] = [] + role: LintFileRole = lint_file_role( + file_path=file_path, project_dir=project_dir, relative_path=relative_path + ) + allows_dynamic_output_star: bool = ( + bool(dynamic_output_paths) and file_path.resolve() in dynamic_output_paths + ) compiled_expansion: CompiledSqlExpansion | None = (compiled_expansions or {}).get(file_path) external_identifiers: tuple[str, ...] = external_identifiers_for_headers( contents=contents, headers=headers @@ -149,11 +165,12 @@ def _prepared_bodies( headers=headers, file_path=file_path, project_dir=project_dir, + role=role, ): bodies.append( replace( prepare_lint_body( - project_dir=project_dir, + role=role, file_path=file_path, contents=contents, body_range=(body_start, body_end), diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index 88563a4875..b46c5cfd9f 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -7,6 +7,7 @@ from pathlib import Path from sqlbuild.compiler.compile.models import ExpansionSpan +from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint.constants import VIOLATION_SEVERITY_FAULT, VIOLATION_SEVERITY_WARNING from sqlbuild.lint.types import LintSeverity @@ -20,6 +21,15 @@ class HeaderSpan: end: int +@dataclass(frozen=True) +class LintFileRole: + """Path-derived classification of one lint input, derived once per file.""" + + declaration_kind: DeclarationKind | None = None + in_project: bool = False + in_hook_directory: bool = False + + @dataclass(frozen=True) class InterpolationSite: """One sqlbuild interpolation occurrence replaced by a unique sentinel.""" diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index d4c8801470..9fb1bf808a 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -13,6 +13,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint._helpers import expansion from sqlbuild.lint._helpers.expansion import build_lint_expansion_context, prepare_lint_body +from sqlbuild.lint._helpers.headers import lint_file_role from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult @@ -268,7 +269,7 @@ def test_given_generic_audit_argument_when_mapping_then_generated_region_is_not_ context: SqlExpansionContext = build_lint_expansion_context(project_dir=tmp_path) body: LintBody = prepare_lint_body( - project_dir=tmp_path, + role=lint_file_role(file_path=audit, project_dir=tmp_path), file_path=audit, contents=contents, body_range=(10, len(contents)), From 64ef355de366598feeb598518b8798727fc2b573 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:23:59 +0100 Subject: [PATCH 2/6] perf(rules): rebind the compiler declaration index for early SQL lint The background SQL lint still loads its own macro instances, but when their indexed metadata matches the compiler's it reuses the already validated scope index and relationship grants instead of rebuilding them. Any metadata difference falls back to a full private build. --- .../_helpers/attachment/declaration_scope.py | 41 +++++++++++++++++++ .../compile/main/sql_expansion_context.py | 21 +++++++--- src/sqlbuild/lint/_helpers/expansion.py | 2 + .../lint/main/build_expansion_context.py | 2 + .../rule_engine/_helpers/run/rules.py | 6 ++- 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 42504d2603..8f671bb97f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -2,7 +2,10 @@ from __future__ import annotations +import inspect +from collections.abc import Mapping from dataclasses import replace +from types import CodeType from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( build_scope_relationship_grants, @@ -64,3 +67,41 @@ def build_declaration_scope( loaded_macros=loaded_macros, ), ) + + +def rebind_declaration_scope( + *, + scope: DeclarationScopeBuild, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], +) -> DeclarationScopeBuild | None: + """Pair a built index with private macro instances, or return None if their metadata differs.""" + + if _indexed_macro_metadata(loaded_macros) != _indexed_macro_metadata(scope.loaded_macros): + return None + return DeclarationScopeBuild( + loaded_macros=loaded_macros, + index=scope.index, + resolver=build_declaration_scope_resolver( + discovered_inputs=discovered_inputs, + scope_index=scope.index, + loaded_macros=loaded_macros, + ), + ) + + +def _indexed_macro_metadata(loaded_macros: Mapping[str, LoadedMacro]) -> tuple[object, ...]: + return tuple( + ( + key, + macro.name, + macro.relative_path, + macro.raw_source, + macro.dependencies, + code.co_firstlineno + if isinstance(code := getattr(macro.function, "__code__", None), CodeType) + else None, + tuple(inspect.signature(macro.function).parameters), + ) + for key, macro in loaded_macros.items() + ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index 69d20e7db9..dfc0042d0d 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( build_declaration_scope, + rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( build_model_declaration_indexes, @@ -40,8 +41,9 @@ def build_sql_expansion_context( cli_vars: dict[str, object] | None = None, discovered_inputs: DiscoveredProjectInputs | None = None, declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, ) -> SqlExpansionContext: - """Assemble project vars, macros and declarations for SQL expansion.""" + """Assemble SQL expansion inputs; a static scope lends only its index to private macros.""" effective_discovered_inputs: DiscoveredProjectInputs = ( discovered_inputs @@ -55,7 +57,7 @@ def build_sql_expansion_context( cli_vars={} if cli_vars is None else cli_vars, ) scope: DeclarationScopeBuild = declaration_scope or _build_declaration_scope( - discovered_inputs=effective_discovered_inputs + discovered_inputs=effective_discovered_inputs, static_scope=static_declaration_scope ) loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros enums: dict[str, EnumDeclaration] @@ -105,9 +107,16 @@ def build_sql_expansion_context( def _build_declaration_scope( - *, discovered_inputs: DiscoveredProjectInputs + *, discovered_inputs: DiscoveredProjectInputs, static_scope: DeclarationScopeBuild | None ) -> DeclarationScopeBuild: - return build_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=load_project_macros(discovered_inputs.macro_files), + loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) + rebound: DeclarationScopeBuild | None = ( + None + if static_scope is None + else rebind_declaration_scope( + scope=static_scope, discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + ) + ) + return rebound or build_declaration_scope( + discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 9a74de19e5..44ca34990e 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -83,6 +83,7 @@ def build_lint_expansion_context( value_renderer: TypedSqlValueRenderer | None = None, discovered_inputs: DiscoveredProjectInputs | None = None, declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, ) -> SqlExpansionContext: """Build the expansion context, reporting compile failures as lint failures.""" @@ -105,6 +106,7 @@ def build_lint_expansion_context( discovered_inputs=effective_discovered_inputs, value_renderer=effective_renderer, declaration_scope=declaration_scope, + static_declaration_scope=static_declaration_scope, ) except (AdapterUserError, CompileInputError, DiscoveryError) as error: raise ProjectCompileError( diff --git a/src/sqlbuild/lint/main/build_expansion_context.py b/src/sqlbuild/lint/main/build_expansion_context.py index de1610fadb..3a3cefe84f 100644 --- a/src/sqlbuild/lint/main/build_expansion_context.py +++ b/src/sqlbuild/lint/main/build_expansion_context.py @@ -12,9 +12,11 @@ def build_expansion_context( project_dir: Path, discovered_inputs: DiscoveredProjectInputs, declaration_scope: DeclarationScopeBuild | None = None, + static_declaration_scope: DeclarationScopeBuild | None = None, ) -> SqlExpansionContext: return build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, declaration_scope=declaration_scope, + static_declaration_scope=static_declaration_scope, ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index cab2fe127f..8132cec324 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -210,6 +210,7 @@ def prepare_sql_rules( dialect=dialect, enabled_native_rules=codes, header_rules_enabled=False ), discovered_inputs=inputs.discovered_inputs, + static_declaration_scope=inputs.declaration_scope, compiled_expansions={ model.model_file.file_path: model.sql_expansion for model in inputs.model_inputs @@ -224,10 +225,13 @@ def _prepare_sql_lint( project_dir: Path, config: LintConfig, discovered_inputs: DiscoveredProjectInputs, + static_declaration_scope: DeclarationScopeBuild | None, compiled_expansions: dict[Path, CompiledSqlExpansion], ) -> PreparedSqlLintResult: context: SqlExpansionContext = build_expansion_context( - project_dir=project_dir, discovered_inputs=discovered_inputs + project_dir=project_dir, + discovered_inputs=discovered_inputs, + static_declaration_scope=static_declaration_scope, ) result: LintRunResult = run_lint( project_dir=project_dir, From 451ddb1ce1f4ff212dd61ab8ca02b2a560405fd3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:24:09 +0100 Subject: [PATCH 3/6] perf(rules): reuse the early SQL lint file snapshot in compiler rules When the early lint matches the selected SQL rules and project directory, the SQL rule pass reuses its full-project file read for rule identities and dynamic-output proofs instead of walking and re-reading every SQL file on the main thread. --- .../rule_engine/_helpers/run/rules.py | 23 ++++++++++++++++++- src/sqlbuild/rule_engine/models.py | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 8132cec324..1b36030178 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -203,6 +203,7 @@ def prepare_sql_rules( return None return PreparedSqlLint( codes=codes, + project_dir=project_dir, future=executor.submit( _prepare_sql_lint, project_dir=project_dir, @@ -291,6 +292,21 @@ def _selected_project( ) +def _prepared_project_files( + *, prepared_sql: PreparedSqlLint | None, project_dir: Path, codes: tuple[str, ...] +) -> dict[Path, str] | None: + """Reuse the full-project read of a matching early SQL lint for this invocation.""" + + if ( + prepared_sql is None + or prepared_sql.project_dir != project_dir + or set(prepared_sql.codes) != set(codes) + or prepared_sql.future.exception() is not None + ): + return None + return dict(prepared_sql.future.result().result.source_texts) + + def _run_prepared_lint( *, project_dir: Path, @@ -339,6 +355,7 @@ def _run_prepared_lint( compiled_expansions=compiled_expansions, dynamic_output_paths=proof_paths, expansion_context=completed.context, + source_files=dict(prepared.source_texts), ) violations = (*violations, *proven.violations) return replace(prepared, violations=violations, files_checked=len(selected_paths)) @@ -388,7 +405,11 @@ def _run_sql_rules( model_paths: frozenset[Path] = frozenset( project_dir / model.relative_path for model in project.models ) - project_files = collect_project_files(project_dir=project_dir, selected_paths=None) + project_files = _prepared_project_files( + prepared_sql=prepared_sql, project_dir=project_dir, codes=codes + ) + if project_files is None: + project_files = collect_project_files(project_dir=project_dir, selected_paths=None) file_path: Path contents: str for file_path, contents in project_files.items(): diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 54d505a742..9a78f61444 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -332,6 +332,7 @@ class PreparedSqlLint: codes: tuple[str, ...] future: Future[PreparedSqlLintResult] + project_dir: Path | None = None @dataclass(frozen=True) From 81a477daa1e7b8c8b8fae4d3919770598765b0b4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:24:14 +0100 Subject: [PATCH 4/6] perf(rules): decide custom-host startup from the rules cache state before evaluation Native evaluation checked whether the rules cache existed only after building its payloads, while the concurrent SQL rule pass could already have created that directory. When that happened the request omitted the custom host, failed, and was repeated with it. The cache state is now captured before either phase starts. --- src/sqlbuild/rule_engine/_helpers/engine/native.py | 13 ++++++++++++- src/sqlbuild/rule_engine/_helpers/run/rules.py | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index cc055cd92b..c2d18d301f 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -75,6 +75,7 @@ def evaluate_native( initial_findings: tuple[Finding, ...] = (), defer_suppressions: bool = False, verify_determinism: bool = False, + rules_cache_present_at_start: bool | None = None, ) -> RulesResult: """Evaluate one compiled model batch through the native engine.""" @@ -117,7 +118,11 @@ def evaluate_native( retry_native_misses: int = 0 request["custom_host"] = None try: - if not config.cache.enabled or not (project_dir / "target" / "rules-cache").exists(): + if not config.cache.enabled or not ( + rules_cache_exists(project_dir) + if rules_cache_present_at_start is None + else rules_cache_present_at_start + ): custom_host, custom_host_input = _custom_host_payload( project=project, config=config, @@ -168,6 +173,12 @@ def evaluate_native( ) +def rules_cache_exists(project_dir: Path) -> bool: + """Return whether this project already has persisted rules-cache state.""" + + return (project_dir / "target" / "rules-cache").exists() + + def _evaluate_request(request: dict[str, object]) -> str: return _native.evaluate_json( orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str).decode() diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 1b36030178..da51274567 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -39,6 +39,7 @@ evaluate_native, finalize_native_findings, native_catalogue, + rules_cache_exists, ) from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES @@ -126,6 +127,7 @@ def evaluate_rules( catalogue=catalogue, dialect=dialect, defer_suppressions=True, + rules_cache_present_at_start=rules_cache_exists(resolved_project_dir), ) sql_started: float = time.monotonic() sql_result: _SqlRulesEvaluation = _run_sql_rules( From 520472bfc51dd84eb311f971721c65c876ee75d7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:31:33 +0100 Subject: [PATCH 5/6] test: cover lint front-end reuse and early SQL rule parity Pin exact header spans for escaped and unterminated quotes, prove that a rebound declaration scope matches a fresh build while keeping private macro instances, check that cold compiles build the scope once and reuse the early file read, reproduce the custom-host request ordering deterministically, and compare compile-time SQL findings with a standalone rule run on CRLF and macro inputs. --- .../cli/commands/main/rules/_test_types.py | 8 + .../main/rules/test_sql_rule_paths.py | 95 +++++++++ .../sqlbuild/cli/commands/main/test_rules.py | 187 +++++++++++++++++- .../compiler/compile/_helpers/_test_types.py | 8 + .../compiler/compile/_helpers/helpers.py | 29 +++ .../_helpers/test_rebind_declaration_scope.py | 107 ++++++++++ .../sqlbuild/lint/_helpers/test_headers.py | 30 +++ 7 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 5d281bde90..37139e5183 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -130,3 +130,11 @@ class HelperImportCase: helper_path: str helper_source: str expected_error: str + + +@dataclass(frozen=True) +class SqlRulePathParityCase: + description: str + files: dict[str, str] + expected_findings: tuple[tuple[str, int, int, str], ...] + expected_exit: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py new file mode 100644 index 0000000000..3fb9d81431 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py @@ -0,0 +1,95 @@ +"""SQL rule findings agree between compile's early lint and a standalone rule run.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import SqlRulePathParityCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRulePathParityCase( + description="macro, CRLF model and CRLF SQL test", + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL"]\n' + ), + "models/staging/orders.sql": ( + 'MODEL (description "Orders");\n' + "WITH unused_orders AS (SELECT 1 AS order_id), " + "kept AS (SELECT 2 AS order_id)\n" + 'SELECT kept.order_id FROM kept WHERE @order_filter("kept.order_id")\n' + ), + "models/staging/customers.sql": ( + 'MODEL (description "Customers");\r\n' + "WITH spare AS (SELECT 1 AS customer_id)\r\n" + "SELECT 1 AS customer_id\r\n" + ), + "models/staging/_sqlbuild/_macros/order_filter.py": ( + 'def order_filter(column: str) -> str:\n return f"{column} > 0"\n' + ), + "tests/unit/test_orders.sql": ( + "TEST ();\r\nWITH\r\n__ref__orders AS (SELECT 2 AS order_id),\r\n" + "unused_rows AS (SELECT 3 AS x),\r\n" + "__expected__orders AS (SELECT order_id FROM __ref__orders LIMIT 1)\r\n" + "SELECT 1\r\n" + ), + }, + expected_findings=( + ("models/staging/customers.sql", 2, 6, "SQBRSQL005"), + ("models/staging/customers.sql", 2, 6, "SQBRSQL041"), + ("models/staging/customers.sql", 3, 1, "SQBRSQL035"), + ("models/staging/orders.sql", 2, 6, "SQBRSQL005"), + ("models/staging/orders.sql", 2, 47, "SQBRSQL041"), + ("models/staging/orders.sql", 3, 1, "SQBRSQL034"), + ("models/staging/orders.sql", 3, 1, "SQBRSQL035"), + ("tests/unit/test_orders.sql", 4, 1, "SQBRSQL005"), + ("tests/unit/test_orders.sql", 5, 59, "SQBRSQL004"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_sql_findings_when_compiling_or_running_rules_then_findings_are_identical( + test_case: SqlRulePathParityCase, + tmp_path: Path, +) -> None: + compile_dir: Path = prepare_inline_project( + tmp_path=tmp_path / "compile", project_name="orders", repo_files=test_case.files + ) + rules_dir: Path = prepare_inline_project( + tmp_path=tmp_path / "rules", project_name="orders", repo_files=test_case.files + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json", "--no-cache"), project_dir=compile_dir + ) + ruled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rules", "--json", "run", "SQBRSQL"), project_dir=rules_dir + ) + + compile_payload: dict[str, Any] = json.loads(compiled.stdout) + rules_payload: dict[str, Any] = json.loads(ruled.stdout) + compile_findings: list[tuple[str, int, int, str, str]] = [ + (item["path"], item["line"], item["column"], item["code"], item["message"]) + for item in compile_payload["diagnostics"] + ] + rules_findings: list[tuple[str, int, int, str, str]] = [ + (item["path"], item["line"], item["column"], item["code"], item["message"]) + for item in rules_payload["findings"] + ] + assert compiled.returncode == ruled.returncode == test_case.expected_exit + assert compile_findings == rules_findings + assert tuple(finding[:4] for finding in compile_findings) == test_case.expected_findings + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index aec2edb2b9..f7236aa8d3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import threading from collections.abc import Callable from pathlib import Path from typing import cast @@ -13,9 +14,11 @@ import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module import sqlbuild.compiler.compile.main._build_compile_inputs as compile_inputs_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module +import sqlbuild.rule_engine._helpers.engine.native as native_module +import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, ExplicitContractOutputRuleIntegrationTestCase, @@ -743,6 +746,188 @@ def counting_build( assert len(scope_builds) == 1 +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("cold SQL rules rebind the compiler scope index", 1, "SQBRSQL004")], + ids=lambda case: case.description, +) +def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_private_macros( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8") + tests_dir: Path = tmp_path / "tests" / "unit" + macro: Path = tests_dir / "_macros" / "row_limit.py" + macro.parent.mkdir(parents=True) + macro.write_text('def row_limit() -> str:\n return "LIMIT 1"\n', encoding="utf-8") + (tests_dir / "test_macro_orders.sql").write_text( + "TEST ();\nWITH\n__ref__orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT order_id FROM __ref__orders @row_limit())\n" + "SELECT 1\n", + encoding="utf-8", + ) + scope_builds: list[Path] = [] + macro_loads: list[dict[str, LoadedMacro]] = [] + original_build: Callable[..., DeclarationScopeBuild] = ( + declaration_scope_module.build_declaration_scope + ) + original_load: Callable[..., dict[str, LoadedMacro]] = ( + expansion_context_module.load_project_macros + ) + + def counting_build( + *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro] + ) -> DeclarationScopeBuild: + scope_builds.append(tmp_path) + return original_build(discovered_inputs=discovered_inputs, loaded_macros=loaded_macros) + + def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, LoadedMacro]: + loaded: dict[str, LoadedMacro] = original_load(macro_files) + macro_loads.append(loaded) + return loaded + + monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(compile_inputs_module, "load_project_macros", recording_load) + monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) + + cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" in cold + assert len(scope_builds) == 1 + assert len(macro_loads) == 2 + assert macro_loads[0]["row_limit"].function is not macro_loads[1]["row_limit"].function + + +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("cold SQL rules reuse the early file read", 1, "SQBRSQL004")], + ids=lambda case: case.description, +) +def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_file_read( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8") + test_file: Path = tmp_path / "tests" / "unit" / "test_orders.sql" + test_file.parent.mkdir(parents=True) + test_file.write_bytes( + b"TEST ();\r\nWITH\r\n__ref__orders AS (SELECT 1 AS order_id),\r\n" + b"__expected__orders AS (SELECT order_id FROM __ref__orders LIMIT 1)\r\n" + b"SELECT 1\r\n" + ) + collections: list[frozenset[Path] | None] = [] + original_collect: Callable[..., dict[Path, str]] = rules_module.collect_project_files + + def counting_collect( + *, project_dir: Path, selected_paths: frozenset[Path] | None + ) -> dict[Path, str]: + collections.append(selected_paths) + return original_collect(project_dir=project_dir, selected_paths=selected_paths) + + monkeypatch.setattr(rules_module, "collect_project_files", counting_collect) + + cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + cold_collections: int = len(collections) + warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert f"tests/unit/test_orders.sql:{test_case.expected_code}" in cold + assert warm == cold + assert cold_collections == 0 + assert collections == [None] + + +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("custom host starts with the first request", 1, "XSQBRARCH001")], + ids=lambda case: case.description, +) +def test_given_sql_rules_create_cache_first_when_compiling_then_custom_host_is_not_retried( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n\n' + f'[rules]\nselect = ["SQBRSQL004", "{test_case.expected_code}"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n", + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8") + test_file: Path = tmp_path / "tests" / "unit" / "test_orders.sql" + test_file.parent.mkdir(parents=True) + test_file.write_text( + "TEST ();\nWITH\n__ref__orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT order_id FROM __ref__orders LIMIT 1)\n" + "SELECT 1\n", + encoding="utf-8", + ) + rule_file: Path = tmp_path / "rules" / "architecture.py" + rule_file.parent.mkdir() + rule_file.write_text( + """from sqlbuild.rules import Finding, Model, RuleContext, rule + +@rule( + code="XSQBRARCH001", + message="Final models must use the final directory", + remediation="Move this model beneath models/final/.", +) +def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: + return [] if "final" in model.path.parts else [ctx.finding(subject=model)] +""", + encoding="utf-8", + ) + sql_cache_written: threading.Event = threading.Event() + requests: list[bool] = [] + original_write: Callable[..., None] = rules_module._write_sql_rule_cache + original_payloads: Callable[..., list[dict[str, object]]] = native_module._custom_rule_payloads + original_request: Callable[[dict[str, object]], str] = native_module._evaluate_request + + def signalling_write(*, project_dir: Path, bucket: dict[str, dict[str, object]]) -> None: + original_write(project_dir=project_dir, bucket=bucket) + sql_cache_written.set() + + def delayed_payloads(**kwargs: object) -> list[dict[str, object]]: + _ = sql_cache_written.wait(timeout=30) + return original_payloads(**kwargs) + + def recording_request(request: dict[str, object]) -> str: + requests.append(request["custom_host"] is not None) + return original_request(request) + + monkeypatch.setattr(rules_module, "_write_sql_rule_cache", signalling_write) + monkeypatch.setattr(native_module, "_custom_rule_payloads", delayed_payloads) + monkeypatch.setattr(native_module, "_evaluate_request", recording_request) + + cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert sql_cache_written.is_set() + assert requests == [True] + assert { + f"models/orders.sql:{test_case.expected_code}", + "tests/unit/test_orders.sql:SQBRSQL004", + } <= cold + + @pytest.mark.parametrize( "test_case", [RulesIntegrationTestCase("custom rule is enforced and cached", 1, "XSQBRARCH001")], diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 667e9f22b7..01ee8dbeb7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -1049,3 +1049,11 @@ class TypedMacroArgumentDependencyTestCase: macro_file_contents: str sql: str expected_dependency_names: frozenset[str] + + +@dataclass(frozen=True) +class RebindDeclarationScopeTestCase: + description: str + macro_name: str + expected_rebound: bool + private_macro_source: str | None = None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 74fa229840..afe0f15b03 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -441,3 +441,32 @@ def render_compile_diagnostics(*, project: CompiledProject) -> str: f"= help: {diagnostic.help}" for diagnostic in project.diagnostics ) + + +def write_scoped_macro_orders_project(project_dir: Path) -> None: + """Write a project with one global macro and one test-scoped macro.""" + + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n', encoding="utf-8" + ) + model: Path = project_dir / "models" / "orders.sql" + model.parent.mkdir(parents=True) + model.write_text( + 'MODEL (description "Orders");\nSELECT 1 AS order_id WHERE @order_filter("1")\n', + encoding="utf-8", + ) + (project_dir / "macros").mkdir() + (project_dir / "macros" / "order_filter.py").write_text( + 'def order_filter(column: str) -> str:\n return f"{column} > 0"\n', encoding="utf-8" + ) + tests_dir: Path = project_dir / "tests" / "unit" + (tests_dir / "_macros").mkdir(parents=True) + (tests_dir / "_macros" / "row_limit.py").write_text( + 'def row_limit() -> str:\n return "LIMIT 1"\n', encoding="utf-8" + ) + (tests_dir / "test_orders.sql").write_text( + "TEST ();\nWITH\n__ref__orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT order_id FROM __ref__orders @row_limit())\n" + "SELECT 1\n", + encoding="utf-8", + ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py new file mode 100644 index 0000000000..3c31f88dd1 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -0,0 +1,107 @@ +"""Rebinding a built declaration index to privately loaded macro instances.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( + build_declaration_scope, + rebind_declaration_scope, +) +from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.compiler.scopes.types import DeclarationKind +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + RebindDeclarationScopeTestCase, +) +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + write_scoped_macro_orders_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RebindDeclarationScopeTestCase( + description="global macro", macro_name="order_filter", expected_rebound=True + ), + RebindDeclarationScopeTestCase( + description="test-scoped macro", macro_name="row_limit", expected_rebound=True + ), + ], + ids=lambda case: case.description, +) +def test_given_same_discovery_when_rebinding_private_macros_then_index_matches_a_fresh_build( + test_case: RebindDeclarationScopeTestCase, + tmp_path: Path, +) -> None: + write_scoped_macro_orders_project(tmp_path) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + shared: DeclarationScopeBuild = build_declaration_scope( + discovered_inputs=discovered, + loaded_macros=load_project_macros(discovered.macro_files), + ) + private_macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) + + rebound: DeclarationScopeBuild | None = rebind_declaration_scope( + scope=shared, discovered_inputs=discovered, loaded_macros=private_macros + ) + + assert (rebound is not None) is test_case.expected_rebound + assert rebound is not None + fresh: DeclarationScopeBuild = build_declaration_scope( + discovered_inputs=discovered, loaded_macros=private_macros + ) + identity: DeclarationIdentity = DeclarationIdentity(DeclarationKind.MACRO, test_case.macro_name) + private_value: object = rebound.resolver.projection.declarations[identity] + shared_value: object = shared.resolver.projection.declarations[identity] + assert rebound.index is shared.index + assert rebound.index == fresh.index + assert rebound.loaded_macros is private_macros + assert private_value is private_macros[test_case.macro_name] + assert isinstance(shared_value, LoadedMacro) + assert private_macros[test_case.macro_name].function is not shared_value.function + + +@pytest.mark.parametrize( + "test_case", + [ + RebindDeclarationScopeTestCase( + description="different macro source", + macro_name="row_limit", + expected_rebound=False, + private_macro_source='def row_limit() -> str:\n return "LIMIT 2"\n', + ) + ], + ids=lambda case: case.description, +) +def test_given_different_private_macro_metadata_when_rebinding_then_full_build_is_required( + test_case: RebindDeclarationScopeTestCase, + tmp_path: Path, +) -> None: + write_scoped_macro_orders_project(tmp_path) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + shared: DeclarationScopeBuild = build_declaration_scope( + discovered_inputs=discovered, + loaded_macros=load_project_macros(discovered.macro_files), + ) + private_macros: dict[str, LoadedMacro] = load_project_macros(discovered.macro_files) + private_macros[test_case.macro_name] = replace( + private_macros[test_case.macro_name], raw_source=str(test_case.private_macro_source) + ) + + rebound: DeclarationScopeBuild | None = rebind_declaration_scope( + scope=shared, discovered_inputs=discovered, loaded_macros=private_macros + ) + + assert (rebound is not None) is test_case.expected_rebound + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py index a2710cb009..8d39b52324 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py @@ -69,6 +69,21 @@ ), expected_kinds=("MODEL",), ), + ScanHeadersTestCase( + description="unterminated header paren is not a header", + contents="MODEL (\n description 'x'\nSELECT (1\n", + expected_kinds=(), + ), + ScanHeadersTestCase( + description="unterminated quote inside header is not a header", + contents="MODEL (\n description 'x)\nSELECT 1\n", + expected_kinds=(), + ), + ScanHeadersTestCase( + description="trailing escape inside header quote is not a header", + contents="MODEL (description '\\", + expected_kinds=(), + ), ScanHeadersTestCase( description="standard SQL backslash does not escape closing quote", contents="SCENARIO ();\nSELECT 'C:\\temp\\'\nTEST ();\nSELECT 1\n", @@ -118,6 +133,21 @@ def test_given_single_header_resource_when_scanning_first_only_then_body_is_not_ contents="MODEL (\n materialized table\n)\nSELECT 1\n", expected_span_text="MODEL (\n materialized table\n)", ), + HeaderSpanTextTestCase( + description="escaped quotes and quoted parens stay inside the span", + contents=( + "MODEL (\n description 'it\\'s (open', note \"a \\\" ) b\", x ((1))\n);" + "\nSELECT ')'\n" + ), + expected_span_text=( + "MODEL (\n description 'it\\'s (open', note \"a \\\" ) b\", x ((1))\n);" + ), + ), + HeaderSpanTextTestCase( + description="other quote kind does not close the open quote", + contents='TEST (name "a \' ) b") \n ;\nSELECT 1\n', + expected_span_text='TEST (name "a \' ) b") \n ;', + ), ], ids=lambda case: case.description, ) From c9ba19c9343cbee866ecfdac1da8ff3c9c6ab9fa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:12:58 +0100 Subject: [PATCH 6/6] test: type the delayed custom-rule payload wrapper --- .../sqlbuild/cli/commands/main/test_rules.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index f7236aa8d3..ebdc72c8ec 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,8 +17,9 @@ import sqlbuild.rule_engine._helpers.engine.native as native_module import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro +from sqlbuild.compiler.compile.models import CompiledProject, DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs +from sqlbuild.rule_engine.models import Rule from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, ExplicitContractOutputRuleIntegrationTestCase, @@ -906,9 +907,20 @@ def signalling_write(*, project_dir: Path, bucket: dict[str, dict[str, object]]) original_write(project_dir=project_dir, bucket=bucket) sql_cache_written.set() - def delayed_payloads(**kwargs: object) -> list[dict[str, object]]: + def delayed_payloads( + *, + catalogue: tuple[Rule, ...], + project: CompiledProject, + project_dir: Path, + cache_enabled: bool, + ) -> list[dict[str, object]]: _ = sql_cache_written.wait(timeout=30) - return original_payloads(**kwargs) + return original_payloads( + catalogue=catalogue, + project=project, + project_dir=project_dir, + cache_enabled=cache_enabled, + ) def recording_request(request: dict[str, object]) -> str: requests.append(request["custom_host"] is not None)