perf: accelerate no-cache cold compiles

src/sqlbuild/cli/commands/_helpers/compile/output.py

@@ -6,6 +6,8 @@
 from importlib.metadata import PackageNotFoundError, version
 from pathlib import Path
 
+import orjson
+
 from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME
 from sqlbuild.cli.commands.types import CompileLineageMode
 from sqlbuild.cli.output.models import (
@@ -175,7 +177,10 @@ def format_compile_json(
         "resources": _resources(graph=graph, lineage=lineage),
         "artifacts": _artifacts(written=written, manifest=manifest),
     }
-    return json.dumps(result, indent=2)
+    try:
+        return orjson.dumps(result, option=orjson.OPT_INDENT_2).decode()
+    except TypeError:
+        return json.dumps(result, indent=2)
 
 
 def _summary(

src/sqlbuild/cli/commands/_helpers/compile/target_writer.py

@@ -332,12 +332,12 @@ def _write_static_tests(
         if project.compile_cache_dir is not None
         else None
     )
-    model_chain_names_by_key: dict[CompiledObjectKey, tuple[str, ...]] = (
-        sql_test_model_chain_names_by_key(
+    model_chain_names_by_key: dict[CompiledObjectKey, tuple[str, ...]] = {}
+    if identity_context is not None:
+        model_chain_names_by_key = sql_test_model_chain_names_by_key(
             project=project,
             tests=project.sql_tests,
         )
-    )
     pending: list[tuple[CompiledSqlTest, str | None, str | None]] = []
     for test in project.sql_tests:
         record_key: str | None = None

src/sqlbuild/compiler/compile/_helpers/analysis/columns.py

@@ -23,6 +23,7 @@
     FULL_JOIN_SIDE,
     LEFT_JOIN_SIDE,
     RIGHT_JOIN_SIDE,
+    SQL_QUALIFIER_SEPARATOR_TOKEN,
     SQL_WILDCARD_TOKEN,
 )
 from sqlbuild.compiler.compile.models import (
@@ -174,6 +175,12 @@
     r'"([A-Za-z_][A-Za-z0-9_]*)"\)\s*(?=\()'
 )
 _PLACEHOLDER_PATTERN: re.Pattern[str] = re.compile(r"@@@(\w+)")
+_QUALIFIED_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(
+    r'(?<![A-Za-z0-9_$])(?:"(?P<double>[^"]+)"|`(?P<backtick>[^`]+)`|'
+    r"\[(?P<bracket>[^\]]+)\]|(?P<plain>[A-Za-z_$][A-Za-z0-9_$]*))"
+    r"(?:\s|--[^\r\n]*(?:\r?\n|$)|/\*.*?\*/)*\.",
+    re.DOTALL,
+)
 
 
 def _infer_columns_with_polyglot(
@@ -1067,17 +1074,19 @@ def relation_stub(match: re.Match[str]) -> str:
 
 
 def _qualified_reference_names(*, query_sql: str, reference_names: Iterable[str]) -> frozenset[str]:
-    qualified: set[str] = set()
+    if SQL_QUALIFIER_SEPARATOR_TOKEN not in query_sql:
+        return frozenset()
+    names_by_normalized: dict[str, list[str]] = {}
     for value in reference_names:
         name: str = str(value)
-        escaped: str = re.escape(name)
-        if re.search(
-            rf'(?<![A-Za-z0-9_$])(?:"{escaped}"|`{escaped}`|\[{escaped}\]|{escaped})'
-            r"(?:\s|--[^\r\n]*(?:\r?\n|$)|/\*.*?\*/)*\.",
-            query_sql,
-            re.IGNORECASE | re.DOTALL,
-        ):
-            qualified.add(name)
+        names_by_normalized.setdefault(name.casefold(), []).append(name)
+    if not names_by_normalized:
+        return frozenset()
+
+    qualified: set[str] = set()
+    for match in _QUALIFIED_IDENTIFIER_PATTERN.finditer(query_sql):
+        identifier: str = next(value for value in match.groups() if value is not None)
+        qualified.update(names_by_normalized.get(identifier.casefold(), ()))
     return frozenset(qualified)
 
 

src/sqlbuild/compiler/compile/_helpers/assembly/project.py

@@ -536,21 +536,23 @@ def _analyze_model_sql_in_parallel(
     request_cache_keys: tuple[str, ...] = tuple(
         request.cache_key for request in requests if request.cache_key is not None
     )
-    compact_batch_plan: CompactAnalysisCachePlan | None = build_compact_analysis_cache_plan(
-        context=analysis_cache,
-        models=tuple(
-            CompactAnalysisCacheModel(
-                name=_model_name(request.model_input),
-                cache_key=request.cache_key,
-                upstream_names=_referenced_model_names(
-                    model_input=request.model_input,
-                    available_names=analyzed_model_names,
-                ),
-            )
-            for request in requests
-        ),
-        min_model_count=_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT,
-    )
+    compact_batch_plan: CompactAnalysisCachePlan | None = None
+    if analysis_cache is not None:
+        compact_batch_plan = build_compact_analysis_cache_plan(
+            context=analysis_cache,
+            models=tuple(
+                CompactAnalysisCacheModel(
+                    name=_model_name(request.model_input),
+                    cache_key=request.cache_key,
+                    upstream_names=_referenced_model_names(
+                        model_input=request.model_input,
+                        available_names=analyzed_model_names,
+                    ),
+                )
+                for request in requests
+            ),
+            min_model_count=_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT,
+        )
     compact_cached_analyses: dict[str, PolyglotAnalysisResult] = {}
     compact_batch_hit_count: int = 0
     compact_candidate: CompactAnalysisCacheCandidate | None = (

src/sqlbuild/compiler/compile/_helpers/attachment/core.py

@@ -251,6 +251,7 @@ class _ReusableModelConfigCache:
     path_defaults: dict[str, dict[str, object]]
     target_config: TargetConfig | None
     reusable_by_path_default: dict[str | None, bool] = field(default_factory=dict)
+    reusable_metadata_by_identity: dict[int, tuple[object, bool]] = field(default_factory=dict)
     configs: dict[str | None, CompileModelConfig] = field(default_factory=dict)
 
     def get(
@@ -284,15 +285,21 @@ def remember(
         }
         self.configs[matched_path_default] = replace(config, values=reusable_values)
 
-    @staticmethod
     def _reusable_metadata(
+        self,
         model_header_values: dict[str, object],
     ) -> dict[str, object] | None:
         if not model_header_values:
             return {}
         if model_header_values.keys() != _REUSABLE_MODEL_HEADER_KEYS:
             return None
-        if _contains_dynamic_or_unsupported_reusable_metadata(model_header_values):
+        metadata: object = model_header_values["columns"]
+        cached: tuple[object, bool] | None = self.reusable_metadata_by_identity.get(id(metadata))
+        if cached is not None and cached[0] is metadata:
+            return model_header_values if cached[1] else None
+        reusable: bool = not _contains_dynamic_or_unsupported_reusable_metadata(metadata)
+        self.reusable_metadata_by_identity[id(metadata)] = (metadata, reusable)
+        if not reusable:
             return None
         return model_header_values
 

src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py

@@ -188,7 +188,7 @@ def build_test_inputs(
                 parent
             )
             if scoped_declarations is not None:
-                scoped_declarations = _rebind_test_declarations(
+                scoped_declarations = _project_test_declaration_consumer(
                     context=scoped_declarations, consumer=resource
                 )
             else:
@@ -223,9 +223,10 @@ def build_test_inputs(
                             f"unused parameters: {', '.join(unused_parameters)} in case "
                             f"'{test_case.name}'"
                         )
-                expanded_test_block: DiscoveredSqlTestBlock = replace(
-                    test_block,
-                    sql_body=parameter_sql,
+                expanded_test_block: DiscoveredSqlTestBlock = (
+                    test_block
+                    if parameter_sql is test_block.sql_body
+                    else replace(test_block, sql_body=parameter_sql)
                 )
                 test_mode: SqlTestMode = test_block.mode
                 tested_resource_names: tuple[str, ...] = ()
@@ -319,39 +320,15 @@ def build_test_inputs(
     return tuple(test_inputs)
 
 
-def _rebind_test_declarations(
+def _project_test_declaration_consumer(
     *, context: DeclarationExpansionContext, consumer: ResourceIdentity
 ) -> DeclarationExpansionContext:
-    declarations: DeclarationResolutionContext = context.declarations
     return replace(
         context,
-        declarations=replace(
-            declarations,
-            consumer=consumer,
-            enum_visibility=_rebind_visibility(
-                visibility=declarations.enum_visibility, consumer=consumer
-            ),
-            constant_visibility=_rebind_visibility(
-                visibility=declarations.constant_visibility, consumer=consumer
-            ),
-            macro_visibility=_rebind_visibility(
-                visibility=declarations.macro_visibility, consumer=consumer
-            ),
-        ),
+        declarations=replace(context.declarations, consumer=consumer),
     )
 
 
-def _rebind_visibility(
-    *,
-    visibility: dict[str, tuple[VisibilityRecord, ...]],
-    consumer: ResourceIdentity,
-) -> dict[str, tuple[VisibilityRecord, ...]]:
-    rebound: dict[str, tuple[VisibilityRecord, ...]] = {}
-    for name, records in visibility.items():
-        rebound[name] = tuple(replace(record, resource=consumer) for record in records)
-    return rebound
-
-
 def _macro_test_declaration_usages(
     *,
     sql: str,

src/sqlbuild/compiler/compile/constants.py

@@ -44,6 +44,7 @@
 SQL_STATEMENT_TERMINATOR_TOKEN: str = ";"
 SQL_SINGLE_QUOTE_TOKEN: str = "'"
 SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`"})
+SQL_QUALIFIER_SEPARATOR_TOKEN: str = "."
 SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'})
 SQL_IDENTIFIER_EXTRA_TOKEN: str = "_"
 SQL_UNION_KEYWORD: str = "UNION"

src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py

@@ -14,6 +14,7 @@
 
 from pydantic import ValidationError
 
+from sqlbuild.compiler.discovery._helpers.filesystem.sql_test_files import discover_sql_test_files
 from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function
 from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file
 from sqlbuild.compiler.discovery._helpers.sql.declarations import (
@@ -32,7 +33,6 @@
     prepare_model_file_headers,
 )
 from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file
-from sqlbuild.compiler.discovery._helpers.sql.tests import parse_sql_test_file
 from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml
 from sqlbuild.compiler.discovery._helpers.yml.sources import parse_sources_yml
 from sqlbuild.compiler.discovery.constants import (
@@ -84,7 +84,6 @@
     DiscoveredSqlHookFile,
     DiscoveredSqlModelFile,
     DiscoveredSqlScenarioFile,
-    DiscoveredSqlTestBlock,
     DiscoveredSqlTestFile,
     DiscoveredTaskFunction,
     DiscoveryFileFault,
@@ -736,33 +735,17 @@ def discover_test_files(
     if not tests_root.is_dir():
         return ()
 
-    discovered_test_files: list[DiscoveredSqlTestFile] = []
-    file_path: Path
-    for file_path in sorted(tests_root.rglob("*.sql")):
-        if selected_paths is not None and file_path.resolve() not in selected_paths:
-            continue
-        if _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir):
-            continue
-        try:
-            contents: str = file_path.read_text(encoding="utf-8")
-            blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file(
-                contents=contents, file_path=file_path
-            )
-        except (OSError, UnicodeError, ValueError, SyntaxError) as error:
-            if on_fault is None:
-                raise
-            on_fault(_discovery_fault(project_dir=project_dir, path=file_path, error=error))
-            continue
-        discovered_test_files.append(
-            DiscoveredSqlTestFile(
-                file_path=file_path,
-                relative_path=file_path.relative_to(project_dir),
-                contents=contents,
-                blocks=blocks,
-                ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT),
-            )
-        )
-    return tuple(discovered_test_files)
+    file_paths: tuple[Path, ...] = tuple(
+        file_path
+        for file_path in sorted(tests_root.rglob("*.sql"))
+        if (selected_paths is None or file_path.resolve() in selected_paths)
+        and not _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir)
+    )
+    return discover_sql_test_files(
+        project_dir=project_dir,
+        file_paths=file_paths,
+        on_fault=on_fault,
+    )
 
 
 def discover_scenario_files(

src/sqlbuild/compiler/discovery/_helpers/filesystem/sql_test_files.py

@@ -0,0 +1,92 @@
+"""Filesystem discovery for SQL-native unit test files."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+from sqlbuild.compiler.discovery._helpers.sql.tests import (
+    parse_sql_test_file,
+    prepare_sql_test_file_headers,
+)
+from sqlbuild.compiler.discovery.constants import SQL_TESTS_OWNERSHIP_ROOT
+from sqlbuild.compiler.discovery.exceptions import SqlTestParseError
+from sqlbuild.compiler.discovery.models import (
+    DiscoveredSqlTestBlock,
+    DiscoveredSqlTestFile,
+    DiscoveryFileFault,
+)
+
+_TEST_FILE_BATCH_SIZE: int = 512
+
+
+def discover_sql_test_files(
+    *,
+    project_dir: Path,
+    file_paths: tuple[Path, ...],
+    on_fault: Callable[[DiscoveryFileFault], None] | None,
+) -> tuple[DiscoveredSqlTestFile, ...]:
+    """Read and parse ordered SQL test paths through bounded native header batches."""
+
+    loaded: list[tuple[Path, str | None, Exception | None]] = []
+    for file_path in file_paths:
+        try:
+            loaded.append((file_path, file_path.read_text(encoding="utf-8"), None))
+        except (OSError, UnicodeError, ValueError, SyntaxError) as error:
+            loaded.append((file_path, None, error))
+
+    discovered: list[DiscoveredSqlTestFile] = []
+    for batch_start in range(0, len(loaded), _TEST_FILE_BATCH_SIZE):
+        batch: list[tuple[Path, str | None, Exception | None]] = loaded[
+            batch_start : batch_start + _TEST_FILE_BATCH_SIZE
+        ]
+        _prepare_batch(batch)
+        for file_path, contents, read_error in batch:
+            if read_error is not None:
+                if on_fault is None:
+                    raise read_error
+                on_fault(_file_fault(project_dir=project_dir, path=file_path, error=read_error))
+                continue
+            if contents is None:
+                raise SqlTestParseError("SQL test file read returned neither contents nor an error")
+            try:
+                blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file(
+                    contents=contents, file_path=file_path
+                )
+            except (OSError, UnicodeError, ValueError, SyntaxError) as error:
+                if on_fault is None:
+                    raise
+                on_fault(_file_fault(project_dir=project_dir, path=file_path, error=error))
+                continue
+            discovered.append(
+                DiscoveredSqlTestFile(
+                    file_path=file_path,
+                    relative_path=file_path.relative_to(project_dir),
+                    contents=contents,
+                    blocks=blocks,
+                    ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT),
+                )
+            )
+    return tuple(discovered)
+
+
+def _prepare_batch(batch: list[tuple[Path, str | None, Exception | None]]) -> None:
+    contents_batch: list[str] = []
+    for _path, contents, error in batch:
+        if contents is not None and error is None:
+            contents_batch.append(contents)
+    try:
+        prepare_sql_test_file_headers(contents_batch)
+    except (OSError, UnicodeError, ValueError, SyntaxError):
+        pass
+
+
+def _file_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault:
+    try:
+        relative_path: Path | None = path.relative_to(project_dir)
+    except ValueError:
+        relative_path = None
+    return DiscoveryFileFault(
+        path=relative_path,
+        message=str(error).replace(str(project_dir), "."),
+    )