perf(compile): cache model attachment on warm and one-edit compiles

src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md

@@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/
 - Arguments
 - Typed reference arguments
 - Keep implementation details private
+- Keep macros deterministic
 - Use macros in tests
 - Use macros in hooks
 - Where macros work
@@ -154,6 +155,27 @@ Python implementation details.
 
 Imported functions are not re-exported as new macros from the importing file.
 
+## Keep macros deterministic
+
+A macro must return the same SQL every time it receives the same arguments and macro context. Do
+not read environment variables, the clock, random values, files, or network services inside a
+macro. Pass such values in through [project variables](interpolation.md#project-variables)
+or the macro context instead.
+
+SQLBuild relies on this when it reuses compiled models between compile runs. A model is compiled
+again when its file, its configuration and defaults, project variables, the target, the adapter,
+any project macro or declaration, or the installed SQLBuild version changes. Models whose
+configuration, hooks, or SQL read an environment variable or `run.id` are compiled again on every
+run. `sqb compile --no-cache` compiles everything from scratch.
+
+The reuse check covers model files, project and local configuration, macro, enum, constant, schema,
+and SQL hook files, the source files that define Python hooks and audit factories, and custom
+adapter files under `adapters/`, along with the target, project variables, and command-line flags.
+It does not cover code those files import from elsewhere: third-party libraries, and project
+modules outside those locations that a hook or adapter imports. Keep macro, hook, and adapter code
+independent of such modules, or run `sqb compile --no-cache` (or delete `target/cache/compiler/`)
+after upgrading or editing one that changes their output.
+
 ## Use macros in tests
 
 Tests are SQL, so they can use macros as reusable fixture generators:

src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py

@@ -35,6 +35,24 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]:
         _ACTIVE_COLLECTOR.reset(token)
 
 
+@contextmanager
+def capture_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics | None]:
+    """Record diagnostics reported inside the block and forward them to the active collector."""
+
+    parent: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get()
+    if parent is None:
+        yield None
+        return
+    captured: CollectedCompileDiagnostics = CollectedCompileDiagnostics()
+    token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(captured)
+    try:
+        yield captured
+    finally:
+        _ACTIVE_COLLECTOR.reset(token)
+        for key, diagnostic in captured.entries:
+            parent.add(key=key, diagnostic=diagnostic)
+
+
 def with_collected_compile_diagnostics[**P, R: (CompileProjectInputs, CompiledProject)](
     build: Callable[P, R],
 ) -> Callable[P, R]:

src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py

@@ -0,0 +1,235 @@
+"""Cache identity for one model's attachment result."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from dataclasses import replace
+from pathlib import Path
+
+from sqlbuild.compiler.compile._helpers.render.volatile_reads import environment_read_label
+from sqlbuild.compiler.compile.models import (
+    DeclarationScopeResolver,
+    ModelAttachmentEnvironment,
+    ModelInputBuildContext,
+)
+from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile
+from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore
+from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity
+from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ResourceRecord
+from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind
+
+_TARGET_ENVIRONMENT_REFERENCE: re.Pattern[str] = re.compile(r"\bENV:\s*([A-Za-z0-9_]+)")
+_PROCESS_LOCAL_REPR_MARKER: str = " at 0x"
+_PROJECT_ADAPTERS_DIRECTORY: str = "adapters"
+_ENVIRONMENT_DIGEST_BYTES: int = 32
+_ATTACHMENT_DECLARATION_KINDS: frozenset[DeclarationKind] = frozenset(
+    {
+        DeclarationKind.MACRO,
+        DeclarationKind.ENUM,
+        DeclarationKind.CONSTANT,
+        DeclarationKind.SCHEMA,
+        DeclarationKind.SQL_HOOK,
+        DeclarationKind.PYTHON_HOOK,
+    }
+)
+
+
+def model_attachment_environment(
+    *,
+    discovered_inputs: DiscoveredProjectInputs,
+    context: ModelInputBuildContext,
+    no_sql_validation: bool,
+    defer_model_sql_validation: bool,
+) -> ModelAttachmentEnvironment | None:
+    """Return the project-wide key part, or None when an input has no stable identity."""
+
+    target_text: str = repr(
+        None if context.target_config is None else replace(context.target_config, connection={})
+    )
+    target_environment_names: tuple[str, ...] = tuple(
+        sorted(set(_TARGET_ENVIRONMENT_REFERENCE.findall(target_text)))
+    )
+    renderer_type: type = type(context.value_renderer)
+    material: str = "\0".join(
+        (
+            compiled_code_identity(),
+            str(discovered_inputs.project_dir.resolve() if discovered_inputs.project_dir else ""),
+            context.macro_context.adapter_name,
+            f"{renderer_type.__module__}.{renderer_type.__qualname__}",
+            str(context.collection_rendering),
+            context.sql_lexical_syntax.cache_key,
+            _project_adapter_sources(discovered_inputs),
+            repr(context.effective_target_name),
+            target_text,
+            repr(tuple((name, os.environ.get(name)) for name in target_environment_names)),
+            repr(context.effective_vars),
+            repr(context.effective_settings),
+            repr(
+                (
+                    context.macro_context.sql_analysis_enabled,
+                    context.macro_context.target_name,
+                    discovered_inputs.project_config.references.enforce_explicit,
+                    no_sql_validation,
+                    defer_model_sql_validation,
+                )
+            ),
+            repr(discovered_inputs.project_config.defaults),
+            repr(discovered_inputs.project_config.materialization_defaults),
+            repr(sorted(item.name for item in discovered_inputs.materialization_files)),
+            repr(sorted(item.name for item in discovered_inputs.providers)),
+            _declaration_surface(
+                discovered_inputs=discovered_inputs, resolver=context.declaration_resolver
+            ),
+        )
+    )
+    if _PROCESS_LOCAL_REPR_MARKER in material:
+        return None
+    return ModelAttachmentEnvironment(
+        digest=hashlib.blake2b(
+            material.encode("utf-8", "surrogatepass"), digest_size=_ENVIRONMENT_DIGEST_BYTES
+        ).hexdigest(),
+        keyed_reads=frozenset(environment_read_label(name) for name in target_environment_names),
+    )
+
+
+def model_attachment_key(
+    *,
+    store: FactCacheStore,
+    environment: ModelAttachmentEnvironment,
+    model_file: DiscoveredSqlModelFile,
+    path_default: tuple[str | None, object],
+    resolver: DeclarationScopeResolver | None,
+) -> str:
+    """Return the exact identity of one model's attachment inputs."""
+
+    relative_path: str = model_file.relative_path.as_posix()
+    return store.key(
+        environment.digest,
+        relative_path,
+        str(model_file.file_path),
+        model_file.contents,
+        repr(model_file.extract_implicit_alias_columns),
+        repr(path_default),
+        "" if resolver is None else _resource_scope(resolver=resolver, model_file=model_file),
+    )
+
+
+def _resource_scope(
+    *, resolver: DeclarationScopeResolver, model_file: DiscoveredSqlModelFile
+) -> str:
+    records: tuple[ResourceRecord, ...] = resolver.lookup.resources_by_path.get(
+        model_file.relative_path.as_posix(), ()
+    )
+    identities: tuple[ResourceIdentity, ...] = (
+        ResourceIdentity(ResourceKind.MODEL, model_file.file_path.stem),
+        *(record.identity for record in records),
+    )
+    return repr(
+        (
+            records,
+            tuple(
+                resolver.lookup.grants_by_resource.get(identity, ())
+                for identity in dict.fromkeys(identities)
+            ),
+        )
+    )
+
+
+def _declaration_surface(
+    *, discovered_inputs: DiscoveredProjectInputs, resolver: DeclarationScopeResolver | None
+) -> str:
+    """Return every shared macro and declaration source; editing one re-attaches every model."""
+
+    parts: list[str] = [
+        repr(
+            (
+                type(item).__name__,
+                item.relative_path.as_posix(),
+                item.contents,
+                item.scope_kind,
+                item.ownership_root,
+                item.owning_path,
+                item.declaration_root,
+            )
+        )
+        for item in (
+            *discovered_inputs.macro_files,
+            *discovered_inputs.enum_files,
+            *discovered_inputs.constant_files,
+            *discovered_inputs.model_schema_files,
+            *discovered_inputs.sql_hook_files,
+        )
+    ]
+    parts.extend(
+        repr(
+            (
+                hook.name,
+                hook.relative_path.as_posix(),
+                hook.description,
+                hook.reads,
+                hook.provider_usages,
+                hook.scope_kind,
+                hook.ownership_root,
+                hook.owning_path,
+                hook.declaration_root,
+                _source_text(hook.file_path),
+            )
+        )
+        for hook in discovered_inputs.hook_functions
+    )
+    parts.extend(
+        repr(
+            (
+                factory.name,
+                factory.relative_path.as_posix(),
+                factory.line,
+                factory.cases,
+                _source_text(factory.file_path),
+            )
+        )
+        for factory in discovered_inputs.audit_factories
+    )
+    if resolver is not None:
+        parts.extend(
+            repr(record)
+            for record in resolver.lookup.index.declarations
+            if _is_shared_attachment_declaration(record)
+        )
+    return "\0".join(parts)
+
+
+def _is_shared_attachment_declaration(record: DeclarationRecord) -> bool:
+    owner: ResourceIdentity | None = record.identity.owner
+    return record.identity.kind in _ATTACHMENT_DECLARATION_KINDS and (
+        owner is None or owner.kind is not ResourceKind.MODEL
+    )
+
+
+def _project_adapter_sources(discovered_inputs: DiscoveredProjectInputs) -> str:
+    project_dir: Path | None = discovered_inputs.project_dir
+    adapter_paths: tuple[Path, ...] = (
+        ()
+        if project_dir is None
+        else tuple(sorted((project_dir / _PROJECT_ADAPTERS_DIRECTORY).rglob("*.py")))
+    )
+    return repr(
+        (
+            _source_text(
+                None
+                if discovered_inputs.adapter_file is None
+                else discovered_inputs.adapter_file.file_path
+            ),
+            tuple((path.as_posix(), _source_text(path)) for path in adapter_paths),
+        )
+    )
+
+
+def _source_text(path: Path | None) -> str:
+    if path is None:
+        return ""
+    try:
+        return path.read_text(encoding="utf-8", errors="surrogateescape")
+    except OSError:
+        return ""

src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py

@@ -17,6 +17,10 @@
     expand_sql_macros_result,
     expand_sql_macros_with_spans,
 )
+from sqlbuild.compiler.compile._helpers.render.volatile_reads import (
+    note_context_read,
+    note_environment_read,
+)
 from sqlbuild.compiler.compile.constants import (
     SQL_CONTEXT_NAME_EXTRA_TOKENS,
     SQL_IDENTIFIER_EXTRA_TOKEN,
@@ -366,6 +370,7 @@ def _render_interpolation_token(
         if env_name_end == env_name_start:
             raise CompileInputError(f"invalid environment interpolation token in '{file_path}'")
         env_name: str = sql[env_name_start:env_name_end]
+        note_environment_read(env_name)
         if env_name not in os.environ:
             raise CompileInputError(
                 f"unknown environment variable '@@ENV:{env_name}' in '{file_path}'"
@@ -387,6 +392,7 @@ def _render_interpolation_token(
             context_values=context_values,
         )
         context_name = sql[context_name_start:context_name_end]
+        note_context_read(context_name)
         if context_name not in context_values:
             raise CompileInputError(
                 f"SQL text in '{file_path}' references unknown CTX key '{context_name}'"

src/sqlbuild/compiler/compile/_helpers/render/templating.py

@@ -8,6 +8,10 @@
 from dataclasses import dataclass
 
 from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text
+from sqlbuild.compiler.compile._helpers.render.volatile_reads import (
+    note_context_read,
+    note_environment_read,
+)
 from sqlbuild.compiler.compile.constants import (
     MISSING_TEMPLATE_CONTEXT_MESSAGE_PART,
     MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART,
@@ -521,6 +525,7 @@ def _lookup_variable(*, name: str, variables: dict[str, object], context_label:
 
 
 def _lookup_environment_variable(*, name: str, context_label: str) -> str:
+    note_environment_read(name)
     if name not in os.environ:
         raise CompileInputError(f"{context_label} references missing ENV variable '{name}'")
     return os.environ[name]
@@ -537,6 +542,7 @@ def _lookup_context_value(
         if preserve_unknown_context:
             return f"${{{TemplateNamespace.CTX}:{name}}}"
         raise CompileInputError(f"{context_label} references unknown CTX key '{name}'")
+    note_context_read(name)
     context_value: str | None = context_values.get(name)
     if context_value is None:
         raise CompileInputError(

src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py

@@ -0,0 +1,47 @@
+"""Record per-invocation values that a reused attachment result cannot reproduce."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar, Token
+
+from sqlbuild.compiler.compile.types import CompileContextKey, TemplateNamespace
+
+_ACTIVE_READS: ContextVar[set[str] | None] = ContextVar(
+    "sqlbuild_volatile_compile_reads", default=None
+)
+
+
+def environment_read_label(name: str) -> str:
+    """Return the recorded label for one environment variable read."""
+
+    return f"{TemplateNamespace.ENV}:{name}"
+
+
+def note_environment_read(name: str) -> None:
+    """Record that template or SQL interpolation read one environment variable."""
+
+    reads: set[str] | None = _ACTIVE_READS.get()
+    if reads is not None:
+        reads.add(environment_read_label(name))
+
+
+def note_context_read(name: str) -> None:
+    """Record a CTX read whose value differs between compile invocations."""
+
+    reads: set[str] | None = _ACTIVE_READS.get()
+    if reads is not None and name == CompileContextKey.RUN_ID:
+        reads.add(f"{TemplateNamespace.CTX}:{name}")
+
+
+@contextmanager
+def record_volatile_reads() -> Iterator[set[str]]:
+    """Collect environment and run-id reads made while the block runs."""
+
+    reads: set[str] = set()
+    token: Token[set[str] | None] = _ACTIVE_READS.set(reads)
+    try:
+        yield reads
+    finally:
+        _ACTIVE_READS.reset(token)