perf(compile): reuse discovery, analysis and writes after model-only edits

src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md

@@ -125,9 +125,14 @@ line to stderr before the report.
 
 When the only changes since the stored compile are edits to existing model files, compile runs in
 full but reuses the stored render of every unchanged model, together with the diagnostics and
-environment reads that rendering reported. Only the edited models are rendered again. Analysis,
-contracts, rules, and artifact writes still run for the whole project, so downstream effects of an
-edit, such as a changed column type, are reported exactly as in an uncached compile. The output is
+environment reads that rendering reported. Only the edited models are rendered again. Declaration
+files other than models are not read again, and an unchanged model reuses its stored column
+analysis only when the analyses of its upstream models in this compile match the ones it was stored
+with. Contracts, semantic validation, and rules still run for the whole project, and a model whose
+upstream analyses changed is analyzed again, so downstream effects of an edit, such as a changed
+column type, are reported exactly as in an uncached compile, even when another command such as
+`sqb build` ran in between. Compiled
+files that still hold the bytes the stored compile wrote are not read or rewritten. The output is
 byte-identical to `sqb compile --no-cache`.
 
 Every other change renders the whole project: adding, removing, or renaming a model, and editing
@@ -162,7 +167,7 @@ A replayed JSON report is byte-identical to the previous one except for `compile
 holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`,
 `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside
 the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or
-rendered again after model-only edits. In text mode, a replayed compile does not print the per-phase progress lines.
+rendered again after model-only edits and `analysis_reuse_hits` for stored model analyses reused. In text mode, a replayed compile does not print the per-phase progress lines.
 Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation`
 events are not emitted because no compile phase runs.
 

src/sqlbuild/cli/commands/_helpers/compile/pipeline.py

@@ -34,6 +34,7 @@
 from sqlbuild.cli.output.models import (
     WrittenTarget,
 )
+from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE
 from sqlbuild.compiler.compile.exceptions import CompileInputError
 from sqlbuild.compiler.compile.models import (
     CompileAnalysisSelection,
@@ -123,6 +124,7 @@ def _analyze_compile_project(
         cache_request=DiscoveryCacheRequest(
             selected_target=selected_target, no_cache=analysis_selection.no_cache
         ),
+        declaration_reuse=COMPILE_RENDER_REUSE.claim_discovery(),
     )
     discover_ms: int = elapsed_ms(discover_start)
     _ = complete_compile_phase(
@@ -327,7 +329,7 @@ def write_compile_dag_artifact(
         python_graph=python_graph,
     )
     _ = resolved_dag_path.write_text(dag_json, encoding="utf-8")
-    COMPILE_ARTIFACT_WRITES.written(
+    _ = COMPILE_ARTIFACT_WRITES.written(
         path=resolved_dag_path, contents=dag_json.replace("\n", os.linesep).encode("utf-8")
     )
     _ = complete_compile_phase(

src/sqlbuild/cli/commands/_helpers/compile/target_writer.py

@@ -672,7 +672,7 @@ def _write_sql(*, path: Path, sql: str, check_existing: bool = True) -> None:
 
 def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = True) -> None:
     with record_compile_timing("physical_write_ms"):
-        COMPILE_ARTIFACT_WRITES.written(
+        _ = COMPILE_ARTIFACT_WRITES.written(
             path=path,
             contents=contents.replace(_POSIX_LINE_SEPARATOR, os.linesep).encode("utf-8"),
         )
@@ -687,7 +687,9 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool =
 
 def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None:
     with record_compile_timing("physical_write_ms"):
-        COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents)
+        unchanged: bool = COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents)
+        if check_existing and unchanged:
+            return
         if check_existing:
             existing: bytes | None = _read_existing_file(path=path)
             if existing == contents:

src/sqlbuild/cli/commands/main/project/_compile.py

@@ -32,6 +32,7 @@
 from sqlbuild.cli.compile_render_reuse.main._render_reuse_session import render_reuse_session
 from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse
 from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput
+from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts
 from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES
 from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings
 from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile
@@ -77,7 +78,11 @@ def run_compile(
             with (
                 collect_compile_timings() as detailed_timings,
                 COMPILE_INPUT_READS.recording() as input_reads,
-                COMPILE_ARTIFACT_WRITES.recording() as artifact_writes,
+                COMPILE_ARTIFACT_WRITES.recording(
+                    stored=StoredArtifacts(
+                        stamps=attempt.prior_target_files, digests=attempt.prior_target_digests
+                    )
+                ) as artifact_writes,
                 activated_render_reuse(render_reuse=render_reuse),
                 PreparedCompileArtifacts(
                     enabled=not request.profile_flags.skip_write

src/sqlbuild/cli/compile_reuse/_helpers/attempt.py

@@ -106,6 +106,8 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt:
             attempt,
             prior_entry=True,
             prior_modules=header.inputs.modules,
+            prior_target_files=header.inputs.target_files,
+            prior_target_digests=header.inputs.target_digests,
             restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot),
             render_state_path=(
                 None

src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py

@@ -254,6 +254,7 @@ def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dic
             for path, item in inputs.project_files.items()
         },
         "target_files": {path: list(stamp) for path, stamp in inputs.target_files.items()},
+        "target_digests": inputs.target_digests,
         "stderr_lines": list(output.stderr_lines),
         "exit_code": output.exit_code,
         "timings_span": None if output.timings_span is None else list(output.timings_span),
@@ -309,6 +310,10 @@ def _header(*, payload: object) -> StoredCompileHeader:
                 _settings_inputs(item) for item in _list(fields["settings_inputs"])
             ),
             settings_digest=_string(fields["settings_digest"]),
+            target_digests={
+                _string(path): _string(digest)
+                for path, digest in _mapping(fields.get("target_digests", {})).items()
+            },
         ),
         output=StoredCompileOutput(
             stderr_lines=tuple(_string(line) for line in _list(fields["stderr_lines"])),

src/sqlbuild/cli/compile_reuse/_helpers/store.py

@@ -164,6 +164,11 @@ def _write_compile_entry(
             target_tree=artifacts_written,
             settings_inputs=settings.inputs,
             settings_digest=settings_inputs_digest(inputs=settings.inputs),
+            target_digests={
+                path: artifact.digest
+                for path, artifact in artifact_writes.artifacts.items()
+                if artifact.digest is not None and path in target_files
+            },
         ),
         output=StoredCompileOutput(
             stderr_lines=output.stderr_lines,

src/sqlbuild/cli/compile_reuse/_helpers/target_files.py

@@ -32,11 +32,11 @@ def snapshot_target_files(
             if entry.is_dir(follow_symlinks=False):
                 pending.append(entry.path)
                 continue
-            stamp: FileStamp | None = _path_stamp(path=entry.path)
+            stamp: FileStamp | None = path_stamp(path=entry.path)
             if stamp is not None:
                 stamps[entry.path] = stamp
     for path in extra_paths:
-        extra: FileStamp | None = _path_stamp(path=path)
+        extra: FileStamp | None = path_stamp(path=path)
         if extra is not None:
             stamps[path] = extra
     return stamps
@@ -81,7 +81,7 @@ def _target_stamps(*, project_dir: str, paths: tuple[str, ...], tree: bool) -> d
     """Stamp the whole compiled tree plus extra paths, or only the given paths."""
 
     if not tree:
-        return {path: stamp for path in paths if (stamp := _path_stamp(path=path)) is not None}
+        return {path: stamp for path in paths if (stamp := path_stamp(path=path)) is not None}
     compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep
     extra_paths: tuple[str, ...] = tuple(
         path for path in paths if not path.startswith(compiled_prefix)
@@ -103,7 +103,9 @@ def _compiled_root(*, project_dir: str) -> str:
     return os.path.join(project_dir, TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME)
 
 
-def _path_stamp(*, path: str) -> FileStamp | None:
+def path_stamp(*, path: str) -> FileStamp | None:
+    """Return the stat identity of one path, or None when it cannot be stated."""
+
     try:
         status: os.stat_result = os.lstat(path)
     except OSError:

src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py

@@ -7,10 +7,14 @@
 import threading
 from collections.abc import Iterator
 from contextlib import contextmanager
+from typing import TYPE_CHECKING
 
 from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites
 from sqlbuild.cli.compile_reuse.models import RecordedArtifact
 
+if TYPE_CHECKING:
+    from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts
+
 
 class CompileArtifactWriteRegistry:
     """Deliver artifact writes from every thread, including background staging threads."""
@@ -19,31 +23,41 @@ def __init__(self, *, digest_size: int) -> None:
         self._digest_size: int = digest_size
         self._lock: threading.Lock = threading.Lock()
         self._recorders: tuple[CompileArtifactWrites, ...] = ()
+        self._stored: StoredArtifacts | None = None
 
     @contextmanager
-    def recording(self) -> Iterator[CompileArtifactWrites]:
-        """Record artifact writes until the block exits."""
+    def recording(
+        self, *, stored: StoredArtifacts | None = None
+    ) -> Iterator[CompileArtifactWrites]:
+        """Record artifact writes until the block exits, trusting unchanged stored artifacts."""
 
         recorder: CompileArtifactWrites = CompileArtifactWrites()
         with self._lock:
             self._recorders = (*self._recorders, recorder)
+            if stored is not None:
+                self._stored = stored
         try:
             yield recorder
         finally:
             with self._lock:
                 self._recorders = tuple(item for item in self._recorders if item is not recorder)
+                if stored is not None:
+                    self._stored = None
 
-    def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None:
-        """Note the exact bytes an artifact holds after a write or an unchanged-file check."""
+    def written(self, *, path: str | os.PathLike[str], contents: bytes) -> bool:
+        """Note the exact bytes an artifact holds; return whether it already holds them."""
 
         recorders: tuple[CompileArtifactWrites, ...] = self._recorders
         if not recorders:
-            return
+            return False
+        absolute_path: str = os.path.abspath(path)
         artifact: RecordedArtifact = RecordedArtifact(
             digest=hashlib.blake2b(contents, digest_size=self._digest_size).hexdigest()
         )
         for recorder in recorders:
-            recorder.record(path=os.path.abspath(path), artifact=artifact)
+            recorder.record(path=absolute_path, artifact=artifact)
+        stored: StoredArtifacts | None = self._stored
+        return stored is not None and stored.holds(path=absolute_path, digest=artifact.digest)
 
     def kept(self, *, path: str | os.PathLike[str], size: int, mtime_ns: int) -> None:
         """Note an earlier artifact kept as-is after its size and mtime were validated."""

src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py

@@ -0,0 +1,29 @@
+"""Artifacts a stored compile left in target/, trusted while their stat identity is unchanged."""
+
+from __future__ import annotations
+
+from sqlbuild.cli.compile_reuse._helpers.target_files import path_stamp
+from sqlbuild.cli.compile_reuse.types import FileStamp
+
+
+class StoredArtifacts:
+    """Answer whether an artifact still holds the bytes a stored compile recorded for it."""
+
+    def __init__(self, *, stamps: dict[str, FileStamp], digests: dict[str, str]) -> None:
+        self._stamps: dict[str, FileStamp] = stamps
+        self._digests: dict[str, str] = digests
+
+    def holds(self, *, path: str, digest: str | None) -> bool:
+        """Return whether the file at path is unchanged since it was stored with this digest."""
+
+        return (
+            digest is not None
+            and self._digests.get(path) == digest
+            and self.unchanged_since_stored(path=path)
+        )
+
+    def unchanged_since_stored(self, *, path: str) -> bool:
+        """Return whether the file at path still has the stat identity it was stored with."""
+
+        stamp: FileStamp | None = self._stamps.get(path)
+        return stamp is not None and path_stamp(path=path) == stamp

src/sqlbuild/cli/compile_reuse/models.py

@@ -81,6 +81,7 @@ class StoredCompileInputs:
     target_tree: bool
     settings_inputs: tuple[SettingsEnvironmentInputs, ...]
     settings_digest: str
+    target_digests: dict[str, str] = field(default_factory=dict)
 
 
 @dataclass(frozen=True)
@@ -133,6 +134,8 @@ class CompileReuseAttempt:
     render_state_path: Path | None = None
     prior_entry: bool = False
     prior_modules: tuple[tuple[str, int, int], ...] = ()
+    prior_target_files: dict[str, FileStamp] = field(default_factory=dict)
+    prior_target_digests: dict[str, str] = field(default_factory=dict)
 
 
 @dataclass(frozen=True)

src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py

@@ -7,7 +7,11 @@
 
 from sqlbuild.adapter.contract.models import ExpressionInferenceProfile
 from sqlbuild.compiler.compile.classes.binding_dataflow import BindingDataflow
-from sqlbuild.compiler.compile.models import CompileModelInput, PolyglotAnalysisResult
+from sqlbuild.compiler.compile.models import (
+    CompileModelInput,
+    DataflowReuse,
+    PolyglotAnalysisResult,
+)
 from sqlbuild.compiler.compile.models import (
     ModelSqlAnalysis as _ModelSqlAnalysis,
 )
@@ -26,8 +30,7 @@ def analyze_binding_waves(
     *,
     requests: tuple[_ModelSqlAnalysisRequest, ...],
     names: tuple[str, ...],
-    cached: dict[str, PolyglotAnalysisResult],
-    previous_signatures: dict[str, str],
+    reuse: DataflowReuse,
     shapes: dict[str, dict[str, str]],
     types: dict[str, dict[str, str]],
     nullability: dict[str, dict[str, InferredNullability]],
@@ -40,8 +43,7 @@ def analyze_binding_waves(
     return BindingDataflow(
         requests=requests,
         names=names,
-        cached=cached,
-        previous_signatures=previous_signatures,
+        reuse=reuse,
         shapes=shapes,
         types=types,
         nullability=nullability,
@@ -55,8 +57,7 @@ def analyze_binding_dataflow(
     *,
     requests: tuple[_ModelSqlAnalysisRequest, ...],
     names: tuple[str, ...],
-    cached: dict[str, PolyglotAnalysisResult],
-    previous_signatures: dict[str, str],
+    reuse: DataflowReuse,
     shapes: dict[str, dict[str, str]],
     types: dict[str, dict[str, str]],
     nullability: dict[str, dict[str, InferredNullability]],
@@ -70,8 +71,7 @@ def analyze_binding_dataflow(
         BindingDataflow,
         requests=requests,
         names=names,
-        cached=cached,
-        previous_signatures=previous_signatures,
+        reuse=reuse,
         shapes=shapes,
         types=types,
         nullability=nullability,