perf: accelerate warm incremental compiles

.github/workflows/verify.yml

@@ -281,6 +281,53 @@ jobs:
             make test-e2e-cold-compile-performance \
               SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
 
+  cache_compile_performance:
+    name: Fresh-process compile cache (${{ matrix.models }} models)
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.performance == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        models: [3000, 5000, 10000]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Run fresh-process compile cache guard under 4 GiB
+        env:
+          BENCHMARK_MODELS: ${{ matrix.models }}
+          MEMORY_LIMIT_BYTES: "4294967296"
+        run: |
+          sudo systemd-run \
+            --unit="sqlbuild-cache-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
+            --uid="$(id -u)" \
+            --gid="$(id -g)" \
+            --same-dir \
+            --setenv="HOME=${HOME}" \
+            --setenv="PATH=${PATH}" \
+            --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
+            --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
+            --property="MemorySwapMax=0" \
+            --wait \
+            --collect \
+            --pipe \
+            make test-e2e-cache-compile-performance \
+              SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
+
   rules_performance:
     name: Rules performance guards
     needs: [changes, lockfile]
@@ -373,6 +420,7 @@ jobs:
         e2e_shards,
         performance,
         cold_compile_performance,
+        cache_compile_performance,
         rules_performance,
         postgres,
       ]
@@ -386,6 +434,7 @@ jobs:
           E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
+          CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
         run: |
@@ -399,6 +448,7 @@ jobs:
             "${E2E_SHARDS_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \
+            "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then

Makefile

@@ -5,7 +5,7 @@ SHELL := /bin/bash
 	test-e2e-duckdb-build-incremental test-e2e-duckdb-build-virtual \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \
 	test-e2e-duckdb-integrations test-e2e-performance \
-	test-e2e-cold-compile-performance
+	test-e2e-cold-compile-performance test-e2e-cache-compile-performance
 
 format:
 	uv run ruff format .
@@ -224,7 +224,7 @@ test-e2e-performance:
 	@log="/tmp/opencode/test-e2e-performance-$$(date +%Y%m%d-%H%M%S).log"; \
 	echo "Logging to $$log"; \
 	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests/e2e \
-		-m "performance and not cold_compile_performance and not real_warehouse and not dbt" \
+		-m "performance and not cold_compile_performance and not cache_compile_performance and not real_warehouse and not dbt" \
 		-vv --log-cli-level=INFO --color=yes 2>&1 | tee "$$log"; \
 	status=$${PIPESTATUS[0]}; \
 	echo "TEST_E2E_PERFORMANCE_EXIT=$$status (log: $$log)" | tee -a "$$log"; \
@@ -238,6 +238,13 @@ test-e2e-cold-compile-performance:
 		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
 		-vv --log-cli-level=INFO --color=yes
 
+test-e2e-cache-compile-performance:
+	test -n "$(SQLBUILD_BENCHMARK_MODELS)"
+	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
+		tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py \
+		-m cache_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
+		-vv --log-cli-level=INFO --color=yes
+
 
 test-virtual:
 	@mkdir -p /tmp/opencode

docs/compiler-cache.md

@@ -0,0 +1,141 @@
+# Compiler cache architecture
+
+The compiler cache exists to avoid proving unchanged facts again. It must never turn an uncertain
+input into a cache hit, and a missing, stale, corrupt, oversized, or contended cache must behave like
+an ordinary uncached compile.
+
+This document defines the target architecture. The current vertical slice implements versioned
+model-analysis entries, dependency-aware compact analysis generations, shared SQL-test closure
+indexes, bounded compact-generation retention, corruption fallback, and analysis hit/miss telemetry.
+Project inventory, discovery/attachment reuse, and command snapshots remain later layers; their
+sections below are design constraints rather than claims of current implementation.
+
+## Goals
+
+- Make unchanged fresh-process compiles proportional to project inventory and requested output, not
+  total SQL complexity.
+- Make a small edit proportional to the changed resource and its semantic dependents.
+- Reuse downstream analysis when an upstream implementation changes without changing its exported
+  column interface.
+- Preserve diagnostics, bindings, types and nullability, lineage, tests, audits, hooks, macros,
+  ordering, artifacts, and exit codes exactly.
+- Keep cache data project-local, bounded, inspectable, non-executable, and safe under concurrent
+  compiler processes.
+
+## Non-goals
+
+- The cache is not a second source of project truth.
+- The cache does not make arbitrary Python execution deterministic. A stage with dynamic inputs that
+  are not represented in its identity must bypass persistent reuse.
+- The cache does not store pickles, code objects, callables, imported modules, adapter instances, or
+  open connections.
+- The cache does not replace warehouse execution state or change build selection semantics.
+
+## Layers
+
+### 1. Project inventory
+
+The inventory records every compiler-owned authored input as a normalized project-relative path,
+resource role, content digest, size, and filesystem identity metadata. Configuration, ignore files,
+adapter identity, compiler and SQL-analysis engine versions, selected target, CLI variables, and
+compile modes are explicit identity inputs.
+
+Filesystem metadata may identify an unchanged candidate, but reusable semantic facts remain
+content-addressed. Ambiguous or concurrently changing observations are misses. Added, removed,
+renamed, and role-changing paths are first-class changes.
+
+### 2. Static discovery facts
+
+Parsing results for SQL, YAML, CSV metadata, and statically inspected Python declarations are keyed
+by content digest, parser version, resource role, and parse options. These records contain only
+validated JSON-compatible values. Runtime Python discovery still imports current project code and
+is never reconstructed from executable cache data.
+
+### 3. Attached resource facts
+
+Attachment resolves configuration, declarations, macros, hooks, schemas, tests, and references.
+Each record carries the exact identities it observed:
+
+- authored resource content;
+- effective target, variables, and relevant configuration;
+- visible enum and constant declarations;
+- macro implementation and dependency closure;
+- referenced schema or hook declarations;
+- adapter rendering contract.
+
+If any observed identity is absent or unstable, that resource is attached normally. A global
+configuration change is broad only when the affected value was observed broadly.
+
+### 4. Semantic analysis facts
+
+Model SQL analysis remains keyed by expanded SQL, analysis profile, referenced relation interfaces,
+binding schema, and engine versions. The persistent representation should retain compact interned
+facts so warm reads do not reconstruct duplicate lineage objects entry by entry.
+
+Each model publishes a deterministic interface signature containing the output columns, types,
+nullability, wildcard status, and any other fact consumed by downstream analysis. A changed model
+invalidates downstream semantic analysis only when this exported interface changes. Its own compiled
+artifact still changes whenever its implementation changes.
+
+### 5. Shared derived indexes
+
+Project-wide indexes are built once per compile and shared by all consumers. They include model
+dependencies, test model closures, declaration visibility, selector indexes, and artifact identities.
+No per-test, per-rule, or per-artifact path may rebuild an unchanged whole-project index.
+
+### 6. Command snapshots
+
+A command snapshot records validated, non-executable output facts after all lower layers complete.
+It may provide a no-op fast path only when every input observed by that command is represented in
+the snapshot identity. Commands involving untracked runtime values bypass this layer while still
+using safe lower-layer facts.
+
+## Invalidation
+
+Invalidation follows recorded dependency edges rather than directory-wide guesses:
+
+1. Compare the current inventory with the last committed inventory.
+2. Invalidate directly changed static facts.
+3. Follow declaration, macro, configuration, schema, hook, and test dependency edges.
+4. Reattach directly affected resources.
+5. Reanalyze changed models and compare exported interface signatures.
+6. Reanalyze downstream models only when a consumed interface changed.
+7. Rebuild tests, audits, rules, and artifacts only when their recorded inputs changed.
+
+Removing an edge is itself an invalidating change. Cache readers must not infer validity from the
+absence of a newly introduced dependency record.
+
+## Storage and publication
+
+- Records are schema-versioned and include algorithm and package fingerprints.
+- Payloads have explicit size and collection-count limits before semantic acceptance.
+- Digests cover keys and canonical serialized payloads.
+- Writers publish in one transaction or by atomic replacement; readers observe one committed
+  generation.
+- Concurrent lock or I/O failure is a cache miss, never a compile failure.
+- Cache growth is bounded by generation and byte limits. Pruning removes old immutable records and
+  does not mutate current semantic state.
+- `--no-cache`, target configuration, and the compile-cache environment override bypass every
+  persistent layer and do not publish new records.
+
+## Observability and guards
+
+The target machine-readable contract reports per-layer hits, misses, invalidations, bytes read and
+written, and phase timings. The current vertical slice reports model-analysis batch hits, entry
+hits, misses, bypasses, and phase timings. Counts describe semantic records, not low-level database
+operations.
+
+Fresh-process performance guards use representative semantic projects and cover:
+
+- cold compile with a clean target;
+- unchanged warm compile;
+- leaf-model implementation edit;
+- upstream implementation edit with an unchanged interface;
+- interface-changing upstream edit;
+- test-only and macro edits;
+- broad project-configuration edit;
+- cache corruption and `--no-cache` fallback.
+
+Every path verifies exact resource counts, zero unexpected diagnostics, deterministic semantic
+fingerprints, bounded peak RSS, and expected hit and invalidation counts. Wall-time ceilings are
+calibrated only after the behavior and identity assertions are stable.

pyproject.toml

@@ -116,6 +116,7 @@ markers = [
   "motherduck: tests that require live MotherDuck credentials",
   "performance: opt-in performance guard tests",
   "cold_compile_performance: fresh-process semantic compile scaling guards",
+  "cache_compile_performance: fresh-process warm and incremental compile cache guards",
 ]
 
 [tool.ruff]

src/sqlbuild/cli/commands/_helpers/compile/target_writer.py

@@ -27,13 +27,13 @@
     compiled_sql_test_output_path,
 )
 from sqlbuild.cli.paths.main._sql_test_output_path import sql_test_output_path
-from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompiledSqlTest
+from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest
 from sqlbuild.compiler.compile.types import FunctionLanguage
 from sqlbuild.compiler.planner.main.execution.sql_test_artifacts import (
     plan_and_render_sql_test_artifacts,
 )
 from sqlbuild.compiler.planner.main.execution.sql_test_model_chain import (
-    sql_test_model_chain_names,
+    sql_test_model_chain_names_by_key,
 )
 from sqlbuild.compiler.planner.models import AuditPlanEntry, NativeSqlTestArtifact, PlanOutput
 from sqlbuild.compiler.profiling.main.record import record_compile_timing
@@ -332,7 +332,12 @@ def _write_static_tests(
         if project.compile_cache_dir is not None
         else None
     )
-    model_map: dict[str, CompiledModel] = {model.name: model for model in project.models}
+    model_chain_names_by_key: dict[CompiledObjectKey, tuple[str, ...]] = (
+        sql_test_model_chain_names_by_key(
+            project=project,
+            tests=project.sql_tests,
+        )
+    )
     pending: list[tuple[CompiledSqlTest, str | None, str | None]] = []
     for test in project.sql_tests:
         record_key: str | None = None
@@ -341,9 +346,7 @@ def _write_static_tests(
             record_key = sql_test_artifact_record_key(test=test)
             artifact_identity = sql_test_artifact_identity(
                 test=test,
-                model_chain_names=sql_test_model_chain_names(
-                    test=test, project=project, model_map=model_map
-                ),
+                model_chain_names=model_chain_names_by_key.get(test.key, ()),
                 context=identity_context,
             )
             cached_record: SqlTestArtifactCacheRecord | None = cached_records.get(record_key)