test(native): regenerate goldens and the fallback allow-list from main with one command

Author: kvlonge

Makefile

@@ -8,7 +8,7 @@ SHELL := /bin/bash
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
 	compiler-differential compiler-differential-shipped compiler-differential-dense \
-	compiler-goldens
+	compiler-baselines
 
 format:
 	uv run ruff format .
@@ -295,8 +295,7 @@ COMPILER_DIFFERENTIAL_SHIPPED_SEEDS ?= 12
 COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
 COMPILER_DIFFERENTIAL_ARGS ?=
 # Native-to-Python fallbacks and analysis deferrals must match the allow-list exactly, and every
-# output its golden. Pass `--native-fallbacks update` in COMPILER_DIFFERENTIAL_ARGS to rewrite
-# the list (see scripts/compiler_differential/README.md).
+# output its golden. `make compiler-baselines` rewrites both (see scripts/compiler_differential/README.md).
 COMPILER_DIFFERENTIAL_GATES := --native-fallbacks check --goldens check
 
 compiler-differential:
@@ -312,13 +311,15 @@ compiler-differential-shipped:
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SHIPPED_SEEDS) \
 		$(COMPILER_DIFFERENTIAL_GATES) $(COMPILER_DIFFERENTIAL_ARGS)
 
-# Rewrite tests/goldens/compiler from the python oracle (from native once Python is deleted);
-# review the diff like code. See scripts/compiler_differential/README.md.
-compiler-goldens:
-	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
-		--engines python native --corpus fixtures examples seeds failures \
-		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
-		--goldens update $(COMPILER_DIFFERENTIAL_ARGS)
+# Rewrite the goldens (from the python oracle) and the fallback allow-list from both CI runs.
+# Review the diff like code; see scripts/compiler_differential/README.md.
+COMPILER_DIFFERENTIAL_CI_ARGS := --stage-captures --require-discovery-coverage --require-render-coverage
+compiler-baselines:
+	rm -rf tests/goldens/compiler
+	$(MAKE) compiler-differential COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
+		--require-analysis-coverage --native-fallbacks update --goldens update"
+	$(MAKE) compiler-differential-shipped COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
+		--native-fallbacks update --goldens update"
 
 compiler-differential-dense:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \

scripts/compiler_differential/README.md

@@ -15,7 +15,8 @@ does not add its own parity harness, property suite or E2E file.
 `native_fallbacks.toml` is the one allow-list of all work native code still hands back to Python.
 It covers both kinds of record:
 - shipped-stage fallbacks, from `report_native_fallback(site=NativeFallbackSite.<SITE>, kind=...)`;
-- preview analysis-stage deferrals, from `record_analysis_deferral` and `record_lineage_deferral`.
+- preview analysis-stage deferrals: every `analysis-deferrals-*.jsonl` record (analysis session,
+  lineage, semantic checks, contracts).
 
 Every entry names an engine, stage, site and kind, with its exact count per corpus. Use
 `max_counts` instead of `counts` only for a count that is not stable, and give the reason in a
@@ -29,28 +30,38 @@ sets. `--native-fallbacks check` runs in both make targets and fails CI in both
 
 The counts hold for the make targets' corpus (`--seeds 12`); other seed ranges are refused.
 
-**How a PR updates the list**
-
-1. A port deletes the Python code it replaces in the same PR. Its sites drop to zero and the gate
-   names the entries to remove; removing them is the port's test that the fallbacks are gone.
-   When the last entry for a `NativeFallbackSite` goes, delete the member and its
-   `report_native_fallback` call too.
-2. If the corpus does not reach a site yet, first add a failure case or seed feature that does.
-3. To rewrite the list from a run, use the same arguments as CI and add `--native-fallbacks
-   update` to both targets:
-
-   ```sh
-   make compiler-differential COMPILER_DIFFERENTIAL_ARGS="--stage-captures \
-     --require-discovery-coverage --require-render-coverage --require-analysis-coverage \
-     --native-fallbacks update"
-   make compiler-differential-shipped COMPILER_DIFFERENTIAL_ARGS="--stage-captures \
-     --require-discovery-coverage --require-render-coverage --native-fallbacks update"
-   ```
-
-   Review the diff. Any added entry or higher count needs a reason in the PR body; never add one
-   just to make CI pass.
-4. The gate is proven by sabotage in `test_native_fallback_gate.py`: forcing a native stage to
-   defer everything must fail it. A new "native did the work" gate needs the same proof.
+**How a PR updates the list and the goldens**
+
+One command rewrites both from the current tree:
+
+```sh
+make compiler-baselines
+```
+
+It deletes `tests/goldens/compiler`, then runs the two CI differentials with `--native-fallbacks
+update --goldens update`. Commit the resulting `native_fallbacks.toml` and `tests/goldens/` diff
+in the same PR as the change that caused it.
+
+- A port deletes the Python code it replaces in the same PR. Its entries then disappear from the
+  list, and that removal is the port's test that the fallbacks are gone. When the last entry for
+  a `NativeFallbackSite` goes, delete the member and its `report_native_fallback` call too.
+- If the corpus does not reach a site yet, first add a failure case or seed feature that does.
+- A PR that merges after a list or golden change on main reruns `make compiler-baselines` after
+  rebasing and commits the result.
+
+**What a reviewer checks in the diff**
+
+- `native_fallbacks.toml`:
+  - Removed entries and lower counts are expected from ports.
+  - Every added entry, higher count or new `max_counts` needs a stated reason in the PR body.
+    Never add one just to make CI pass.
+- Goldens:
+  - Every changed golden is an intended output change, named in the PR body.
+  - An output-neutral port changes no golden.
+  - New goldens belong to new corpus cases.
+- Proof by sabotage: `test_native_fallback_gate.py` proves the gate by sabotage, so forcing a
+  native stage to defer everything must fail it. A new "native did the work" gate needs the same
+  proof.
 
 ### Golden outputs
 
@@ -63,16 +74,17 @@ Project paths, the SQLBuild version, invocation ids and timings are masked.
 
 - `--goldens check` (in both make targets) compares every engine with the goldens, so an
   output-neutral port needs no new parity suite.
-- `make compiler-goldens` rewrites them from the python oracle. Run it only when output changes
-  on purpose, and review the diff like code.
+- `make compiler-baselines` rewrites them from the python oracle. Run it only when output
+  changes on purpose, and review the diff like code.
 - Seeds outside the recorded range have no golden and are skipped; any other project without a
   golden fails.
 - Once the Python stages are deleted, goldens are rewritten from `native`.
 
 ### All-engine user-facing errors
 
-`engine_error_cases()` in `_helpers/corpus/failure_cases.py` lists user-facing errors with their exact text. Each case
-is part of the failure corpus, so it runs in both differentials and has a golden. It also runs in
+`engine_error_cases()` in `_helpers/corpus/failure_cases.py` lists user-facing errors with their
+exact text. Each case is part of the failure corpus, so it runs in both differentials and has a
+golden. It also runs in
 `tests/e2e/scripts/compiler_differential/test_engine_error_cases.py` on the `python`, `native`
 and `native-preview` engines, which must all report its `expected_code` and `expected_message`.
 
@@ -87,4 +99,4 @@ failure_case(
 )
 ```
 
-Then it runs `make compiler-goldens` to record the new case's golden.
+Then it runs `make compiler-baselines` to record the new case's golden.

scripts/compiler_differential/constants.py

@@ -24,6 +24,8 @@
     "type_recovery.py": "semantic_checks",
     "recovery.py": "semantic_checks",
     "metadata_validation.py": "semantic_checks",
+    "contracts/columns.py": "contracts",
+    "contracts/promotion.py": "contracts",
 }
 NATIVE_FALLBACK_DEFERRAL_UNKNOWN_STAGE: str = "analysis"
 NATIVE_FALLBACK_LIST_HEADER: str = (
@@ -48,7 +50,7 @@
 )
 GOLDEN_PATH_MASK: str = "<project>"
 GOLDEN_VERSION_MASK: str = "<sqlbuild-version>"
-GOLDEN_MISSING_HINT: str = "no golden; record it with `make compiler-goldens`"
+GOLDEN_MISSING_HINT: str = "no golden; record it with `make compiler-baselines`"
 ENGINE_NAMES: tuple[str, ...] = ("python", "native", "native-preview")
 DEFAULT_ENGINES: tuple[str, str] = ("python", "native-preview")
 SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())"

scripts/compiler_differential/native_fallbacks.toml

@@ -79,14 +79,14 @@ engine = "native"
 stage = "model_config"
 site = "model_config.python_templates"
 kind = "config"
-counts = { failure = 98, seed = 36 }
+counts = { failure = 108, seed = 36 }
 
 [[entry]]
 engine = "native"
 stage = "model_config"
 site = "model_config.python_templates"
 kind = "connection"
-counts = { failure = 100, seed = 63 }
+counts = { failure = 110, seed = 63 }
 
 [[entry]]
 engine = "native"
@@ -114,7 +114,7 @@ engine = "native"
 stage = "model_loop"
 site = "model_loop.cursor_intrinsic_validation"
 kind = "deferred"
-counts = { failure = 366, seed = 1100 }
+counts = { failure = 387, seed = 1100 }
 
 [[entry]]
 engine = "native"
@@ -191,7 +191,7 @@ engine = "native-preview"
 stage = "model_analysis"
 site = "analysis_session"
 kind = "input_enrichment"
-counts = { example = 52, failure = 170, fixture = 52, seed = 614 }
+counts = { example = 52, failure = 182, fixture = 52, seed = 614 }
 
 [[entry]]
 engine = "native-preview"
@@ -226,14 +226,14 @@ engine = "native-preview"
 stage = "model_config"
 site = "model_config.python_templates"
 kind = "config"
-counts = { example = 12, failure = 98, fixture = 8, seed = 36 }
+counts = { example = 12, failure = 108, fixture = 8, seed = 36 }
 
 [[entry]]
 engine = "native-preview"
 stage = "model_config"
 site = "model_config.python_templates"
 kind = "connection"
-counts = { example = 18, failure = 100, fixture = 9, seed = 63 }
+counts = { example = 18, failure = 110, fixture = 9, seed = 63 }
 
 [[entry]]
 engine = "native-preview"
@@ -261,7 +261,7 @@ engine = "native-preview"
 stage = "model_loop"
 site = "model_loop.cursor_intrinsic_validation"
 kind = "deferred"
-counts = { example = 56, failure = 366, fixture = 64, seed = 1100 }
+counts = { example = 56, failure = 387, fixture = 64, seed = 1100 }
 
 [[entry]]
 engine = "native-preview"
@@ -283,10 +283,3 @@ stage = "model_loop"
 site = "model_loop.sql_variables"
 kind = "error"
 counts = { failure = 2 }
-
-[[entry]]
-engine = "native-preview"
-stage = "semantic_checks"
-site = "metadata_validation.py"
-kind = "metadata_checks"
-counts = { example = 12, failure = 95, fixture = 8, seed = 36 }