test(native): count remaining native-to-Python fallbacks and add golden compile outputs

.github/workflows/verify.yml

@@ -96,7 +96,7 @@ jobs:
                 ;;
             esac
             case "${path}" in
-              scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|.github/workflows/verify.yml|Makefile)
+              scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|src/sqlbuild/cli/commands/_helpers/compile/*|.github/workflows/verify.yml|Makefile)
                 compiler_differential=true
                 ;;
             esac
@@ -317,7 +317,7 @@ jobs:
           path: ${{ runner.temp }}/compiler-differential-wheel-sites.json
           if-no-files-found: warn
           retention-days: 30
-      - name: Compare generated seeds and the failure corpus (python vs shipped native)
+      - name: Compare fixtures, examples, generated seeds and the failure corpus (python vs shipped native)
         run: >-
           make compiler-differential-shipped
           COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage

Makefile

@@ -7,7 +7,8 @@ SHELL := /bin/bash
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
-	compiler-differential compiler-differential-shipped compiler-differential-dense
+	compiler-differential compiler-differential-shipped compiler-differential-dense \
+	compiler-baselines
 
 format:
 	uv run ruff format .
@@ -293,19 +294,32 @@ COMPILER_DIFFERENTIAL_SEEDS ?= 12
 COMPILER_DIFFERENTIAL_SHIPPED_SEEDS ?= 12
 COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000
 COMPILER_DIFFERENTIAL_ARGS ?=
+# Native-to-Python fallbacks and analysis deferrals must match the allow-list exactly, and every
+# output its golden. `make compiler-baselines` rewrites both (see scripts/compiler_differential/README.md).
+COMPILER_DIFFERENTIAL_GATES := --native-fallbacks check --goldens check
 
 compiler-differential:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \
-		$(COMPILER_DIFFERENTIAL_ARGS)
+		$(COMPILER_DIFFERENTIAL_GATES) $(COMPILER_DIFFERENTIAL_ARGS)
 
 # python vs native-preview above covers every native stage; this keeps the shipped `native` default
-# (native discovery and rendering) covered on a bounded corpus.
+# (native discovery and rendering) and its goldens covered on the same corpus.
 compiler-differential-shipped:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \
-		--engines python native --corpus seeds failures \
+		--engines python native --corpus fixtures examples seeds failures \
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SHIPPED_SEEDS) \
-		$(COMPILER_DIFFERENTIAL_ARGS)
+		$(COMPILER_DIFFERENTIAL_GATES) $(COMPILER_DIFFERENTIAL_ARGS)
+
+# Rewrite the goldens (from the python oracle) and the fallback allow-list from both CI runs.
+# Review the diff like code; see scripts/compiler_differential/README.md.
+COMPILER_DIFFERENTIAL_CI_ARGS := --stage-captures --require-discovery-coverage --require-render-coverage
+compiler-baselines:
+	rm -rf tests/goldens/compiler
+	$(MAKE) compiler-differential COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
+		--require-analysis-coverage --native-fallbacks update --goldens update"
+	$(MAKE) compiler-differential-shipped COMPILER_DIFFERENTIAL_ARGS="$(COMPILER_DIFFERENTIAL_CI_ARGS) \
+		--native-fallbacks update --goldens update"
 
 compiler-differential-dense:
 	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \

scripts/compiler_differential/README.md

@@ -0,0 +1,147 @@
+# Compiler differential
+
+`make compiler-differential` compiles fixtures, examples, generated seeds and the failure corpus
+under two compiler engines and fails on the first differing artifact. `make
+compiler-differential-shipped` does the same for the shipped `native` default. Both also enforce
+the shared gates below.
+
+For ad hoc runs (one project, other seeds, other engines), call the script directly, for example
+`uv run python -m scripts.run_compiler_differential --corpus --project <dir> --engines python
+native`. The make targets fix the corpus the allow-list and goldens are recorded for.
+
+## Shared gates for native ports
+
+A port that moves work from the Python compiler to native code reuses these three mechanisms. It
+does not add its own parity harness, property suite or E2E file.
+
+### Native-to-Python fallbacks and analysis deferrals
+
+`native_fallbacks.toml` is the one allow-list of all work native code still hands back to Python,
+and of the work each native stage answers itself. It covers three kinds of record:
+- fallbacks, from `report_native_fallback(site=NativeFallbackSite.<SITE>, kind=...)` on every
+  path where a native stage hands work to Python, or runs Python with no native path yet;
+- preview analysis-stage deferrals: every `analysis-deferrals-*.jsonl` record (analysis session,
+  lineage, semantic checks, contracts);
+- native answers, from `report_native_answer(stage=NativeStage.<STAGE>, kind=...)` where a
+  stage's native path produced the result: files, models, scans or calls. Their site is
+  `<stage>.native`.
+
+Counting only fallbacks cannot tell a stage that answers natively from one that no longer runs
+natively at all. The answer counts can: if a stage is switched to Python, its `<stage>.native`
+entries vanish and the gate says so.
+
+Every entry names an engine, stage, site and kind, with its exact count per corpus. Use
+`max_counts` instead of `counts` only for a count that is not stable, and give the reason in a
+comment and in the PR body.
+
+The records cost nothing unless `SQLBUILD_ANALYSIS_RECORD_DIR` is set, which only the harness
+sets. `--native-fallbacks check` runs in both make targets and fails CI in both directions:
+- an entry, engine or corpus count that is not on the list fails;
+- a listed entry whose count changed or that no longer occurs also fails, and the message says to
+  update or remove it, so the fallback list only shrinks;
+- a vanished native answer fails as work that now runs in Python;
+- a listed fallback that vanished is refused outright, with "fallback disappeared but native
+  answers did not appear; the stage may be switched off", unless the same engine, stage and
+  corpus report more `<stage>.native` answers than the list records. A stage with only fallback
+  entries (`sql_test_glue`, `project_assembly`) would otherwise look like a finished port when it
+  is switched off.
+
+The counts hold for the make targets' corpus (`--seeds 12`); other seed ranges are refused.
+
+Two stages the list cannot watch:
+- `type_system`: the compile corpus never reaches type normalization. Its callers are the planner
+  and executor and the preview stages' Python fallbacks. `tests/integration/.../type_system/
+  test_native_type_parity.py` proves instead that the public `normalize_type` asks native first
+  under `native-preview`.
+- `discovery.native model_file_listings`: it is reported unconditionally inside the native model
+  file discovery, which has no fallback to Python, so this count cannot detect anything. It stays
+  only as a record that native discovery ran.
+
+**How a PR updates the list and the goldens**
+
+One command rewrites both from the current tree:
+
+```sh
+make compiler-baselines
+```
+
+It deletes `tests/goldens/compiler`, then runs the two CI differentials with `--native-fallbacks
+update --goldens update`. Commit the resulting `native_fallbacks.toml` and `tests/goldens/` diff
+in the same PR as the change that caused it.
+
+- A port deletes the Python code it replaces in the same PR. Its fallback entries then disappear
+  from the list, and that removal is the port's test that the fallbacks are gone. Its answer
+  counts usually rise. When the last entry for a `NativeFallbackSite` goes, delete the member
+  and its `report_native_fallback` call too.
+- If the corpus does not reach a site yet, first add a failure case or seed feature that does.
+- A PR that merges after a list or golden change on main reruns `make compiler-baselines` after
+  rebasing and commits the result.
+
+**What a reviewer checks in the diff**
+
+- `native_fallbacks.toml`:
+  - Removed fallback entries and lower fallback counts are expected from ports.
+  - Every added fallback entry, higher fallback count or new `max_counts` needs a stated reason
+    in the PR body. Never add one just to make CI pass.
+  - A removed or lower `<stage>.native` answer count means less native work. It needs a reason
+    in the PR body.
+  - A removed fallback entry needs new or higher `<stage>.native` answers for the same stage in the
+    same diff: the port must report the work it now answers. A fallback that vanished with no new
+    answers is a stage switched off, not a port; the gate refuses it.
+- Code: every new path that hands work to Python calls `report_native_fallback`, and every new
+  native path calls `report_native_answer` where its result is used.
+- Goldens:
+  - Every changed golden is an intended output change, named in the PR body.
+  - An output-neutral port changes no golden.
+  - New goldens belong to new corpus cases.
+- Proof by sabotage: `test_native_fallback_gate.py` proves the gate by sabotage, so forcing a
+  native stage to defer everything must fail it. A new "native did the work" gate needs the same
+  proof.
+
+### Golden outputs
+
+`tests/goldens/compiler/<corpus>/<project>.json` holds one project's compile outputs:
+- each command's exit code and diagnostics;
+- the compiled SQL, one array item per line;
+- the manifest without its volatile metadata and the SQL it repeats.
+
+The project directory is masked as `<project>` and the harness work directory as `<work>`,
+whether or not the path is resolved. The SQLBuild version, invocation ids and timings are also
+masked.
+
+Goldens hold outputs only, not stage captures: a seed's captures are about 690 KB, too large to
+review. Cache facts are covered by the incremental-versus-no-cache tests instead.
+
+- `--goldens check` (in both make targets) compares every engine with the goldens, so an
+  output-neutral port needs no new parity suite.
+- `make compiler-baselines` rewrites them from the python oracle. Run it only when output
+  changes on purpose, and review the diff like code.
+- `tests/goldens/compiler/seed_range.toml` records the seed range the goldens were written for.
+  Every seed in that range, and every fixture, example and failure case, must have a golden.
+  Seeds outside the range (ad hoc runs with more seeds) are skipped. A missing `seed_range.toml`
+  fails the check for every seed project rather than skipping them.
+- Dialect variants (`<seed>-postgres`, `<seed>-snowflake`) are generated for the first seed of a
+  run only, so a golden is required only for the variants of the recorded `seed_start`. An ad hoc
+  `--seed-start N` run checks its base seeds in the range and skips its variants.
+- Once the Python stages are deleted, goldens are rewritten from `native`.
+
+### All-engine user-facing errors
+
+`engine_error_cases()` in `_helpers/corpus/failure_cases.py` lists user-facing errors with their
+exact text. Each case is part of the failure corpus, so it runs in both differentials and has a
+golden. It also runs in
+`tests/e2e/scripts/compiler_differential/test_engine_error_cases.py` on the `python`, `native`
+and `native-preview` engines, which must all report its `expected_code` and `expected_message`.
+
+A port that adds or changes a user-facing error adds one `failure_case(...)` to it:
+
+```python
+failure_case(
+    name="engine-error-<what-goes-wrong>",
+    expected_code="P001",
+    files=mart_body_files("SELECT ...\n"),
+    expected_message="<the exact message, with <project> for the project path>",
+)
+```
+
+Then it runs `make compiler-baselines` to record the new case's golden.

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -13,6 +13,10 @@
 from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
 from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
+from scripts.compiler_differential._helpers.goldens.goldens import (
+    golden_differences,
+    golden_path_masks,
+)
 from scripts.compiler_differential._helpers.running.execution import run_engine
 from scripts.compiler_differential.constants import (
     ERROR_SEVERITY,
@@ -64,6 +68,17 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         *_timeout_differences(project=project, runs=(left, right)),
         *_corpus_differences(project=project, run=left),
         *compare_engine_runs(project=project.name, left=left, right=right),
+        *(
+            golden_differences(
+                project=project,
+                runs=(left, right),
+                golden_dir=options.golden_dir,
+                mode=options.golden_mode,
+                masked_paths=golden_path_masks(case_dir=case_dir, work_dir=options.work_dir),
+            )
+            if options.golden_mode is not None and options.golden_dir is not None
+            else ()
+        ),
     ]
     return ProjectComparison(
         project=project.name,

scripts/compiler_differential/_helpers/corpus/emitted_codes.py

@@ -12,9 +12,11 @@
     first_diagnostic_message,
 )
 from scripts.compiler_differential._helpers.corpus.corpus import build_corpus
+from scripts.compiler_differential._helpers.goldens.goldens import golden_path_masks, masked_text
 from scripts.compiler_differential._helpers.running.execution import run_engine
 from scripts.compiler_differential.constants import (
     ANALYSIS_CODE_SCAN_ROOTS,
+    COLD_COMPILE,
     CORPUS_FAILURES,
     DIAGNOSTIC_CODE_PATTERN,
     ERROR_SEVERITY,
@@ -29,6 +31,8 @@
     DifferentialOptions,
     EmittedCodes,
     EngineRun,
+    ExpectedOutcome,
+    FailureCase,
 )
 from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE
 from sqlbuild.compiler.compile import constants as compile_constants
@@ -60,6 +64,20 @@ def emitted_failure_codes(*, options: DifferentialOptions) -> dict[str, EmittedC
     return {project.name: codes for project, codes in zip(corpus, emitted, strict=True)}
 
 
+def case_emitted_codes(*, case: FailureCase, options: DifferentialOptions) -> EmittedCodes:
+    """Compile one failure case under the first engine and return what it really emitted."""
+
+    return _emitted_codes(
+        project=CorpusProject(
+            name=f"{CORPUS_FAILURES}/{case.name}",
+            commands=(COLD_COMPILE,),
+            expected=ExpectedOutcome(error_code=case.expected_code),
+            writer=case.write,
+        ),
+        options=options,
+    )
+
+
 def discovery_error_codes() -> frozenset[str]:
     """Return the code of every `DiscoveryError` subclass, walking the whole hierarchy."""
 
@@ -182,14 +200,20 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E
         options=options,
     )
     first: dict[str, object] = first_diagnostic(outcome=run.outcomes[0], severity=ERROR_SEVERITY)
+    message: str | None = first_diagnostic_message(outcome=run.outcomes[0], severity=ERROR_SEVERITY)
     notes: object = first.get("notes")
     line: object = first.get("line")
     column: object = first.get("column")
     return EmittedCodes(
         errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY),
         warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY),
-        first_error_message=first_diagnostic_message(
-            outcome=run.outcomes[0], severity=ERROR_SEVERITY
+        first_error_message=(
+            None
+            if message is None
+            else masked_text(
+                text=message,
+                masks=golden_path_masks(case_dir=case_dir, work_dir=options.work_dir),
+            )
         ),
         first_error_help=None if first.get("help") is None else str(first.get("help")),
         first_error_notes=tuple(str(note) for note in notes) if isinstance(notes, list) else (),

scripts/compiler_differential/_helpers/corpus/failure_cases.py

@@ -36,6 +36,8 @@
 )
 from scripts.compiler_differential.models import FailureCase
 
+_MART_HEADER_START: str = 'MODEL (\n  description "Order totals per customer",\n'
+
 _RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n'
 
 
@@ -57,6 +59,7 @@ def all_failure_cases() -> tuple[FailureCase, ...]:
         *reference_failure_cases(),
         *attachment_failure_cases(),
         *analysis_failure_cases(),
+        *engine_error_cases(),
     )
 
 
@@ -591,3 +594,51 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]:
             ),
         ),
     )
+
+
+def engine_error_cases() -> tuple[FailureCase, ...]:
+    """User-facing errors every engine must report with the exact text; see the README."""
+
+    return (
+        failure_case(
+            name="engine-error-unset-environment-variable",
+            expected_code="P001",
+            files=mart_body_files(
+                "SELECT customer_id, '@@ENV:SQB_ENGINE_ERROR_UNSET_REGION' AS region\n"
+                'FROM __ref("stg_orders")\n'
+            ),
+            expected_message=(
+                "unknown environment variable '@@ENV:SQB_ENGINE_ERROR_UNSET_REGION' in "
+                f"'<project>/{FAILURE_MART_PATH}'"
+            ),
+        ),
+        failure_case(
+            name="engine-error-case-folded-duplicate-column",
+            expected_code="P001",
+            files={
+                FAILURE_MART_PATH: _MART_HEADER_START
+                + '  columns (\n    Größe (description "Size"),\n'
+                + '    größe (description "Size again"),\n  ),\n);\n\n'
+                + 'SELECT customer_id AS größe\nFROM __ref("stg_orders")\n'
+            },
+            expected_message=(
+                f"{FAILURE_MART_PATH} model has duplicate column 'größe' "
+                "(column names are case-insensitive)"
+            ),
+        ),
+        failure_case(
+            name="engine-error-week-date-cursor-start",
+            expected_code="P001",
+            files={
+                FAILURE_MART_PATH: _MART_HEADER_START
+                + "  materialized incremental,\n  incremental_strategy append,\n"
+                + "  cursor order_ts,\n  cursor_type timestamp,\n  cursor_grain day,\n"
+                + '  cursor_start "2024-W01-1T25:00",\n);\n\n'
+                + 'SELECT customer_id, CURRENT_TIMESTAMP AS order_ts\nFROM __ref("stg_orders")\n'
+            },
+            expected_message=(
+                "model 'customer_totals': cursor_start value '2024-W01-1T25:00' is not a valid "
+                "ISO timestamp: hour must be in 0..23"
+            ),
+        ),
+    )

scripts/compiler_differential/_helpers/goldens/__init__.py

@@ -0,0 +1 @@
+"""Golden compile outputs for the shipped native stages."""