test: keep coded stderr in compiler goldens when --json prints an error document

plan --json now prints {"error": ...} on stdout when it fails. The golden builder read that document as a report with no diagnostics; it now records the coded stderr for it, so the goldens keep checking the error.

Author: kvlonge

scripts/compiler_differential/_helpers/goldens/goldens.py

@@ -32,6 +32,7 @@
     GOLDEN_SUFFIX,
     GOLDEN_VERSION_MASK,
     GOLDEN_WORK_MASK,
+    MACHINE_ERROR_KEY,
     PROJECT_DIRECTORY,
 )
 from scripts.compiler_differential.models import (
@@ -155,7 +156,7 @@ def _command(
         "exit_code": "timed out" if outcome.timed_out else outcome.exit_code,
         "diagnostics": (
             report.get("diagnostics", [])
-            if report is not None
+            if report is not None and MACHINE_ERROR_KEY not in report
             else _lines(
                 text=normalize_stderr(_masked(text=outcome.stderr, masked_paths=masked_paths))
             )

scripts/compiler_differential/constants.py

@@ -23,6 +23,7 @@
 GOLDEN_LABEL: str = "golden"
 GOLDEN_FULL_REPORT_LABEL_PREFIX: str = "refactor-"
 GOLDEN_REPORT_KEY: str = "report"
+MACHINE_ERROR_KEY: str = "error"
 GOLDEN_MANIFEST_DROPPED_KEYS: frozenset[str] = frozenset({"metadata", "child_map"})
 GOLDEN_MANIFEST_RESOURCE_SECTIONS: tuple[str, ...] = ("nodes", "sources", "macros")
 GOLDEN_RESOURCE_DROPPED_FIELDS: frozenset[str] = frozenset(

tests/unit/scripts/compiler_differential/_helpers/goldens/helpers.py

@@ -42,3 +42,21 @@ def project_run(*, message: str, compiled_sql: str) -> ProjectRun:
             }
         ),
     )
+
+
+def machine_error_run() -> ProjectRun:
+    """A `plan --json` run that failed with the machine error document and a coded stderr."""
+
+    error: dict[str, object] = {"error": {"code": "C214", "message": "no manifest", "help": None}}
+    return ProjectRun(
+        outcomes=(
+            CommandOutcome(
+                label="plan",
+                exit_code=1,
+                stdout=json.dumps(error),
+                stderr="error[C214]: no manifest\n",
+            ),
+        ),
+        compiled={},
+        manifest=None,
+    )

tests/unit/scripts/compiler_differential/_helpers/goldens/test_compiler_goldens.py

@@ -20,6 +20,7 @@
 )
 from tests.unit.scripts.compiler_differential._helpers.goldens.helpers import (
     PROJECT_PATH,
+    machine_error_run,
     project_run,
 )
 
@@ -98,7 +99,23 @@ def test_given_corpus_project_when_locating_its_golden_then_only_golden_corpora_
                     "parent_map": {"model.orders.customer_totals": []},
                 },
             },
-        )
+        ),
+        GoldenPayloadTestCase(
+            description="machine_error_document_keeps_the_coded_stderr",
+            run=machine_error_run(),
+            masked_paths=(),
+            expected_payload={
+                "commands": [
+                    {
+                        "label": "plan",
+                        "exit_code": 1,
+                        "diagnostics": ["error[C214]: no manifest", ""],
+                    }
+                ],
+                "compiled": {},
+                "manifest": None,
+            },
+        ),
     ],
     ids=lambda case: case.description,
 )