fix(compiler): keep shared-analysis lineage and stage captures deterministic

scripts/compiler_differential/_helpers/comparing/capture.py

@@ -0,0 +1,53 @@
+"""Compare two stage captures on disk without loading either one whole."""
+
+from __future__ import annotations
+
+import itertools
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+from scripts.compiler_differential._helpers.comparing.compare import (
+    first_json_difference,
+    preview,
+)
+from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text
+from scripts.compiler_differential.classes.capture_file import CaptureFile
+from scripts.compiler_differential.constants import MISSING_VALUE
+from scripts.compiler_differential.models import Divergence
+
+
+def first_capture_difference(*, left: Path, right: Path) -> Divergence | None:
+    """Return the first logical JSON pointer, or line for invalid JSON, where captures differ."""
+
+    try:
+        left_capture: CaptureFile = CaptureFile(left)
+        right_capture: CaptureFile = CaptureFile(right)
+        return first_json_difference(
+            left=left_capture.root,
+            right=right_capture.root,
+            resolve=(left_capture.resolve, right_capture.resolve),
+        )
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        return _first_line_difference(left=left, right=right)
+
+
+def _first_line_difference(*, left: Path, right: Path) -> Divergence | None:
+    with left.open("rb") as left_stream, right.open("rb") as right_stream:
+        lines: Iterator[tuple[bytes | None, bytes | None]] = itertools.zip_longest(
+            left_stream, right_stream
+        )
+        for number, (left_line, right_line) in enumerate(lines, start=1):
+            left_text: str = _line_text(left_line)
+            right_text: str = _line_text(right_line)
+            if left_text != right_text:
+                return Divergence(
+                    location=f"line {number}", left=preview(left_text), right=preview(right_text)
+                )
+    return None
+
+
+def _line_text(line: bytes | None) -> str:
+    if line is None:
+        return MISSING_VALUE
+    return normalize_artifact_text(line.decode("utf-8", "surrogateescape").rstrip("\n"))

scripts/compiler_differential/_helpers/comparing/compare.py

@@ -3,34 +3,50 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Callable
 from typing import cast
 
 from scripts.compiler_differential.constants import MISSING_VALUE, VALUE_PREVIEW_CHARACTERS
 from scripts.compiler_differential.models import Divergence
+from scripts.compiler_differential.types import NodeResolvers
+from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SHARED_MARKER
 
 
-def first_json_difference(*, left: object, right: object, pointer: str = "") -> Divergence | None:
-    """Return the first JSON pointer whose value, type, or object key order differs."""
+def first_json_difference(
+    *, left: object, right: object, pointer: str = "", resolve: NodeResolvers | None = None
+) -> Divergence | None:
+    """Return the first JSON pointer whose value, type, or key order differs, via `resolve`."""
 
+    if resolve is not None:
+        left_digest: str | None = shared_digest(left)
+        if left_digest is not None and left_digest == shared_digest(right):
+            return None
+        left, right = resolve[0](left), resolve[1](right)
     if type(left) is not type(right):
         return _divergence(location=pointer or "/", left=left, right=right)
     left_object: dict[str, object] | None = as_json_object(left)
     right_object: dict[str, object] | None = as_json_object(right)
     if left_object is not None and right_object is not None:
-        return _first_object_difference(left=left_object, right=right_object, pointer=pointer)
+        return _first_object_difference(
+            left=left_object, right=right_object, pointer=pointer, resolve=resolve
+        )
     if isinstance(left, list) and isinstance(right, list):
         for index, (left_item, right_item) in enumerate(zip(left, right, strict=False)):
             found: Divergence | None = first_json_difference(
-                left=left_item, right=right_item, pointer=f"{pointer}/{index}"
+                left=left_item, right=right_item, pointer=f"{pointer}/{index}", resolve=resolve
             )
             if found is not None:
                 return found
         if len(left) != len(right):
             index: int = min(len(left), len(right))
             return _divergence(
                 location=f"{pointer}/{index}",
-                left=left[index] if index < len(left) else MISSING_VALUE,
-                right=right[index] if index < len(right) else MISSING_VALUE,
+                left=_shown(value=left[index], resolve=resolve, side=0)
+                if index < len(left)
+                else MISSING_VALUE,
+                right=_shown(value=right[index], resolve=resolve, side=1)
+                if index < len(right)
+                else MISSING_VALUE,
             )
         return None
     if left != right:
@@ -50,20 +66,19 @@ def first_text_difference(*, left: str, right: str) -> Divergence | None:
         right_line: str = right_lines[index] if index < len(right_lines) else MISSING_VALUE
         if left_line != right_line:
             return Divergence(
-                location=f"line {index + 1}", left=_preview(left_line), right=_preview(right_line)
+                location=f"line {index + 1}", left=preview(left_line), right=preview(right_line)
             )
     return Divergence(location="end of file", left="<bytes differ>", right="<bytes differ>")
 
 
-def first_document_difference(*, left: str, right: str) -> Divergence | None:
-    """Compare two documents as JSON when both parse, otherwise line by line."""
+def shared_digest(value: object) -> str | None:
+    """Return the digest a stage-capture `{"__shared__": digest}` reference names."""
 
-    try:
-        left_payload: object = json.loads(left)
-        right_payload: object = json.loads(right)
-    except json.JSONDecodeError:
-        return first_text_difference(left=left, right=right)
-    return first_json_difference(left=left_payload, right=right_payload)
+    mapping: dict[str, object] | None = as_json_object(value)
+    if mapping is None or len(mapping) != 1:
+        return None
+    digest: object = mapping.get(STAGE_CAPTURE_SHARED_MARKER)
+    return digest if isinstance(digest, str) else None
 
 
 def as_json_object(value: object) -> dict[str, object] | None:
@@ -79,35 +94,70 @@ def json_pointer_token(key: str) -> str:
 
 
 def _first_object_difference(
-    *, left: dict[str, object], right: dict[str, object], pointer: str
+    *,
+    left: dict[str, object],
+    right: dict[str, object],
+    pointer: str,
+    resolve: NodeResolvers | None,
 ) -> Divergence | None:
     for key in (*left, *(key for key in right if key not in left)):
         location: str = f"{pointer}/{json_pointer_token(key)}"
         if key not in left or key not in right:
             return _divergence(
                 location=location,
-                left=left.get(key, MISSING_VALUE),
-                right=right.get(key, MISSING_VALUE),
+                left=_shown(value=left.get(key, MISSING_VALUE), resolve=resolve, side=0),
+                right=_shown(value=right.get(key, MISSING_VALUE), resolve=resolve, side=1),
             )
         found: Divergence | None = first_json_difference(
-            left=left[key], right=right[key], pointer=location
+            left=left[key], right=right[key], pointer=location, resolve=resolve
         )
         if found is not None:
             return found
     if list(left) != list(right):
         return Divergence(
             location=f"{pointer or '/'} (key order)",
-            left=_preview(list(left)),
-            right=_preview(list(right)),
+            left=preview(list(left)),
+            right=preview(list(right)),
         )
     return None
 
 
+def _shown(*, value: object, resolve: NodeResolvers | None, side: int) -> object:
+    if resolve is None:
+        return value
+    return _previewed(value=value, resolve=resolve[side], remaining=VALUE_PREVIEW_CHARACTERS)[0]
+
+
+def _previewed(
+    *, value: object, resolve: Callable[[object], object], remaining: int
+) -> tuple[object, int]:
+    if remaining <= 0:
+        return value, remaining
+    resolved: object = resolve(value)
+    mapping: dict[str, object] | None = as_json_object(resolved)
+    if mapping is not None:
+        shown: dict[str, object] = {}
+        for key, item in mapping.items():
+            shown[key], remaining = _previewed(
+                value=item, resolve=resolve, remaining=remaining - len(key)
+            )
+        return shown, remaining
+    if isinstance(resolved, list):
+        items: list[object] = []
+        for item in resolved:
+            shown_item, remaining = _previewed(value=item, resolve=resolve, remaining=remaining)
+            items.append(shown_item)
+        return items, remaining
+    return resolved, remaining - len(json.dumps(resolved, ensure_ascii=False))
+
+
 def _divergence(*, location: str, left: object, right: object) -> Divergence:
-    return Divergence(location=location, left=_preview(left), right=_preview(right))
+    return Divergence(location=location, left=preview(left), right=preview(right))
+
 
+def preview(value: object) -> str:
+    """Render a value for a report, shortened to the preview width."""
 
-def _preview(value: object) -> str:
     rendered: str = (
         value
         if isinstance(value, str) and value == MISSING_VALUE

scripts/compiler_differential/_helpers/comparing/comparison.py

@@ -4,10 +4,11 @@
 
 import json
 from collections.abc import Callable
+from pathlib import Path
 
+from scripts.compiler_differential._helpers.comparing.capture import first_capture_difference
 from scripts.compiler_differential._helpers.comparing.compare import (
     as_json_object,
-    first_document_difference,
     first_json_difference,
     first_text_difference,
 )
@@ -237,12 +238,12 @@ def _loads(text: str | None) -> object:
 
 
 def _capture_differences(
-    *, left: dict[str, dict[str, str]], right: dict[str, dict[str, str]]
+    *, left: dict[str, dict[str, Path]], right: dict[str, dict[str, Path]]
 ) -> list[tuple[str, Divergence | None]]:
     differences: list[tuple[str, Divergence | None]] = []
     for command in sorted(set(left) | set(right)):
-        left_files: dict[str, str] = left.get(command, {})
-        right_files: dict[str, str] = right.get(command, {})
+        left_files: dict[str, Path] = left.get(command, {})
+        right_files: dict[str, Path] = right.get(command, {})
         for name in sorted(set(left_files) | set(right_files)):
             artifact: str = f"stage capture {command}/{name}"
             if name not in left_files or name not in right_files:
@@ -258,10 +259,7 @@ def _capture_differences(
             differences.append(
                 (
                     artifact,
-                    first_document_difference(
-                        left=normalize_artifact_text(left_files[name]),
-                        right=normalize_artifact_text(right_files[name]),
-                    ),
+                    first_capture_difference(left=left_files[name], right=right_files[name]),
                 )
             )
     return differences

scripts/compiler_differential/_helpers/comparing/normalize.py

@@ -6,18 +6,16 @@
 from scripts.compiler_differential.constants import (
     ELAPSED_TIME_MASK,
     ELAPSED_TIME_PATTERN,
-    ENGINE_NAMESPACE_PATTERN,
-    INVOCATION_ID_MASK,
-    INVOCATION_ID_PATTERN,
     MANIFEST_VOLATILE_METADATA,
     STRIPPED_REPORT_FIELDS,
 )
+from sqlbuild.compiler.frontier.classes.stage_capture_encoder import mask_capture_noise
 
 
 def normalize_artifact_text(text: str) -> str:
     """Mask invocation ids and engine store suffixes, which differ by design."""
 
-    return ENGINE_NAMESPACE_PATTERN.sub("", INVOCATION_ID_PATTERN.sub(INVOCATION_ID_MASK, text))
+    return mask_capture_noise(text)
 
 
 def normalize_stderr(text: str) -> str:

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -13,7 +13,12 @@
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
 from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
 from scripts.compiler_differential._helpers.running.execution import run_engine
-from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
+from scripts.compiler_differential.constants import (
+    ERROR_SEVERITY,
+    LEFT_SIDE,
+    RIGHT_SIDE,
+    WARNING_SEVERITY,
+)
 from scripts.compiler_differential.models import (
     CommandOutcome,
     CorpusProject,
@@ -42,13 +47,15 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         source_dir=source_dir,
         case_dir=case_dir,
         engine=left_engine,
+        side=LEFT_SIDE,
         options=options,
     )
     right: EngineRun = run_engine(
         project=project,
         source_dir=source_dir,
         case_dir=case_dir,
         engine=right_engine,
+        side=RIGHT_SIDE,
         options=options,
     )
     counted: bool = project.seed_coverage and options.stage_captures

scripts/compiler_differential/_helpers/corpus/emitted_codes.py

@@ -17,6 +17,7 @@
     DIAGNOSTIC_CODE_PATTERN,
     ERROR_SEVERITY,
     EXPECT_SUCCESS,
+    LEFT_SIDE,
     RENDER_CODE_SCAN_EXCLUDED,
     RENDER_RAISED_CODES,
     WARNING_SEVERITY,
@@ -139,6 +140,7 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E
         source_dir=source_dir,
         case_dir=case_dir,
         engine=options.engines[0],
+        side=LEFT_SIDE,
         options=options,
     )
     return EmittedCodes(

scripts/compiler_differential/_helpers/coverage/capture_values.py

@@ -2,11 +2,11 @@
 
 from __future__ import annotations
 
-import json
 import re
 from collections.abc import Iterator
 
 from scripts.compiler_differential._helpers.comparing.compare import as_json_object
+from scripts.compiler_differential.classes.capture_file import expand_capture_text
 from scripts.compiler_differential.constants import (
     CALLABLE_MARKER,
     COLLECTION_CAPTURE_MARKERS,
@@ -87,7 +87,7 @@ def capture_problems(
 ) -> tuple[str, ...]:
     """Return why a capture is incomplete or not canonical; empty when it is sound."""
 
-    capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {}
+    capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {}
     problems: list[str] = []
     captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER)
     if captured != fields:

scripts/compiler_differential/_helpers/coverage/discovery.py

@@ -17,6 +17,7 @@
     named_values,
     records,
 )
+from scripts.compiler_differential.classes.capture_file import expand_capture_text
 from scripts.compiler_differential.constants import (
     CALLABLE_CAPTURE_FIELDS,
     CRLF_BYTES,
@@ -59,7 +60,7 @@ def discovery_collection_kinds() -> tuple[str, ...]:
 
 
 def project_discovery_kinds(
-    *, captures: dict[str, dict[str, str]], source_dir: Path
+    *, captures: dict[str, dict[str, Path]], source_dir: Path
 ) -> frozenset[str]:
     """Return the kinds of every discovery capture plus CRLF, which only the bytes show."""
 
@@ -75,7 +76,7 @@ def project_discovery_kinds(
 def discovered_relative_paths(capture_text: str) -> frozenset[str]:
     """Return the project-relative path of every file a discovery capture says it read."""
 
-    capture: object = json.loads(capture_text)
+    capture: object = expand_capture_text(capture_text)
     return frozenset(
         str(path)
         for _, value in named_values(value=capture, names=RELATIVE_PATH_FIELDS)
@@ -94,17 +95,17 @@ def authored_byte_kinds(*, source_dir: Path, relative_paths: frozenset[str]) ->
     )
 
 
-def _discovery_capture_texts(captures: dict[str, dict[str, str]]) -> Iterator[str]:
+def _discovery_capture_texts(captures: dict[str, dict[str, Path]]) -> Iterator[str]:
     for files in captures.values():
-        for name, text in files.items():
+        for name, path in files.items():
             if name.endswith(DISCOVERY_STAGE_CAPTURE_SUFFIX):
-                yield text
+                yield path.read_text(encoding="utf-8")
 
 
 def discovered_input_kinds(capture_text: str) -> frozenset[str]:
     """Return the input kinds one canonical discovery capture proves were discovered."""
 
-    capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {}
+    capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {}
     kinds: set[str] = {
         kind for kind in discovery_collection_kinds() if has_content(capture.get(kind))
     }