test(compiler): compare large stage captures without loading them whole

Captures embedded every repeated subtree in full, so their size grew
quadratically with sources per file and the harness held and parsed both
engines' captures in memory; dense 3k ran out of memory.

The capture writer now stores every container or string of at least 1 KiB
once under the SHA-256 of its compact JSON, with invocation ids and native
store suffixes masked, and refers to it as {"__shared__": digest}. Sets and
unordered maps sort by the same masked form, so ordering never depends on
noise the comparison ignores, and authored mappings using a reserved marker
key are encoded as pairs. Each shared node and the root sit on their own
line, so the harness indexes a capture by digest, treats equal digests as
equal without reading them, and parses only the nodes on a differing path
while still reporting the logical JSON pointer and resolved previews.
Captures stay on disk, one directory per compared side, instead of in memory.

Author: kvlonge

scripts/compiler_differential/_helpers/comparing/capture.py

@@ -0,0 +1,53 @@
+"""Compare two stage captures on disk without loading either one whole."""
+
+from __future__ import annotations
+
+import itertools
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+from scripts.compiler_differential._helpers.comparing.compare import (
+    first_json_difference,
+    preview,
+)
+from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text
+from scripts.compiler_differential.classes.capture_file import CaptureFile
+from scripts.compiler_differential.constants import MISSING_VALUE
+from scripts.compiler_differential.models import Divergence
+
+
+def first_capture_difference(*, left: Path, right: Path) -> Divergence | None:
+    """Return the first logical JSON pointer, or line for invalid JSON, where captures differ."""
+
+    try:
+        left_capture: CaptureFile = CaptureFile(left)
+        right_capture: CaptureFile = CaptureFile(right)
+        return first_json_difference(
+            left=left_capture.root,
+            right=right_capture.root,
+            resolve=(left_capture.resolve, right_capture.resolve),
+        )
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        return _first_line_difference(left=left, right=right)
+
+
+def _first_line_difference(*, left: Path, right: Path) -> Divergence | None:
+    with left.open("rb") as left_stream, right.open("rb") as right_stream:
+        lines: Iterator[tuple[bytes | None, bytes | None]] = itertools.zip_longest(
+            left_stream, right_stream
+        )
+        for number, (left_line, right_line) in enumerate(lines, start=1):
+            left_text: str = _line_text(left_line)
+            right_text: str = _line_text(right_line)
+            if left_text != right_text:
+                return Divergence(
+                    location=f"line {number}", left=preview(left_text), right=preview(right_text)
+                )
+    return None
+
+
+def _line_text(line: bytes | None) -> str:
+    if line is None:
+        return MISSING_VALUE
+    return normalize_artifact_text(line.decode("utf-8", "surrogateescape").rstrip("\n"))

scripts/compiler_differential/_helpers/comparing/compare.py

@@ -3,34 +3,50 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Callable
 from typing import cast
 
 from scripts.compiler_differential.constants import MISSING_VALUE, VALUE_PREVIEW_CHARACTERS
 from scripts.compiler_differential.models import Divergence
+from scripts.compiler_differential.types import NodeResolvers
+from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SHARED_MARKER
 
 
-def first_json_difference(*, left: object, right: object, pointer: str = "") -> Divergence | None:
-    """Return the first JSON pointer whose value, type, or object key order differs."""
+def first_json_difference(
+    *, left: object, right: object, pointer: str = "", resolve: NodeResolvers | None = None
+) -> Divergence | None:
+    """Return the first JSON pointer whose value, type, or key order differs, via `resolve`."""
 
+    if resolve is not None:
+        left_digest: str | None = shared_digest(left)
+        if left_digest is not None and left_digest == shared_digest(right):
+            return None
+        left, right = resolve[0](left), resolve[1](right)
     if type(left) is not type(right):
         return _divergence(location=pointer or "/", left=left, right=right)
     left_object: dict[str, object] | None = as_json_object(left)
     right_object: dict[str, object] | None = as_json_object(right)
     if left_object is not None and right_object is not None:
-        return _first_object_difference(left=left_object, right=right_object, pointer=pointer)
+        return _first_object_difference(
+            left=left_object, right=right_object, pointer=pointer, resolve=resolve
+        )
     if isinstance(left, list) and isinstance(right, list):
         for index, (left_item, right_item) in enumerate(zip(left, right, strict=False)):
             found: Divergence | None = first_json_difference(
-                left=left_item, right=right_item, pointer=f"{pointer}/{index}"
+                left=left_item, right=right_item, pointer=f"{pointer}/{index}", resolve=resolve
             )
             if found is not None:
                 return found
         if len(left) != len(right):
             index: int = min(len(left), len(right))
             return _divergence(
                 location=f"{pointer}/{index}",
-                left=left[index] if index < len(left) else MISSING_VALUE,
-                right=right[index] if index < len(right) else MISSING_VALUE,
+                left=_shown(value=left[index], resolve=resolve, side=0)
+                if index < len(left)
+                else MISSING_VALUE,
+                right=_shown(value=right[index], resolve=resolve, side=1)
+                if index < len(right)
+                else MISSING_VALUE,
             )
         return None
     if left != right:
@@ -50,20 +66,19 @@ def first_text_difference(*, left: str, right: str) -> Divergence | None:
         right_line: str = right_lines[index] if index < len(right_lines) else MISSING_VALUE
         if left_line != right_line:
             return Divergence(
-                location=f"line {index + 1}", left=_preview(left_line), right=_preview(right_line)
+                location=f"line {index + 1}", left=preview(left_line), right=preview(right_line)
             )
     return Divergence(location="end of file", left="<bytes differ>", right="<bytes differ>")
 
 
-def first_document_difference(*, left: str, right: str) -> Divergence | None:
-    """Compare two documents as JSON when both parse, otherwise line by line."""
+def shared_digest(value: object) -> str | None:
+    """Return the digest a stage-capture `{"__shared__": digest}` reference names."""
 
-    try:
-        left_payload: object = json.loads(left)
-        right_payload: object = json.loads(right)
-    except json.JSONDecodeError:
-        return first_text_difference(left=left, right=right)
-    return first_json_difference(left=left_payload, right=right_payload)
+    mapping: dict[str, object] | None = as_json_object(value)
+    if mapping is None or len(mapping) != 1:
+        return None
+    digest: object = mapping.get(STAGE_CAPTURE_SHARED_MARKER)
+    return digest if isinstance(digest, str) else None
 
 
 def as_json_object(value: object) -> dict[str, object] | None:
@@ -79,35 +94,70 @@ def json_pointer_token(key: str) -> str:
 
 
 def _first_object_difference(
-    *, left: dict[str, object], right: dict[str, object], pointer: str
+    *,
+    left: dict[str, object],
+    right: dict[str, object],
+    pointer: str,
+    resolve: NodeResolvers | None,
 ) -> Divergence | None:
     for key in (*left, *(key for key in right if key not in left)):
         location: str = f"{pointer}/{json_pointer_token(key)}"
         if key not in left or key not in right:
             return _divergence(
                 location=location,
-                left=left.get(key, MISSING_VALUE),
-                right=right.get(key, MISSING_VALUE),
+                left=_shown(value=left.get(key, MISSING_VALUE), resolve=resolve, side=0),
+                right=_shown(value=right.get(key, MISSING_VALUE), resolve=resolve, side=1),
             )
         found: Divergence | None = first_json_difference(
-            left=left[key], right=right[key], pointer=location
+            left=left[key], right=right[key], pointer=location, resolve=resolve
         )
         if found is not None:
             return found
     if list(left) != list(right):
         return Divergence(
             location=f"{pointer or '/'} (key order)",
-            left=_preview(list(left)),
-            right=_preview(list(right)),
+            left=preview(list(left)),
+            right=preview(list(right)),
         )
     return None
 
 
+def _shown(*, value: object, resolve: NodeResolvers | None, side: int) -> object:
+    if resolve is None:
+        return value
+    return _previewed(value=value, resolve=resolve[side], remaining=VALUE_PREVIEW_CHARACTERS)[0]
+
+
+def _previewed(
+    *, value: object, resolve: Callable[[object], object], remaining: int
+) -> tuple[object, int]:
+    if remaining <= 0:
+        return value, remaining
+    resolved: object = resolve(value)
+    mapping: dict[str, object] | None = as_json_object(resolved)
+    if mapping is not None:
+        shown: dict[str, object] = {}
+        for key, item in mapping.items():
+            shown[key], remaining = _previewed(
+                value=item, resolve=resolve, remaining=remaining - len(key)
+            )
+        return shown, remaining
+    if isinstance(resolved, list):
+        items: list[object] = []
+        for item in resolved:
+            shown_item, remaining = _previewed(value=item, resolve=resolve, remaining=remaining)
+            items.append(shown_item)
+        return items, remaining
+    return resolved, remaining - len(json.dumps(resolved, ensure_ascii=False))
+
+
 def _divergence(*, location: str, left: object, right: object) -> Divergence:
-    return Divergence(location=location, left=_preview(left), right=_preview(right))
+    return Divergence(location=location, left=preview(left), right=preview(right))
+
 
+def preview(value: object) -> str:
+    """Render a value for a report, shortened to the preview width."""
 
-def _preview(value: object) -> str:
     rendered: str = (
         value
         if isinstance(value, str) and value == MISSING_VALUE

scripts/compiler_differential/_helpers/comparing/comparison.py

@@ -4,10 +4,11 @@
 
 import json
 from collections.abc import Callable
+from pathlib import Path
 
+from scripts.compiler_differential._helpers.comparing.capture import first_capture_difference
 from scripts.compiler_differential._helpers.comparing.compare import (
     as_json_object,
-    first_document_difference,
     first_json_difference,
     first_text_difference,
 )
@@ -237,12 +238,12 @@ def _loads(text: str | None) -> object:
 
 
 def _capture_differences(
-    *, left: dict[str, dict[str, str]], right: dict[str, dict[str, str]]
+    *, left: dict[str, dict[str, Path]], right: dict[str, dict[str, Path]]
 ) -> list[tuple[str, Divergence | None]]:
     differences: list[tuple[str, Divergence | None]] = []
     for command in sorted(set(left) | set(right)):
-        left_files: dict[str, str] = left.get(command, {})
-        right_files: dict[str, str] = right.get(command, {})
+        left_files: dict[str, Path] = left.get(command, {})
+        right_files: dict[str, Path] = right.get(command, {})
         for name in sorted(set(left_files) | set(right_files)):
             artifact: str = f"stage capture {command}/{name}"
             if name not in left_files or name not in right_files:
@@ -258,10 +259,7 @@ def _capture_differences(
             differences.append(
                 (
                     artifact,
-                    first_document_difference(
-                        left=normalize_artifact_text(left_files[name]),
-                        right=normalize_artifact_text(right_files[name]),
-                    ),
+                    first_capture_difference(left=left_files[name], right=right_files[name]),
                 )
             )
     return differences

scripts/compiler_differential/_helpers/comparing/normalize.py

@@ -6,18 +6,16 @@
 from scripts.compiler_differential.constants import (
     ELAPSED_TIME_MASK,
     ELAPSED_TIME_PATTERN,
-    ENGINE_NAMESPACE_PATTERN,
-    INVOCATION_ID_MASK,
-    INVOCATION_ID_PATTERN,
     MANIFEST_VOLATILE_METADATA,
     STRIPPED_REPORT_FIELDS,
 )
+from sqlbuild.compiler.frontier.classes.stage_capture_encoder import mask_capture_noise
 
 
 def normalize_artifact_text(text: str) -> str:
     """Mask invocation ids and engine store suffixes, which differ by design."""
 
-    return ENGINE_NAMESPACE_PATTERN.sub("", INVOCATION_ID_PATTERN.sub(INVOCATION_ID_MASK, text))
+    return mask_capture_noise(text)
 
 
 def normalize_stderr(text: str) -> str:

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -13,7 +13,12 @@
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
 from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
 from scripts.compiler_differential._helpers.running.execution import run_engine
-from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
+from scripts.compiler_differential.constants import (
+    ERROR_SEVERITY,
+    LEFT_SIDE,
+    RIGHT_SIDE,
+    WARNING_SEVERITY,
+)
 from scripts.compiler_differential.models import (
     CommandOutcome,
     CorpusProject,
@@ -42,13 +47,15 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         source_dir=source_dir,
         case_dir=case_dir,
         engine=left_engine,
+        side=LEFT_SIDE,
         options=options,
     )
     right: EngineRun = run_engine(
         project=project,
         source_dir=source_dir,
         case_dir=case_dir,
         engine=right_engine,
+        side=RIGHT_SIDE,
         options=options,
     )
     counted: bool = project.seed_coverage and options.stage_captures

scripts/compiler_differential/_helpers/corpus/emitted_codes.py

@@ -17,6 +17,7 @@
     DIAGNOSTIC_CODE_PATTERN,
     ERROR_SEVERITY,
     EXPECT_SUCCESS,
+    LEFT_SIDE,
     RENDER_CODE_SCAN_EXCLUDED,
     RENDER_RAISED_CODES,
     WARNING_SEVERITY,
@@ -139,6 +140,7 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E
         source_dir=source_dir,
         case_dir=case_dir,
         engine=options.engines[0],
+        side=LEFT_SIDE,
         options=options,
     )
     return EmittedCodes(

scripts/compiler_differential/_helpers/coverage/capture_values.py

@@ -2,11 +2,11 @@
 
 from __future__ import annotations
 
-import json
 import re
 from collections.abc import Iterator
 
 from scripts.compiler_differential._helpers.comparing.compare import as_json_object
+from scripts.compiler_differential.classes.capture_file import expand_capture_text
 from scripts.compiler_differential.constants import (
     CALLABLE_MARKER,
     COLLECTION_CAPTURE_MARKERS,
@@ -87,7 +87,7 @@ def capture_problems(
 ) -> tuple[str, ...]:
     """Return why a capture is incomplete or not canonical; empty when it is sound."""
 
-    capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {}
+    capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {}
     problems: list[str] = []
     captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER)
     if captured != fields:

scripts/compiler_differential/_helpers/coverage/discovery.py

@@ -17,6 +17,7 @@
     named_values,
     records,
 )
+from scripts.compiler_differential.classes.capture_file import expand_capture_text
 from scripts.compiler_differential.constants import (
     CALLABLE_CAPTURE_FIELDS,
     CRLF_BYTES,
@@ -59,7 +60,7 @@ def discovery_collection_kinds() -> tuple[str, ...]:
 
 
 def project_discovery_kinds(
-    *, captures: dict[str, dict[str, str]], source_dir: Path
+    *, captures: dict[str, dict[str, Path]], source_dir: Path
 ) -> frozenset[str]:
     """Return the kinds of every discovery capture plus CRLF, which only the bytes show."""
 
@@ -75,7 +76,7 @@ def project_discovery_kinds(
 def discovered_relative_paths(capture_text: str) -> frozenset[str]:
     """Return the project-relative path of every file a discovery capture says it read."""
 
-    capture: object = json.loads(capture_text)
+    capture: object = expand_capture_text(capture_text)
     return frozenset(
         str(path)
         for _, value in named_values(value=capture, names=RELATIVE_PATH_FIELDS)
@@ -94,17 +95,17 @@ def authored_byte_kinds(*, source_dir: Path, relative_paths: frozenset[str]) ->
     )
 
 
-def _discovery_capture_texts(captures: dict[str, dict[str, str]]) -> Iterator[str]:
+def _discovery_capture_texts(captures: dict[str, dict[str, Path]]) -> Iterator[str]:
     for files in captures.values():
-        for name, text in files.items():
+        for name, path in files.items():
             if name.endswith(DISCOVERY_STAGE_CAPTURE_SUFFIX):
-                yield text
+                yield path.read_text(encoding="utf-8")
 
 
 def discovered_input_kinds(capture_text: str) -> frozenset[str]:
     """Return the input kinds one canonical discovery capture proves were discovered."""
 
-    capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {}
+    capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {}
     kinds: set[str] = {
         kind for kind in discovery_collection_kinds() if has_content(capture.get(kind))
     }