diff --git a/README.md b/README.md index cefeb1f..c8ed571 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,10 @@ accepting either tree as authoritative: python scripts/artifact_contract_diff.py --expected path/to/committed --actual path/to/regenerated ``` +Add `--json-out path/to/artifact-diff.json` for a deterministic +[`artifact-contract-diff/v1`](schemas/artifact_contract_diff.schema.json) +report. Keep that report path outside both compared trees. + See [`docs/reviewer-artifact-diff.md`](docs/reviewer-artifact-diff.md#executable-human-triage) for comparison semantics, limits, and exit codes. diff --git a/docs/reviewer-artifact-diff.md b/docs/reviewer-artifact-diff.md index a51e09d..e33ab89 100644 --- a/docs/reviewer-artifact-diff.md +++ b/docs/reviewer-artifact-diff.md @@ -21,6 +21,16 @@ python scripts/artifact_contract_diff.py \ --actual path/to/regenerated-artifacts ``` +Add a strict machine-readable projection when automation or an attached review +artifact needs the same semantics: + +```bash +python scripts/artifact_contract_diff.py \ + --expected path/to/committed-artifacts \ + --actual path/to/regenerated-artifacts \ + --json-out path/outside-both-trees/artifact-diff.json +``` + The output lists missing, extra, and changed relative paths in stable order. CSV, Markdown, text, JSON, and JSONL use strict UTF-8 and normalize CRLF and lone CR to LF before comparison. A binary path that exists in both trees is @@ -34,6 +44,14 @@ plain content changes from structure, schema-version, and provenance-digest changes. Identical JSON and JSONL are not parsed after their normalized bytes match. Artifact bodies are never printed. +The optional JSON output conforms to +[`artifact-contract-diff/v1`](../schemas/artifact_contract_diff.schema.json). +It contains no timestamp, checkout root, or artifact body, so repeated runs on +the same trees are byte-identical. The destination must resolve outside both +input roots. A new report is written in the destination directory and atomically +replaces an older report only after serialization and file synchronization +succeed; a comparison or write failure leaves an existing report unchanged. + Exit status is `0` when comparable artifacts are unchanged, `1` when the tool finds contract differences, and `2` when an input cannot be compared safely. The output contains no artifact bodies, timestamps, or absolute checkout paths. @@ -48,6 +66,19 @@ linked entries, special files, unsafe relative paths or metadata, malformed digest fields, unreadable files, invalid JSON/JSONL, exceeded limits, and invalid UTF-8 text fail closed with exit `2`. +### JSON Report Semantics + +| Field | Contract | +| --- | --- | +| `status` | `unchanged` requires no differences; `changed` requires at least one. | +| `summary` | Counts expected, actual, unchanged, missing, extra, changed, and presence-only files. Core report invariants reconcile these counts. | +| `differences[].status` | `missing` has only an expected snapshot, `extra` has only an actual snapshot, and `changed` has both. | +| `change_reasons` | Missing and extra use one path reason. Changed comparable artifacts begin with `content-changed` and may add structure, schema-version, or run-manifest-digest reasons. | +| `comparison_digest` | SHA-256 of comparison bytes; text-like artifacts use normalized strict UTF-8 bytes. | +| `comparison_size_bytes` | Length of the comparison bytes, which may differ from the on-disk size after newline normalization. | +| `structure` | Present only for summarized JSON/JSONL snapshots. | +| `presence_only_paths` | Sorted binary paths present in both trees; their bytes are not compared or reported. | + ## Required Release Diff Sections Each release artifact diff must include: diff --git a/schemas/artifact_contract_diff.schema.json b/schemas/artifact_contract_diff.schema.json new file mode 100644 index 0000000..751b1f5 --- /dev/null +++ b/schemas/artifact_contract_diff.schema.json @@ -0,0 +1,257 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/stacknil/telemetry-lab/schemas/artifact_contract_diff.schema.json", + "title": "Telemetry lab artifact contract diff v1", + "description": "Deterministic bounded comparison of two local reviewer artifact trees.", + "type": "object", + "additionalProperties": false, + "required": [ + "report_schema_version", + "status", + "summary", + "differences", + "presence_only_paths" + ], + "properties": { + "report_schema_version": {"const": "artifact-contract-diff/v1"}, + "status": {"enum": ["unchanged", "changed"]}, + "summary": {"$ref": "#/$defs/summary"}, + "differences": { + "type": "array", + "maxItems": 20000, + "uniqueItems": true, + "items": {"$ref": "#/$defs/difference"} + }, + "presence_only_paths": { + "type": "array", + "maxItems": 10000, + "uniqueItems": true, + "items": {"$ref": "#/$defs/relativePath"} + } + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "unchanged"}}}, + "then": {"properties": {"differences": {"maxItems": 0}}} + }, + { + "if": {"properties": {"status": {"const": "changed"}}}, + "then": {"properties": {"differences": {"minItems": 1}}} + } + ], + "$defs": { + "relativePath": { + "type": "string", + "maxLength": 1024, + "pattern": "^(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.?(?:/|$))[^/]+(?:/[^/]+)*$" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "metadataString": {"type": "string", "maxLength": 1024}, + "count": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "expected_files", + "actual_files", + "unchanged_files", + "missing_files", + "extra_files", + "changed_files", + "presence_only_files" + ], + "properties": { + "expected_files": {"$ref": "#/$defs/count"}, + "actual_files": {"$ref": "#/$defs/count"}, + "unchanged_files": {"$ref": "#/$defs/count"}, + "missing_files": {"$ref": "#/$defs/count"}, + "extra_files": {"$ref": "#/$defs/count"}, + "changed_files": {"$ref": "#/$defs/count"}, + "presence_only_files": {"$ref": "#/$defs/count"} + } + }, + "changeReason": { + "enum": [ + "missing-from-actual", + "extra-in-actual", + "content-changed", + "structure-changed", + "schema-version-changed", + "run-manifest-digest-changed" + ] + }, + "difference": { + "type": "object", + "additionalProperties": false, + "required": ["path", "status", "artifact_kind", "change_reasons"], + "properties": { + "path": {"$ref": "#/$defs/relativePath"}, + "status": {"enum": ["missing", "extra", "changed"]}, + "artifact_kind": {"enum": ["json", "jsonl", "text", "binary"]}, + "change_reasons": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/changeReason"} + }, + "expected": {"$ref": "#/$defs/snapshot"}, + "actual": {"$ref": "#/$defs/snapshot"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "missing"}}}, + "then": { + "required": ["expected"], + "properties": { + "change_reasons": {"const": ["missing-from-actual"]} + }, + "not": {"required": ["actual"]} + } + }, + { + "if": {"properties": {"status": {"const": "extra"}}}, + "then": { + "required": ["actual"], + "properties": { + "change_reasons": {"const": ["extra-in-actual"]} + }, + "not": {"required": ["expected"]} + } + }, + { + "if": {"properties": {"status": {"const": "changed"}}}, + "then": { + "required": ["expected", "actual"], + "properties": { + "artifact_kind": {"enum": ["json", "jsonl", "text"]}, + "change_reasons": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "prefixItems": [{"const": "content-changed"}], + "items": { + "enum": [ + "structure-changed", + "schema-version-changed", + "run-manifest-digest-changed" + ] + } + } + } + } + }, + { + "if": { + "properties": {"artifact_kind": {"enum": ["json", "jsonl"]}} + }, + "then": { + "properties": { + "expected": {"$ref": "#/$defs/structuredSnapshot"}, + "actual": {"$ref": "#/$defs/structuredSnapshot"} + } + } + }, + { + "if": { + "properties": {"artifact_kind": {"enum": ["text", "binary"]}} + }, + "then": { + "properties": { + "expected": {"$ref": "#/$defs/plainSnapshot"}, + "actual": {"$ref": "#/$defs/plainSnapshot"} + } + } + } + ] + }, + "snapshot": { + "type": "object", + "additionalProperties": false, + "required": ["comparison_digest", "comparison_size_bytes"], + "properties": { + "comparison_digest": {"$ref": "#/$defs/digest"}, + "comparison_size_bytes": {"type": "integer", "minimum": 0}, + "structure": {"$ref": "#/$defs/structure"} + } + }, + "structuredSnapshot": { + "allOf": [ + {"$ref": "#/$defs/snapshot"}, + {"required": ["structure"]} + ] + }, + "plainSnapshot": { + "allOf": [ + {"$ref": "#/$defs/snapshot"}, + {"not": {"required": ["structure"]}} + ] + }, + "structure": { + "type": "object", + "additionalProperties": false, + "required": ["container", "record_count", "top_level_keys"], + "properties": { + "container": { + "enum": [ + "object", + "array", + "jsonl", + "string", + "number", + "boolean", + "null" + ] + }, + "record_count": { + "type": "integer", + "minimum": 0, + "maximum": 67108864 + }, + "top_level_keys": { + "type": "array", + "maxItems": 4096, + "uniqueItems": true, + "items": {"$ref": "#/$defs/metadataString"} + }, + "schema_versions": { + "type": "object", + "minProperties": 1, + "maxProperties": 4096, + "propertyNames": {"$ref": "#/$defs/metadataString"}, + "additionalProperties": { + "type": "array", + "minItems": 1, + "maxItems": 4096, + "uniqueItems": true, + "items": {"$ref": "#/$defs/metadataString"} + } + }, + "run_manifest_digests": {"$ref": "#/$defs/runManifestDigests"} + } + }, + "runManifestDigests": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "input_digest": {"$ref": "#/$defs/digest"}, + "config_digest": {"$ref": "#/$defs/digest"}, + "input_file_digests": {"$ref": "#/$defs/fileDigestMap"}, + "config_file_digests": {"$ref": "#/$defs/fileDigestMap"} + } + }, + "fileDigestMap": { + "type": "object", + "maxProperties": 10000, + "propertyNames": {"$ref": "#/$defs/relativePath"}, + "additionalProperties": {"$ref": "#/$defs/digest"} + } + } +} diff --git a/scripts/artifact_contract_diff.py b/scripts/artifact_contract_diff.py index f475b9a..6946ba3 100644 --- a/scripts/artifact_contract_diff.py +++ b/scripts/artifact_contract_diff.py @@ -1,7 +1,10 @@ from __future__ import annotations import argparse +import json +import os import sys +import tempfile from pathlib import Path from typing import Sequence @@ -20,8 +23,25 @@ def main(argv: Sequence[str] | None = None) -> int: args = _parser().parse_args(argv) + expected = Path(args.expected) + actual = Path(args.actual) try: - report = compare_artifact_trees(Path(args.expected), Path(args.actual)) + json_out = ( + _resolve_path(Path(args.json_out), "JSON report") + if args.json_out + else None + ) + if json_out is not None: + expected_boundary = _resolve_path(expected, "expected artifact root") + actual_boundary = _resolve_path(actual, "actual artifact root") + _validate_report_destination( + json_out, + expected_boundary, + actual_boundary, + ) + report = compare_artifact_trees(expected, actual) + if json_out is not None: + _write_json_report(report, json_out) except ArtifactContractDiffError as exc: print(f"[ERROR] {exc}", file=sys.stderr) return 2 @@ -29,6 +49,64 @@ def main(argv: Sequence[str] | None = None) -> int: return 1 if report.has_differences else 0 +def _write_json_report(report: ArtifactContractDiffReport, path: Path) -> None: + temporary_path: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="\n", + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + json.dump(report.to_dict(), handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + temporary_path = None + except (OSError, TypeError, ValueError) as exc: + raise ArtifactContractDiffError("cannot write JSON report") from exc + finally: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + + +def _resolve_path(path: Path, label: str) -> Path: + try: + return path.resolve(strict=False) + except OSError as exc: + raise ArtifactContractDiffError(f"cannot resolve {label}") from exc + + +def _validate_report_destination( + report_path: Path, + expected_root: Path, + actual_root: Path, +) -> None: + if _is_within(report_path, expected_root) or _is_within( + report_path, actual_root + ): + raise ArtifactContractDiffError( + "JSON report must be outside both artifact roots" + ) + + +def _is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + def _print_summary(report: ArtifactContractDiffReport) -> None: if report.has_differences: print(f"[DIFF] {len(report.differences)} artifact contract difference(s)") @@ -52,10 +130,14 @@ def _print_summary(report: ArtifactContractDiffReport) -> None: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description="Explain shallow contract differences between two artifact trees." + description="Explain bounded contract differences between two artifact trees." ) parser.add_argument("--expected", required=True, help="Expected artifact directory") parser.add_argument("--actual", required=True, help="Actual artifact directory") + parser.add_argument( + "--json-out", + help="Optional path for a deterministic artifact-contract-diff/v1 report", + ) return parser diff --git a/src/telemetry_lab/artifact_contract_diff.py b/src/telemetry_lab/artifact_contract_diff.py index 1c82c32..ddeb73e 100644 --- a/src/telemetry_lab/artifact_contract_diff.py +++ b/src/telemetry_lab/artifact_contract_diff.py @@ -9,6 +9,7 @@ from collections import Counter from collections.abc import Iterable, Iterator, Mapping from contextlib import contextmanager +from copy import deepcopy from dataclasses import dataclass, replace from hashlib import sha256 from pathlib import Path @@ -18,6 +19,7 @@ TEXT_ARTIFACT_SUFFIXES: Final = frozenset( {".csv", ".json", ".jsonl", ".md", ".txt"} ) +REPORT_SCHEMA_VERSION: Final = "artifact-contract-diff/v1" MAX_FILES: Final = 10_000 MAX_STRUCTURED_BYTES: Final = 64 * 1024 * 1024 MAX_STRUCTURE_ITEMS: Final = 4_096 @@ -75,6 +77,15 @@ def __post_init__(self) -> None: if self.comparison_size_bytes < 0: raise ArtifactContractDiffError("invalid comparison size") + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "comparison_digest": self.comparison_digest, + "comparison_size_bytes": self.comparison_size_bytes, + } + if self.structure is not None: + result["structure"] = deepcopy(dict(self.structure)) + return result + @dataclass(frozen=True) class ArtifactDifference: @@ -121,6 +132,19 @@ def __post_init__(self) -> None: if not valid: raise ArtifactContractDiffError(f"{self.status} difference is inconsistent") + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "path": self.path, + "status": self.status, + "artifact_kind": self.artifact_kind, + "change_reasons": list(self.change_reasons), + } + if self.expected is not None: + result["expected"] = self.expected.to_dict() + if self.actual is not None: + result["actual"] = self.actual.to_dict() + return result + @dataclass(frozen=True) class ArtifactContractDiffReport: @@ -178,6 +202,23 @@ def __post_init__(self) -> None: def has_differences(self) -> bool: return bool(self.differences) + def to_dict(self) -> dict[str, Any]: + return { + "report_schema_version": REPORT_SCHEMA_VERSION, + "status": "changed" if self.has_differences else "unchanged", + "summary": { + "expected_files": self.expected_files, + "actual_files": self.actual_files, + "unchanged_files": self.unchanged_files, + "missing_files": self.missing_files, + "extra_files": self.extra_files, + "changed_files": self.changed_files, + "presence_only_files": len(self.presence_only_paths), + }, + "differences": [difference.to_dict() for difference in self.differences], + "presence_only_paths": list(self.presence_only_paths), + } + def normalize_artifact_text_bytes(content: bytes) -> bytes: """Validate UTF-8 and canonicalize CRLF or lone CR to LF.""" diff --git a/tests/test_artifact_contract_diff_json_report.py b/tests/test_artifact_contract_diff_json_report.py new file mode 100644 index 0000000..51e76fd --- /dev/null +++ b/tests/test_artifact_contract_diff_json_report.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from telemetry_lab.artifact_contract_diff import ( + ArtifactContractDiffError, + compare_artifact_trees, +) + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "artifact_contract_diff.py" +SCHEMA_PATH = REPO_ROOT / "schemas" / "artifact_contract_diff.schema.json" + + +@pytest.fixture +def artifact_roots(tmp_path: Path) -> tuple[Path, Path]: + expected = tmp_path / "expected" + actual = tmp_path / "actual" + expected.mkdir() + actual.mkdir() + return expected, actual + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True) + "\n", encoding="utf-8") + + +def _run_cli( + expected: Path, actual: Path, report_path: Path +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--expected", + str(expected), + "--actual", + str(actual), + "--json-out", + str(report_path), + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + ) + + +def _load_cli_script(): + spec = importlib.util.spec_from_file_location( + "artifact_contract_diff_cli", + SCRIPT_PATH, + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _validator() -> Draft202012Validator: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def test_cli_writes_deterministic_schema_valid_report( + artifact_roots: tuple[Path, Path], tmp_path: Path +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", [{"id": 1}]) + _write_json(actual / "summary.json", [{"id": 1}, {"id": 2}]) + report_path = tmp_path / "reports" / "artifact-diff.json" + + first = _run_cli(expected, actual, report_path) + first_bytes = report_path.read_bytes() + second = _run_cli(expected, actual, report_path) + payload = json.loads(report_path.read_text(encoding="utf-8")) + + assert first.returncode == second.returncode == 1 + assert first.stdout == second.stdout + assert first_bytes == report_path.read_bytes() + assert first_bytes.endswith(b"\n") + assert b"\r\n" not in first_bytes + assert payload["report_schema_version"] == "artifact-contract-diff/v1" + assert payload["status"] == "changed" + assert payload["differences"][0]["change_reasons"] == [ + "content-changed", + "structure-changed", + ] + serialized = json.dumps(payload, sort_keys=True) + assert str(expected) not in serialized + assert str(actual) not in serialized + assert "generated_at" not in serialized + assert list(_validator().iter_errors(payload)) == [] + + +def test_cli_writes_unchanged_report_with_presence_only_binary( + artifact_roots: tuple[Path, Path], tmp_path: Path +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", {"count": 1}) + _write_json(actual / "summary.json", {"count": 1}) + (expected / "plot.png").write_bytes(b"renderer-a") + (actual / "plot.png").write_bytes(b"renderer-b") + report_path = tmp_path / "artifact-diff.json" + + result = _run_cli(expected, actual, report_path) + payload = json.loads(report_path.read_text(encoding="utf-8")) + + assert result.returncode == 0 + assert payload["status"] == "unchanged" + assert payload["differences"] == [] + assert payload["presence_only_paths"] == ["plot.png"] + assert payload["summary"]["presence_only_files"] == 1 + assert list(_validator().iter_errors(payload)) == [] + + +@pytest.mark.parametrize("root_name", ["expected", "actual"]) +def test_cli_refuses_output_inside_an_input_tree( + artifact_roots: tuple[Path, Path], root_name: str +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", {"count": 1}) + _write_json(actual / "summary.json", {"count": 1}) + root = expected if root_name == "expected" else actual + report_path = root / "reports" / "artifact-diff.json" + original = b'{"keep":"artifact"}\n' + report_path.parent.mkdir() + report_path.write_bytes(original) + + result = _run_cli(expected, actual, report_path) + + assert result.returncode == 2 + assert "JSON report must be outside both artifact roots" in result.stderr + assert report_path.read_bytes() == original + + +def test_json_report_write_is_atomic_on_replace_failure( + artifact_roots: tuple[Path, Path], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", {"count": 1}) + _write_json(actual / "summary.json", {"count": 2}) + report = compare_artifact_trees(expected, actual) + report_path = tmp_path / "artifact-diff.json" + original = b'{"keep":"previous-report"}\n' + report_path.write_bytes(original) + cli = _load_cli_script() + + def fail_replace(_source: object, _destination: object) -> None: + raise OSError("simulated replace failure") + + monkeypatch.setattr(cli.os, "replace", fail_replace) + + with pytest.raises(ArtifactContractDiffError, match="cannot write JSON report"): + cli._write_json_report(report, report_path) + + assert report_path.read_bytes() == original + assert list(tmp_path.glob(".artifact-diff.json.*.tmp")) == [] + + +def test_cli_preserves_existing_report_when_comparison_fails( + artifact_roots: tuple[Path, Path], tmp_path: Path +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", {"count": 1}) + (actual / "summary.json").write_bytes(b"{\n") + report_path = tmp_path / "artifact-diff.json" + original = b'{"keep":"previous-report"}\n' + report_path.write_bytes(original) + + result = _run_cli(expected, actual, report_path) + + assert result.returncode == 2 + assert "invalid JSON artifact" in result.stderr + assert str(expected) not in result.stderr + assert str(actual) not in result.stderr + assert report_path.read_bytes() == original + + +def test_cli_does_not_resolve_away_symlink_root_rejection( + artifact_roots: tuple[Path, Path], tmp_path: Path +) -> None: + expected, actual = artifact_roots + linked = tmp_path / "linked-expected" + try: + linked.symlink_to(expected, target_is_directory=True) + except OSError: + pytest.skip("directory symlinks are unavailable") + report_path = tmp_path / "artifact-diff.json" + + result = _run_cli(linked, actual, report_path) + + assert result.returncode == 2 + assert "non-symlink directory" in result.stderr + assert not report_path.exists() + + +def test_report_schema_rejects_cross_field_contradictions( + artifact_roots: tuple[Path, Path] +) -> None: + expected, actual = artifact_roots + _write_json(expected / "finding.json", {"status": "old"}) + missing = compare_artifact_trees(expected, actual).to_dict() + validator = _validator() + + invalid_payloads = [dict(missing, status="unchanged")] + wrong_missing_reason = json.loads(json.dumps(missing)) + wrong_missing_reason["differences"][0]["change_reasons"] = ["content-changed"] + invalid_payloads.append(wrong_missing_reason) + missing_structure = json.loads(json.dumps(missing)) + del missing_structure["differences"][0]["expected"]["structure"] + invalid_payloads.append(missing_structure) + + _write_json(actual / "finding.json", {"status": "new"}) + changed = compare_artifact_trees(expected, actual).to_dict() + invalid_payloads.append(dict(changed, differences=[])) + wrong_changed_reason = json.loads(json.dumps(changed)) + wrong_changed_reason["differences"][0]["change_reasons"] = [ + "missing-from-actual" + ] + invalid_payloads.append(wrong_changed_reason) + changed_binary = json.loads(json.dumps(changed)) + changed_binary["differences"][0]["artifact_kind"] = "binary" + invalid_payloads.append(changed_binary) + structured_text = json.loads(json.dumps(changed)) + structured_text["differences"][0]["artifact_kind"] = "text" + invalid_payloads.append(structured_text) + + for payload in invalid_payloads: + assert list(validator.iter_errors(payload)) + + +def test_report_payload_is_detached_from_internal_structure( + artifact_roots: tuple[Path, Path], +) -> None: + expected, actual = artifact_roots + _write_json(expected / "summary.json", [{"old": True}]) + _write_json(actual / "summary.json", [{"new": True}]) + report = compare_artifact_trees(expected, actual) + + payload = report.to_dict() + payload["differences"][0]["expected"]["structure"]["top_level_keys"].append( + "mutated" + ) + + difference = report.differences[0] + assert difference.expected is not None + assert difference.expected.structure is not None + assert difference.expected.structure["top_level_keys"] == ["old"]