Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,16 @@ For the same reviewer-friendly gate with labeled steps, run:
telemetry-lab verify
```

When regeneration reports a mismatch, compare two artifact trees without
accepting either tree as authoritative:

```bash
python scripts/artifact_contract_diff.py --expected path/to/committed --actual path/to/regenerated
```

See [`docs/reviewer-artifact-diff.md`](docs/reviewer-artifact-diff.md#executable-human-triage)
for comparison semantics, limits, and exit codes.

Other demo entrypoints:

- `telemetry-lab run ai-assisted`
Expand Down
28 changes: 28 additions & 0 deletions docs/reviewer-artifact-diff.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,34 @@ the local, file-based artifacts listed in [`docs/reviewer-pack.md`](reviewer-pac
and the schema-covered evidence artifacts in
[`docs/evidence-pipeline-contract.md`](evidence-pipeline-contract.md).

## Executable Human Triage

Use the standalone comparator when a regeneration mismatch needs path-level
context:

```bash
python scripts/artifact_contract_diff.py \
--expected path/to/committed-artifacts \
--actual path/to/regenerated-artifacts
```

The output lists missing, extra, and changed relative paths in stable order.
CSV, Markdown, text, JSON, and JSONL use strict UTF-8 and normalize CRLF and
lone CR to LF before comparison. A binary path that exists in both trees is
checked for presence only; renderer-dependent bytes are not treated as a
reproducibility contract.

Exit status is `0` when comparable artifacts are unchanged, `1` when the tool
finds contract differences, and `2` when an input cannot be compared safely.
The output contains no artifact bodies, timestamps, or absolute checkout paths.
This tool explains a mismatch; it does not replace
`python scripts/regenerate_artifacts.py --check`, accept regenerated output, or
assign a release compatibility label.

Each root is limited to 10,000 files. Symlink or reparse-point roots, linked
entries, special files, unsafe relative paths, unreadable files, and invalid
UTF-8 text fail closed with exit `2`.

## Required Release Diff Sections

Each release artifact diff must include:
Expand Down
63 changes: 63 additions & 0 deletions scripts/artifact_contract_diff.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
from __future__ import annotations

import argparse
import sys
from pathlib import Path
from typing import Sequence


REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))

from telemetry_lab.artifact_contract_diff import ( # noqa: E402
ArtifactContractDiffError,
ArtifactContractDiffReport,
compare_artifact_trees,
)


def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
try:
report = compare_artifact_trees(Path(args.expected), Path(args.actual))
except ArtifactContractDiffError as exc:
print(f"[ERROR] {exc}", file=sys.stderr)
return 2
_print_summary(report)
return 1 if report.has_differences else 0


def _print_summary(report: ArtifactContractDiffReport) -> None:
if report.has_differences:
print(f"[DIFF] {len(report.differences)} artifact contract difference(s)")
for difference in report.differences:
reasons = ", ".join(difference.change_reasons)
print(
f"- {difference.path}: {difference.status} "
f"({difference.artifact_kind}; {reasons})"
)
else:
print(
"[OK] No artifact contract differences "
f"({report.unchanged_files} comparable file(s))"
)
if report.presence_only_paths:
print(
f"[INFO] {len(report.presence_only_paths)} binary artifact(s) "
"checked for presence only"
)


def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Explain shallow contract differences between two artifact trees."
)
parser.add_argument("--expected", required=True, help="Expected artifact directory")
parser.add_argument("--actual", required=True, help="Actual artifact directory")
return parser


if __name__ == "__main__":
raise SystemExit(main())
11 changes: 8 additions & 3 deletions scripts/regenerate_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))

from telemetry_lab.artifact_contract_diff import ( # noqa: E402
TEXT_ARTIFACT_SUFFIXES,
normalize_artifact_text_bytes,
)


@dataclass(frozen=True)
class ArtifactSet:
Expand Down Expand Up @@ -172,7 +177,7 @@ def compare_artifact_set(


def artifacts_match(committed_path: Path, generated_path: Path) -> bool:
if committed_path.suffix.lower() in {".csv", ".json", ".jsonl", ".md", ".txt"}:
if committed_path.suffix.lower() in TEXT_ARTIFACT_SUFFIXES:
return _normalized_text(committed_path) == _normalized_text(generated_path)
return committed_path.read_bytes() == generated_path.read_bytes()

Expand Down Expand Up @@ -419,8 +424,8 @@ def _slug(value: str) -> str:
return "".join(char if char.isalnum() else "-" for char in value.lower()).strip("-")


def _normalized_text(path: Path) -> str:
return path.read_bytes().decode("utf-8").replace("\r\n", "\n")
def _normalized_text(path: Path) -> bytes:
return normalize_artifact_text_bytes(path.read_bytes())


@contextmanager
Expand Down
Loading
Loading