diff --git a/.aii/config.yaml b/.aii/config.yaml index 8a487c5..d27dc8b 100644 --- a/.aii/config.yaml +++ b/.aii/config.yaml @@ -1,7 +1,7 @@ # AII sidecar — InstituteOS metadata. Schema: aii_platform.aii_sidecar meta: sidecar_version: 1.2.1 - updated: '2026-11-11' + updated: '2026-11-12' repo: name: Knowledge-Engineering slug: knowledge-engineering @@ -88,16 +88,42 @@ capabilities: - ci - citation tasks: - inventory: - cmd: git ls-files - description: List tracked files. - test: + links: + cmd: python .github/scripts/gate.py links + description: Check documentation links and Publishing digests; docs index is docs/documentation-index.md. + python: + cmd: python .github/scripts/gate.py python + description: Compile tracked Python sources. + notebooks: + cmd: python .github/scripts/gate.py notebooks + description: Check notebook JSON and absence of outputs. + checkpoints: + cmd: python .github/scripts/gate.py checkpoints + description: Reject tracked Jupyter checkpoints. + archive: + cmd: python .github/scripts/gate.py archive + description: Run the archive checks and Publishing digest checks. + tests: + cmd: python .github/scripts/gate.py tests + description: Run dependency-free gate tests. + gnn: + cmd: python .github/scripts/gate.py gnn + description: Check GNN conformance using .github/gnn-pin.json. + fast: cmd: python .github/scripts/gate.py fast - description: Run the dependency-free gate arm (links, syntax, notebooks, tests). + description: Run archive checks and tests; no manuscript command exists. + full: + cmd: python .github/scripts/gate.py full + description: Run fast and GNN conformance. + timings: + cmd: python .github/scripts/gate.py timings + description: Measure offline arms and regenerate docs/gate-timings.md. docs: - README.md - AGENTS.md -- docs/INTEGRATION.md +- .aii/docs/INTEGRATION.md - docs/README.md -- docs/AGENTS.md +- .aii/docs/AGENTS.md +- docs/documentation-index.md +- docs/workflows/publishing.md schema: aii-sidecar/v1 diff --git a/.aii/docs/INTEGRATION.md b/.aii/docs/INTEGRATION.md index 6bb80c9..b4ac62e 100644 --- a/.aii/docs/INTEGRATION.md +++ b/.aii/docs/INTEGRATION.md @@ -55,10 +55,11 @@ ## Portable tasks -| Task | Command | Working directory | Description | -| --- | --- | --- | --- | -| `inventory` | `git ls-files` | `.` | List tracked files. | -| `test` | `python .github/scripts/gate.py fast` | `.` | Run the default dependency-free gate arm (links, syntax, notebooks, gate tests). | +Portable tasks in [config.yaml](../config.yaml) map directly to the gate's arms. +The [run guide](../../docs/RUN_GUIDE.md) owns their commands and descriptions; +the [documentation index](../../docs/documentation-index.md) is the docs entry +point. There is no manuscript command. GNN conformance uses +[the pin](../../.github/gnn-pin.json). ## Validation diff --git a/.github/scripts/gate.py b/.github/scripts/gate.py index 0495eaf..3fcf2c0 100755 --- a/.github/scripts/gate.py +++ b/.github/scripts/gate.py @@ -97,6 +97,20 @@ def _report(label: str, inspected: int, errors: list[str], note: str = "") -> in def arm_archive(names: tuple[str, ...], label: str) -> int: inspected, errors = archive_checks.run(names, ROOT) + if "links" in names: + for artifact in ( + "docs/workflows/record-identity.schema.json", + "scripts/gate/identity.py", + ".github/gnn-pin.json", + ): + result = subprocess.run( + [sys.executable, "-B", "scripts/gate/identity.py", "--verify-copy", artifact], + cwd=ROOT, + stdin=subprocess.DEVNULL, + check=False, + ) + if result.returncode: + errors.append(f"{artifact}: Publishing digest verification failed") notes = { "links": "links resolve", "python": "Python compiles", diff --git a/.github/scripts/tests/test_publishing.py b/.github/scripts/tests/test_publishing.py new file mode 100644 index 0000000..69e1942 --- /dev/null +++ b/.github/scripts/tests/test_publishing.py @@ -0,0 +1,49 @@ +"""Publishing drift must fail the member's existing documentation gate.""" + +from pathlib import Path +import shutil +import tempfile +import unittest +from unittest.mock import patch + +import gate + + +class PublishingDriftTests(unittest.TestCase): + def setUp(self) -> None: + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + for name in ( + "scripts/gate/identity.py", + "docs/workflows/record-identity.schema.json", + "docs/workflows/publishing.md", + ".github/gnn-pin.json", + ): + destination = self.root / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(gate.ROOT / name, destination) + + def check_links(self) -> int: + with patch.object(gate, "ROOT", self.root), patch.object( + gate.archive_checks, "run", return_value=(0, []) + ): + return gate.arm_archive(("links",), "links") + + def test_matching_artifacts_pass(self) -> None: + self.assertEqual(self.check_links(), 0) + + def test_changed_artifact_fails_docs_gate(self) -> None: + for name in ( + "scripts/gate/identity.py", + "docs/workflows/record-identity.schema.json", + ".github/gnn-pin.json", + ): + with self.subTest(artifact=name): + path = self.root / name + original = path.read_bytes() + try: + path.write_bytes(original + b"\n") + self.assertEqual(self.check_links(), 1) + finally: + path.write_bytes(original) diff --git a/docs/RUN_GUIDE.md b/docs/RUN_GUIDE.md index 8d06a04..158f117 100644 --- a/docs/RUN_GUIDE.md +++ b/docs/RUN_GUIDE.md @@ -14,7 +14,7 @@ $ python .github/scripts/gate.py --list | Arm | Command | What it checks | | --- | --- | --- | -| `links` | `python .github/scripts/gate.py links` | Relative Markdown link and image targets resolve inside the repository. | +| `links` | `python .github/scripts/gate.py links` | Relative Markdown link and image targets resolve inside the repository; Publishing artifact digests match. | | `python` | `python .github/scripts/gate.py python` | Tracked Python files compile under the running interpreter. | | `notebooks` | `python .github/scripts/gate.py notebooks` | Tracked notebooks are valid JSON with no committed cell outputs. | | `checkpoints` | `python .github/scripts/gate.py checkpoints` | No Jupyter checkpoint file is tracked. | @@ -30,6 +30,11 @@ implementation in `.github/scripts/archive_checks.py`; running `archive` reads `git ls-files` once instead of four times. `gnn` lives in `.github/scripts/gnn_conformance.py` and is the only arm that needs the network. +The `links` check also invokes the byte-identical Publishing checker with +`--verify-copy` for the schema, checker module, and GNN pin recorded in the +[member contract](workflows/publishing.md). This runs through `archive`, `fast`, +and `full` too; no additional gate arm or corpus is introduced. + ## Where the arms are measured [`gate-timings.md`](gate-timings.md) is generated; never edit it by hand. It diff --git a/docs/documentation-index.md b/docs/documentation-index.md index bc668c9..1df4e34 100644 --- a/docs/documentation-index.md +++ b/docs/documentation-index.md @@ -17,6 +17,8 @@ place. Generated pages are marked as such and are refreshed by a script. - [docs/index.md](index.md) — the same map rendered as guide and reference tables. - [docs/RUN_GUIDE.md](RUN_GUIDE.md) — gate entry point, arms, CI jobs, GNN pin. - [docs/gate-timings.md](gate-timings.md) — **generated** per-arm wall times. +- [docs/workflows/publishing.md](workflows/publishing.md) — member migration, canonical contract pointer, and artifact digests. +- [docs/workflows/record-identity.schema.json](workflows/record-identity.schema.json) — digest-pinned canonical Publishing identity schema. ## Guides diff --git a/docs/workflows/publishing.md b/docs/workflows/publishing.md new file mode 100644 index 0000000..56c29b2 --- /dev/null +++ b/docs/workflows/publishing.md @@ -0,0 +1,39 @@ +# Publishing — Knowledge-Engineering member + +The [Publishing contract](https://github.com/ActiveInferenceInstitute/ActiveInferenceJournal/blob/d8ec888a8f3051a66bed9af047ee4b206050bdce/docs/workflows/publishing.md) +in ActiveInferenceJournal owns the workflow and identity grammar. This member +points there rather than restating the schema or creating identities for historical +research exports. + +## Member mapping + +- Gate: `python .github/scripts/gate.py `; commands and arm descriptions live + in the [run guide](../RUN_GUIDE.md) and [sidecar tasks](../../.aii/config.yaml). +- Docs index: [docs/documentation-index.md](../documentation-index.md). +- Manuscript command: none. +- GNN pin: [.github/gnn-pin.json](../../.github/gnn-pin.json). + +The sidecar lists only documented gate-arm commands and uses existing sidecar +documentation paths. The archive's existing link, syntax, notebook, checkpoint, +test, timing and GNN checks remain authoritative for what they already prove, +as permitted by the contract; no parallel implementations are introduced. + +## Canonical copies and drift checks + +[identity.py](../../scripts/gate/identity.py) and +[record-identity.schema.json](record-identity.schema.json) are unchanged copies +from the lead revision linked above. The identity grammar is defined only in the +schema, not mirrored in prose here. The existing GNN pin is also digest-bound. + +```text +record-identity.schema.json 84f6c4df6efc3f15f0d8a9a05c8aabd0cbdc08b0f4feac5fcc8ae860a81e306b +identity.py 847328ff163fa90c869495164869caf4e83313748e8a444dbab8d4e459b11074 +gnn-pin.json eb83f7f1b73eb31e116bd05b1f457dd07e6874a2a4ba0e9e95e5bc91f094cd55 +``` + +The `links` arm invokes the canonical module's `--verify-copy` command for every +artifact above. `archive`, `fast`, and `full` include that check. Gate tests prove +that matching copies pass and changed copies fail. Checks are offline and +stdlib-only; no JSON Schema package, journal corpus, or fabricated record data is +required. These checks prove artifact agreement, not identity coverage of the +historical research exports. diff --git a/docs/workflows/record-identity.schema.json b/docs/workflows/record-identity.schema.json new file mode 100644 index 0000000..46096c1 --- /dev/null +++ b/docs/workflows/record-identity.schema.json @@ -0,0 +1,137 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ActiveInferenceInstitute/ActiveInferenceJournal/blob/main/docs/workflows/record-identity.schema.json", + "title": "Publishing record identity", + "schema_version": "record-identity/v1", + "description": "Canonical identity of one publishing record in the Active Inference Institute estate. The journal's metadata.json is the record; item, video_id and session ids are the join keys every other repository uses. Machine-readable rules are under x-identity, which scripts/gate/identity.py enforces, so this file is the only copy of the grammar.", + "type": "object", + "required": [ + "series", + "item", + "source", + "channel", + "category", + "episode", + "parts" + ], + "properties": { + "series": { + "type": "string", + "description": "Name of a directory above the item directory inside the corpus.", + "pattern": "^(?!\\.)[^/]{0,119}[^/\\s]$" + }, + "item": { + "type": "string", + "description": "Item directory name; the record's item id.", + "pattern": "^(?!\\.)[^/]{0,119}[^/\\s]$" + }, + "source": { + "type": "string", + "description": "Producer family of the content, e.g. youtube." + }, + "channel": { + "type": "string", + "description": "Channel or collection the source belongs to." + }, + "category": { + "type": "string", + "description": "Top-level category used by the human index." + }, + "episode": { + "type": "string", + "description": "Episode ordinal within the series, as text." + }, + "duplicate_of": { + "type": "string", + "description": "Record path of the canonical item this one duplicates, with the leading data/// removed.", + "pattern": "^[^/]+(/[^/]+)+$" + }, + "doi": { + "type": "string", + "description": "Paper DOI when the record is tied to a publication." + }, + "parts": { + "type": "array", + "description": "Video parts of the record; each part is one video id.", + "items": { + "type": "object", + "required": ["video_id", "url", "title"], + "properties": { + "video_id": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "url": { + "type": "string", + "format": "uri" + }, + "title": { + "type": "string" + } + } + } + } + }, + "x-identity": { + "record_path": "data////", + "derived_prefix_components": 3, + "leaf_key": "item", + "ancestor_key": "series", + "duplicate_key": "duplicate_of", + "index_file": "INDEX.json", + "index_record_keys": ["path", "series", "item", "parts", "has_transcript", "duplicate_of"], + "transcript_flag": { + "key": "has_transcript", + "artifact": "transcript.txt" + }, + "patterns": { + "video_id": "^[A-Za-z0-9_-]{11}$", + "session_id": "^(?P[A-Za-z0-9_-]{11})_sess[0-9]{2}$", + "paper_id": "^(?:doi|arxiv|s2|openalex|title):.+$" + }, + "namespaces": { + "item": { + "owner": "ActiveInferenceInstitute/ActiveInferenceJournal", + "home": "docs/workflows/record-identity.schema.json#/x-identity/leaf_key", + "carried_by": "metadata.json item + INDEX.json path", + "pattern": "item" + }, + "video": { + "owner": "ActiveInferenceInstitute/ActiveInferenceJournal", + "home": "docs/workflows/record-identity.schema.json#/properties/parts/items/properties/video_id", + "carried_by": "metadata.json parts[].video_id; INDEX.json parts[]", + "pattern": "video_id" + }, + "session": { + "owner": "ActiveInferenceInstitute/ActiveInferenceJournal", + "home": "docs/workflows/record-identity.schema.json#/x-identity/patterns/session_id", + "carried_by": "metadata.json sessions[].session_name; transcript.json blocks", + "pattern": "session_id" + }, + "paper": { + "owner": "ActiveInferenceInstitute/act_inf_metaanalysis", + "home": "src/literature/models.py Paper.canonical_id", + "carried_by": "corpus.jsonl paper_id; journal metadata.json doi", + "pattern": "paper_id" + }, + "doc": { + "owner": "ActiveInferenceInstitute/ISSS", + "home": "metadata_archiver.py document_name", + "carried_by": "archived_documents/metadata.json document_name", + "pattern": null + }, + "pub": { + "owner": "ActiveInferenceInstitute/act_inf_metaanalysis", + "home": "manuscript/config.yaml doi + version", + "carried_by": "manuscript/config.yaml; output/reports/zenodo_deposit_metadata.json", + "pattern": null + } + }, + "identity_free_artifacts": { + "captions/": "Filenames are verbatim source names (measured 2026-11-12: 421 of 624 begin youtube_captions, the rest are human titles). Never derive a record id from them.", + "translations/": "Filenames carry an id prefix for most files but not all (measured: 249 distinct prefixes, several are titles). Resolve the record through metadata.json first.", + "assets/": "Curated content, organised by type. Carries no identity." + }, + "join_rule": "Consumers join on video (video_id), documents on paper (doi), publications on pub (doi + version). An item id is only meaningful inside this corpus and must be paired with its path." + } +} diff --git a/scripts/gate/identity.py b/scripts/gate/identity.py new file mode 100644 index 0000000..c5dd98e --- /dev/null +++ b/scripts/gate/identity.py @@ -0,0 +1,346 @@ +"""Record identity for the publishing group: one schema, one grammar. + +``docs/workflows/record-identity.schema.json`` is the only copy of the identity +grammar (namespaces, patterns, the record path rule). This module enforces its +``x-identity`` block; nothing here restates a pattern, so editing the schema +changes behaviour instead of creating a second source of truth. + +The group contract, ``docs/workflows/publishing.md``, records the schema's +sha256. A member repository does not re-implement any of this: copy this module +and the schema, record their digest in the contract copy, and run +``--verify-copy`` before every release. The check fails as soon as the member's +copy and the digest the contract records disagree, which is the whole point of +the drift test. + +Stdlib only, so the same bytes run on a GitHub runner and on a laptop. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +SCHEMA_VERSION = "record-identity/v1" +# A pinned copy is any artifact kind a member can carry, not just a schema: the +# contract pins the Python modules too, and a `.json`-only parser would make +# those entries unverifiable. +ARTIFACT_SUFFIXES = ("json", "py", "mjs", "sh", "md", "ts") +DIGEST_LINE = re.compile( + r"^\s*([\w.+@-]+\.(?:" + "|".join(ARTIFACT_SUFFIXES) + r"))\s{2,}([0-9a-f]{64})\s*$" +) +# Where a recorded name may live, relative to the repository root. A fixed list +# keeps resolution deterministic: no directory walk, no ambiguity by accident. +ARTIFACT_LOCATIONS = ("scripts/gate", "docs/workflows", ".github", "") + + +class IdentityError(Exception): + """A contract or corpus problem with a reviewer-facing message.""" + + +@dataclass(frozen=True) +class Spec: + """The compiled ``x-identity`` block of the canonical schema.""" + + path: Path + document: dict + patterns: dict[str, re.Pattern[str]] + leaf_key: str + ancestor_key: str + duplicate_key: str + index_file: str + derived_prefix_components: int + transcript_flag: dict + + def digest(self) -> str: + return hashlib.sha256(self.path.read_bytes()).hexdigest() + + def relative(self, root: Path) -> str: + try: + return self.path.relative_to(root).as_posix() + except ValueError: + return str(self.path) + + +def _require(mapping: dict, key: str, where: str) -> object: + if key not in mapping: + raise IdentityError(f"{where} lacks `{key}`") + return mapping[key] + + +def load_schema(path: Path) -> Spec: + """Parse and compile the canonical schema, refusing an incomplete contract.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + raise IdentityError(f"missing identity schema {path}") from None + except json.JSONDecodeError as exc: + raise IdentityError(f"{path} is not valid JSON: {exc}") from None + if not isinstance(document, dict): + raise IdentityError(f"{path} must hold a JSON object") + + version = document.get("schema_version") + if version != SCHEMA_VERSION: + raise IdentityError( + f"{path}: schema_version is {version!r}, expected {SCHEMA_VERSION!r}" + ) + block = _require(document, "x-identity", f"{path}") + if not isinstance(block, dict): + raise IdentityError(f"{path}: x-identity must be an object") + + raw_patterns = _require(block, "patterns", f"{path}: x-identity") + patterns: dict[str, re.Pattern[str]] = {} + if not isinstance(raw_patterns, dict): + raise IdentityError(f"{path}: x-identity.patterns must be an object") + for name, pattern in raw_patterns.items(): + if not isinstance(pattern, str): + raise IdentityError(f"{path}: pattern `{name}` must be a string") + try: + patterns[name] = re.compile(pattern) + except re.error as exc: + raise IdentityError(f"{path}: pattern `{name}` is invalid: {exc}") from None + for required in ("video_id", "session_id"): + if required not in patterns: + raise IdentityError(f"{path}: x-identity.patterns lacks `{required}`") + + flag = _require(block, "transcript_flag", f"{path}: x-identity") + if not isinstance(flag, dict): + raise IdentityError(f"{path}: x-identity.transcript_flag must be an object") + + components = _require(block, "derived_prefix_components", f"{path}: x-identity") + if not isinstance(components, int) or components < 1: + raise IdentityError(f"{path}: derived_prefix_components must be a positive integer") + + return Spec( + path=path, + document=document, + patterns=patterns, + leaf_key=str(_require(block, "leaf_key", f"{path}: x-identity")), + ancestor_key=str(_require(block, "ancestor_key", f"{path}: x-identity")), + duplicate_key=str(_require(block, "duplicate_key", f"{path}: x-identity")), + index_file=str(_require(block, "index_file", f"{path}: x-identity")), + derived_prefix_components=components, + transcript_flag=flag, + ) + + +def recorded_digests(contract: Path) -> dict[str, str]: + """Read `` `` lines; the contract's digest table in one format.""" + try: + text = contract.read_text(encoding="utf-8") + except FileNotFoundError: + raise IdentityError(f"missing contract {contract}") from None + recorded: dict[str, str] = {} + for line in text.splitlines(): + match = DIGEST_LINE.match(line) + if match: + recorded[match.group(1)] = match.group(2) + return recorded + + +def repository_root(contract: Path) -> Path: + """The checkout a contract belongs to: nearest ancestor holding `.git`.""" + for parent in contract.resolve().parents: + if (parent / ".git").exists(): + return parent + return contract.resolve().parents[2] + + +def resolve_artifact(root: Path, name: str) -> Path | None: + """Find the file a recorded digest names, or None when it is not unique.""" + matches: list[Path] = [] + for location in ARTIFACT_LOCATIONS: + candidate = root / location / name if location else root / name + if candidate.is_file(): + matches.append(candidate) + return matches[0] if len(matches) == 1 else None + + +def check_contract(contract: Path, schema: Spec) -> list[str]: + """Every recorded digest must match its file; the schema must be recorded.""" + failures: list[str] = [] + recorded = recorded_digests(contract) + if not recorded: + failures.append( + f"{contract.name}: no `name sha256` digest block; members cannot verify a copy" + ) + return failures + root = repository_root(contract) + for name, expected in sorted(recorded.items()): + path = schema.path if name == schema.path.name else resolve_artifact(root, name) + if path is None: + failures.append( + f"{contract.name}: records {name}, which is not one file under " + f"{root} at {', '.join(location or '.' for location in ARTIFACT_LOCATIONS)}" + ) + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != expected: + failures.append( + f"{contract.name}: records {name} as {expected[:12]}, " + f"but the file is {digest[:12]}; update the digest with the edit" + ) + if schema.path.name not in recorded: + failures.append(f"{contract.name}: records no digest for {schema.path.name}") + return failures + + +def verify_copy(copy: Path, contract: Path) -> str | None: + """Return a failure message when a member's artifact copy diverges.""" + try: + digest = hashlib.sha256(copy.read_bytes()).hexdigest() + except FileNotFoundError: + return f"{copy}: no such file" + recorded = recorded_digests(contract) + expected = recorded.get(copy.name) + if expected is None: + return f"{contract}: records no digest for {copy.name}" + if expected != digest: + return ( + f"{copy}: copy is {digest[:12]}, contract records {expected[:12]}; " + "the copy drifted from the canonical artifact" + ) + return None + + +def check_corpus(root: Path, spec: Spec) -> tuple[list[str], dict[str, int]]: + """Enforce the identity grammar over one corpus; return failures and counts.""" + failures: list[str] = [] + counts = {"records": 0, "parts": 0, "duplicates": 0} + index_path = root / spec.index_file + try: + index = json.loads(index_path.read_text(encoding="utf-8")) + except FileNotFoundError: + return [f"missing {spec.index_file} under {root}"], counts + except json.JSONDecodeError as exc: + return [f"{spec.index_file} is not valid JSON: {exc}"], counts + + records = index.get("items") if isinstance(index, dict) else None + if not isinstance(records, list): + return [f"{spec.index_file}: `items` must be a list"], counts + + known_paths = { + str(item.get("path", "")) for item in records if isinstance(item, dict) + } + seen: set[str] = set() + for position, record in enumerate(records): + if not isinstance(record, dict): + failures.append(f"{spec.index_file}: items[{position}] is not an object") + continue + relative = str(record.get("path", "")) + if not relative: + failures.append(f"{spec.index_file}: items[{position}] has no path") + continue + if relative in seen: + failures.append(f"{spec.index_file}: duplicate record path {relative}") + continue + seen.add(relative) + record_dir = root / relative + metadata_path = record_dir / "metadata.json" + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except FileNotFoundError: + failures.append(f"{relative}: no metadata.json") + continue + except json.JSONDecodeError as exc: + failures.append(f"{relative}: metadata.json is not valid JSON: {exc}") + continue + if not isinstance(metadata, dict): + failures.append(f"{relative}: metadata.json must hold an object") + continue + + counts["records"] += 1 + leaf = metadata.get(spec.leaf_key) + if leaf != record_dir.name: + failures.append( + f"{relative}: {spec.leaf_key} is {leaf!r}, but the directory is " + f"{record_dir.name!r}; the item id must name its own directory" + ) + ancestors = {parent.name for parent in record_dir.parents if parent.name} + series = metadata.get(spec.ancestor_key) + if series not in ancestors: + failures.append( + f"{relative}: {spec.ancestor_key} is {series!r}, which names no " + "directory above the item" + ) + + parts = metadata.get("parts") + if not isinstance(parts, list): + failures.append(f"{relative}: `parts` must be a list") + continue + for position, part in enumerate(parts): + if not isinstance(part, dict): + failures.append(f"{relative}: parts[{position}] is not an object") + continue + video_id = part.get("video_id") + counts["parts"] += 1 + if not isinstance(video_id, str) or not spec.patterns["video_id"].match(video_id): + failures.append( + f"{relative}: parts[{position}].video_id is {video_id!r}, which is " + "not an 11-character YouTube id" + ) + + duplicate = metadata.get(spec.duplicate_key) + if duplicate is not None: + counts["duplicates"] += 1 + if not isinstance(duplicate, str): + failures.append(f"{relative}: {spec.duplicate_key} must be a string") + else: + prefix = "/".join(relative.split("/")[: spec.derived_prefix_components]) + target = f"{prefix}/{duplicate}" + if target not in known_paths: + failures.append( + f"{relative}: {spec.duplicate_key} {duplicate!r} resolves to " + f"{target}, which is not a record path" + ) + return failures, counts + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--schema", default="docs/workflows/record-identity.schema.json") + parser.add_argument("--contract", default="docs/workflows/publishing.md") + parser.add_argument("--corpus", help="corpus root holding INDEX.json (default: none)") + parser.add_argument("--verify-copy", metavar="PATH", help="check a member's schema copy") + parser.add_argument("--digest", action="store_true", help="print the schema sha256") + args = parser.parse_args(argv) + + schema_path = Path(args.schema) + try: + spec = load_schema(schema_path) + except IdentityError as exc: + print(f"identity: {exc}", file=sys.stderr) + return 1 + + if args.digest: + print(f"{schema_path.name} {spec.digest()}") + return 0 + + if args.verify_copy: + problem = verify_copy(Path(args.verify_copy), Path(args.contract)) + if problem: + print(f"identity: {problem}", file=sys.stderr) + return 1 + print(f"identity: {args.verify_copy} matches {Path(args.contract).name}") + return 0 + + failures = check_contract(Path(args.contract), spec) + counts: dict[str, int] = {} + if args.corpus: + corpus_failures, counts = check_corpus(Path(args.corpus), spec) + failures += corpus_failures + for message in failures: + print(f"identity: FAIL {message}", file=sys.stderr) + if failures: + return 1 + suffix = "".join(f", {value} {name}" for name, value in counts.items()) + print(f"identity: ok ({schema_path.name}{suffix})") + return 0 + + +if __name__ == "__main__": + sys.exit(main())