From 414fb0eecdb7f6236d37b3704657c732a97c408c Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:52:40 +0800 Subject: [PATCH 1/2] facts: add a read-only engine README plan scripts/facts_engine_plan.py plans an update of the engine README's fact markers without writing anything remote. It reads the hub's main commit, facts/facts.json, facts/facts.schema.json and VERSION through GitHub's public REST API (GET only, no redirects, no proxies), selects the release named by VERSION (scripts/facts_engine_release.py), resolves the engine's release tag to its commit, renders the engine README through the pure fact-marker renderer (scripts/facts_markdown.mjs, run by the Node 20 bridge scripts/facts_engine_render.mjs), and checks any open proposal pull request by its plan identity. The output is a local review directory: plan.json and, when the README changes, the proposed README. The plan is marked superseded when the hub's main commit, the engine's default branch or its commit moved during the run. tests/test_facts_engine_plan.py runs the planner against loopback HTTP servers and the real Node 20 bridge: 33 tests covering the create, noop, already-planned, update-needed and superseded outcomes and the refusals. --- scripts/facts_engine_plan.py | 455 ++++++++++++++++++++ scripts/facts_engine_release.py | 56 +++ scripts/facts_engine_render.mjs | 49 +++ scripts/facts_markdown.mjs | 180 ++++++++ tests/test_facts_engine_plan.py | 742 ++++++++++++++++++++++++++++++++ 5 files changed, 1482 insertions(+) create mode 100644 scripts/facts_engine_plan.py create mode 100644 scripts/facts_engine_release.py create mode 100644 scripts/facts_engine_render.mjs create mode 100644 scripts/facts_markdown.mjs create mode 100644 tests/test_facts_engine_plan.py diff --git a/scripts/facts_engine_plan.py b/scripts/facts_engine_plan.py new file mode 100644 index 0000000..733c8e9 --- /dev/null +++ b/scripts/facts_engine_plan.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Build a public, read-only proposal for the engine README.""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import subprocess +import sys +from http.client import HTTPException +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, quote, urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +from facts_engine_release import select_engine_release +from facts_validate import (FACTS_SCHEMA, MAX_INPUT_BYTES, InputError, decode, digest, + read_bytes, require) + +HUB = "pineforge-4pass/pineforge-release" +ENGINE = "pineforge-4pass/pineforge-engine" +OWNER = "pineforge-4pass" +SCHEMA = "pineforge/engine-readme-plan/v1" +API_ORIGIN = "https://api.github.com" +BRIDGE = Path(__file__).with_name("facts_engine_render.mjs") +COMMIT = re.compile(r"[0-9a-f]{40}\Z") +IDENTITY = re.compile(r"[0-9a-f]{64}\Z") +MACHINE_MARKER = re.compile(r"") +GENERATION_LIMIT = ( + "Changed facts under the same release tag produce a different identity and branch. " + "A separately authorized publisher must settle superseded open generations before activation." +) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True, allow_nan=False).encode("utf-8") + + +def commit(value): + require(type(value) is str and COMMIT.fullmatch(value), "invalid commit identity") + return value + + +def branch(value): + require(type(value) is str + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}", value) + and not any(part in value for part in ("..", "//", "@{")) + and all(part and not part.startswith(".") and not part.endswith((".", ".lock")) + for part in value.split("/")), "unsafe Git ref name") + return value + + +def repository(value, expected, expected_id=None): + require(type(value) is dict and expected in (HUB, ENGINE), "invalid repository metadata") + owner, name = expected.split("/") + require(value.get("private") is False and value.get("full_name") == expected + and value.get("name") == name and type(value.get("owner")) is dict + and value["owner"].get("login") == owner + and type(value.get("id")) is int and value["id"] > 0 + and value.get("html_url") == "https://github.com/" + expected + and value.get("url") == API_ORIGIN + "/repos/" + expected + and value.get("visibility", "public") == "public", + "exact public repository was not confirmed") + if expected_id is not None: + require(value["id"] == expected_id, "repository numeric identity changed") + branch(value.get("default_branch")) + return value + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise InputError("GitHub redirect refused") + + +class PublicGitHub: + """A dedicated GET transport; no write credential or mutation interface.""" + + def __init__(self, test_api=None): + self.origin = API_ORIGIN + self.token = os.environ.get("FACTS_READ_TOKEN", "") + require(len(self.token) <= 4096 and (not self.token or + re.fullmatch(r"[A-Za-z0-9_.-]+", self.token)), "invalid read credential") + if test_api is not None: + try: + parsed = urlsplit(test_api) + valid = (parsed.scheme == "http" and parsed.hostname == "127.0.0.1" + and parsed.port is not None and 1 <= parsed.port <= 65535 + and parsed.netloc == "127.0.0.1:" + str(parsed.port) + and not parsed.path and not parsed.query and not parsed.fragment) + except ValueError: + valid = False + require(valid, "test API must be an explicit loopback origin") + require(not self.token or self.token.startswith("test-"), + "real credentials are forbidden on the test transport") + self.origin = test_api + self.opener = build_opener(ProxyHandler({}), NoRedirect()) + + def _get(self, path): + require(type(path) is str and len(path) <= 4096 + and any(path == "/repos/" + repo or path.startswith("/repos/" + repo + "/") + for repo in (HUB, ENGINE)) + and not any(char in path for char in ("\r", "\n", "\\", "#")) + and ".." not in path and "//" not in path, "invalid public API path") + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "pineforge-engine-readme-plan", "Cache-Control": "no-cache", + "Accept-Encoding": "identity"} + if self.token: + headers["Authorization"] = "Bearer " + self.token + try: + with self.opener.open(Request(self.origin + path, headers=headers, method="GET"), + timeout=30) as response: + require(response.status == 200, "unexpected GitHub response status") + require(response.headers.get("Content-Encoding", "identity") == "identity", + "encoded GitHub response refused") + length = response.headers.get("Content-Length") + require(length is None or (re.fullmatch(r"[0-9]{1,10}", length) + and int(length) <= MAX_INPUT_BYTES), "GitHub response exceeds size limit") + raw = response.read(MAX_INPUT_BYTES + 1) + require(length is None or len(raw) == int(length), "truncated GitHub response") + links = response.headers.get("Link", "") + require(len(links) <= 8192, "oversize pagination metadata") + return decode(raw), links + except HTTPError as error: + error.close() + raise InputError("GitHub GET failed") from None + except (URLError, OSError, TimeoutError, HTTPException): + raise InputError("GitHub GET unavailable") from None + + def get(self, path): + return self._get(path)[0] + + def public_repository(self, name, expected_id=None): + require(name in (HUB, ENGINE), "repository is outside the public plan") + return repository(self.get("/repos/" + name), name, expected_id) + + def ref(self, repo, kind, name): + require(repo in (HUB, ENGINE) and kind in ("heads", "tags"), "invalid ref role") + branch(name) + value = self.get(f"/repos/{repo}/git/ref/{kind}/{quote(name, safe='')}") + require(type(value) is dict and value.get("ref") == f"refs/{kind}/{name}", + "Git ref identity differs") + obj = value.get("object") + require(type(obj) is dict and obj.get("type") in ("commit", "tag"), "invalid Git ref object") + commit(obj.get("sha")) + if kind == "heads": + require(obj["type"] == "commit", "branch does not identify a commit") + return obj + + def branch_commit(self, repo, name): + return self.ref(repo, "heads", name)["sha"] + + def tag_commit(self, release_tag): + obj = self.ref(ENGINE, "tags", release_tag) + seen = set() + depth = 0 + while obj["type"] == "tag": + sha = commit(obj["sha"]) + require(sha not in seen and depth < 8, "tag cycle or depth limit") + seen.add(sha) + value = self.get(f"/repos/{ENGINE}/git/tags/{sha}") + require(type(value) is dict and value.get("sha") == sha + and type(value.get("tag")) is str, "annotated tag identity differs") + branch(value["tag"]) + obj = value.get("object") + require(type(obj) is dict and obj.get("type") in ("commit", "tag"), + "invalid annotated tag target") + commit(obj.get("sha")) + depth += 1 + return commit(obj["sha"]) + + def contents(self, repo, sha, name): + commit(sha) + require((repo == HUB and name in ("facts/facts.json", "facts/facts.schema.json", "VERSION")) + or (repo == ENGINE and name == "README.md"), "invalid immutable content role") + value = self.get(f"/repos/{repo}/contents/{name}?ref={sha}") + require(type(value) is dict and value.get("type") == "file" + and value.get("path") == name and value.get("name") == name.rsplit("/", 1)[-1] + and value.get("encoding") == "base64" and type(value.get("content")) is str + and type(value.get("size")) is int and 0 <= value["size"] <= MAX_INPUT_BYTES, + "invalid Contents envelope") + try: + raw = base64.b64decode(value["content"].replace("\n", ""), validate=True) + except (ValueError, UnicodeError): + raise InputError("invalid Contents encoding") from None + require(len(raw) == value["size"] and len(raw) <= MAX_INPUT_BYTES, "Contents size differs") + blob = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() + require(value.get("sha") == blob, "Contents Git blob digest differs") + return raw + + def pull_page(self, head, base, page): + require(type(page) is int and 1 <= page <= 20, "pull request pagination limit") + query = {"state": "open", "head": OWNER + ":" + branch(head), "base": branch(base), + "per_page": "100", "page": str(page)} + path = f"/repos/{ENGINE}/pulls" + rows, links = self._get(path + "?" + urlencode(query)) + require(type(rows) is list and len(rows) <= 100, "invalid pull request page") + relations = {} + if links: + for part in links.split(","): + match = re.fullmatch(r'\s*<([^<>]+)>;\s*rel="(next|last|first|prev)"\s*', part) + require(match is not None and match[2] not in relations, "invalid pagination metadata") + url = urlsplit(match[1]) + expected = urlsplit(self.origin) + params = parse_qs(url.query, keep_blank_values=True) + require(url.scheme == expected.scheme and url.netloc == expected.netloc + and url.path == path and not url.fragment + and set(params) == set(query) + and all(params[key] == [value] for key, value in query.items() if key != "page") + and len(params["page"]) == 1 + and re.fullmatch(r"[1-9][0-9]*", params["page"][0]), + "foreign or malformed pagination target") + number = int(params["page"][0]) + require(1 <= number <= 20, "pull request pagination limit") + relations[match[2]] = number + require("next" not in relations or relations["next"] == page + 1, + "discontinuous pull request pagination") + require("first" not in relations or relations["first"] == 1, + "invalid first pagination page") + require("prev" not in relations or relations["prev"] == page - 1, + "invalid previous pagination page") + require("last" not in relations or relations["last"] >= page, + "invalid last pagination page") + require("last" not in relations or relations["last"] <= page or "next" in relations, + "truncated pull request pagination") + require(not (len(rows) < 100 and "next" in relations), "truncated pull request page") + return rows + + def pull(self, number): + require(type(number) is int and number > 0, "invalid pull request number") + return self.get(f"/repos/{ENGINE}/pulls/{number}") + + +def utf8(raw): + require(type(raw) is bytes and len(raw) <= MAX_INPUT_BYTES, "invalid README bytes") + try: + text = raw.decode("utf-8") + require(text.encode("utf-8") == raw, "README UTF-8 roundtrip differs") + return text + except UnicodeError: + raise InputError("README is not UTF-8") from None + + +def checked_render(value): + require(type(value) is dict and set(value) == {"text", "markers"} + and type(value["text"]) is str and type(value["markers"]) is int + and 0 < value["markers"] <= MAX_INPUT_BYTES, "invalid or empty renderer result") + try: + raw = value["text"].encode("utf-8") + except UnicodeError: + raise InputError("invalid renderer UTF-8") from None + utf8(raw) + return raw + + +def render_with_node(text, facts): + """Execute the local Node 20 bridge; renderer errors never qualify as a plan.""" + payload = canonical({"text": text, "facts": facts}) + require(len(payload) <= MAX_INPUT_BYTES, "renderer request exceeds size limit") + try: + result = subprocess.run(["node", "--max-old-space-size=128", str(BRIDGE)], input=payload, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30, + check=False, shell=False, + env={"PATH": os.environ.get("PATH", os.defpath)}) + except (OSError, subprocess.SubprocessError): + raise InputError("engine README renderer unavailable") from None + require(result.returncode == 0 and not result.stderr, "engine README renderer failed") + value = decode(result.stdout) + checked_render(value) + return value + + +def pull_identity(value, repo_id, head, base, plan_identity): + require(type(value) is dict and type(value.get("id")) is int and value["id"] > 0 + and type(value.get("number")) is int and value["number"] > 0 + and value.get("state") == "open", "invalid open pull request") + require(value.get("html_url") == f'https://github.com/{ENGINE}/pull/{value["number"]}' + and value.get("url") == f'{API_ORIGIN}/repos/{ENGINE}/pulls/{value["number"]}', + "pull request public URL differs") + for side, expected in (("head", head), ("base", base)): + endpoint = value.get(side) + require(type(endpoint) is dict and endpoint.get("ref") == expected + and endpoint.get("label") == OWNER + ":" + expected, + "pull request branch differs") + repository(endpoint.get("repo"), ENGINE, repo_id) + commit(endpoint.get("sha")) + body = value.get("body") + require(type(body) is str, "pull request body is unavailable") + try: + require(len(body.encode("utf-8")) <= MAX_INPUT_BYTES, "pull request body exceeds size limit") + except UnicodeError: + raise InputError("invalid pull request body UTF-8") from None + markers = list(MACHINE_MARKER.finditer(body)) + require(len(markers) == 1 and markers[0][1] == plan_identity + and body.count("pineforge:engine-readme-plan") == 1, + "foreign or corrupt pull request machine identity") + return value + + +def existing_pull(api, repo_id, head, base, plan_identity): + matches = [] + seen_ids, seen_numbers = set(), set() + for page in range(1, 21): + rows = api.pull_page(head, base, page) + for row in rows: + pull_identity(row, repo_id, head, base, plan_identity) + require(row["id"] not in seen_ids and row["number"] not in seen_numbers, + "duplicate pull request result") + seen_ids.add(row["id"]) + seen_numbers.add(row["number"]) + matches.append(row) + if len(rows) < 100: + break + else: + raise InputError("pull request pagination limit") + require(len(matches) <= 1, "ambiguous open pull requests") + if not matches: + return None + listed = matches[0] + current = pull_identity(api.pull(listed["number"]), repo_id, head, base, plan_identity) + require(current["id"] == listed["id"] and current["number"] == listed["number"], + "pull request identity changed") + return current + + +def build_engine_plan(api, renderer): + """Return a JSON-safe review artifact. This function has no remote write path.""" + hub_repo = api.public_repository(HUB) + target_repo = api.public_repository(ENGINE) + hub_commit = api.branch_commit(HUB, "main") + facts_raw = api.contents(HUB, hub_commit, "facts/facts.json") + schema_raw = api.contents(HUB, hub_commit, "facts/facts.schema.json") + version_raw = api.contents(HUB, hub_commit, "VERSION") + require(schema_raw == read_bytes(FACTS_SCHEMA), "source schema differs from the local schema") + selection = select_engine_release(facts_raw, version_raw, hub_commit=hub_commit, + expected_facts_sha256=digest(facts_raw)) + tag_commit = api.tag_commit(selection.release_tag) + base = target_repo["default_branch"] + target_commit = api.branch_commit(ENGINE, base) + before = api.contents(ENGINE, target_commit, "README.md") + rendered = renderer(utf8(before), decode(facts_raw)) + after = checked_render(rendered) + identity_input = {"repository_role": "engine", "repository": ENGINE, + "release_tag": selection.release_tag, "facts_sha256": selection.facts_sha256} + plan_identity = digest(canonical(identity_input)) + require(IDENTITY.fullmatch(plan_identity), "invalid plan identity") + head = "automation/facts-engine-" + plan_identity[:24] + require(head != base, "proposal branch equals target branch") + binding = {"plan_identity": plan_identity, "repository": ENGINE, + "hub_commit": hub_commit, "facts_sha256": selection.facts_sha256, + "version": selection.release_version, "version_sha256": digest(version_raw), + "release_tag": selection.release_tag, "tag_commit": tag_commit, + "base": base, "base_commit": target_commit, "head": OWNER + ":" + head, + "readme_before_sha256": digest(before), "readme_after_sha256": digest(after)} + body = (f"\n\n" + "Update the existing engine README fact markers from the selected public release facts.\n\n" + "```json\n" + json.dumps(binding, sort_keys=True, indent=2) + "\n```\n\n" + + GENERATION_LIMIT + "\n") + expected_body_sha = digest(body.encode("utf-8")) + current = existing_pull(api, target_repo["id"], head, base, plan_identity) + existing = None + equal = False + if current is not None: + current_head = api.branch_commit(ENGINE, head) + current_readme = api.contents(ENGINE, current_head, "README.md") + body_sha = digest(current["body"].encode("utf-8")) + equal = (current_head == current["head"]["sha"] and current["base"]["sha"] == target_commit + and digest(current_readme) == digest(after) and body_sha == expected_body_sha) + existing = {"id": current["id"], "number": current["number"], "url": current["html_url"], + "head_commit": current_head, "reported_head_commit": current["head"]["sha"], + "readme_sha256": digest(current_readme), "body_sha256": body_sha} + changed = before != after + status = ("noop" if not changed else "create-needed" if current is None + else "already-planned" if equal else "update-needed") + plan = {"schema": SCHEMA, "status": status, "plan_identity": plan_identity, + "identity_input": identity_input, + "snapshot": {"repository": HUB, "repository_id": hub_repo["id"], "ref": "main", + "commit": hub_commit, "facts_sha256": selection.facts_sha256, + "schema_sha256": digest(schema_raw), "version": selection.release_version, + "version_sha256": digest(version_raw), + "version_bytes_base64": base64.b64encode(version_raw).decode("ascii")}, + "release": {"tag": selection.release_tag, "commit": tag_commit, + "source_url": selection.source_url}, + "target": {"repository": ENGINE, "repository_id": target_repo["id"], "base": base, + "base_commit": target_commit, "head": head}, + "readme": {"path": "README.md", "before_sha256": digest(before), + "after_sha256": digest(after), "markers": rendered["markers"], "changed": changed}, + "limitations": [GENERATION_LIMIT], + "review_only": True, "remote_writes": 0} + if existing is not None: + plan["existing_pr"] = existing + # The last source reads precede emission. A changed default branch also invalidates the plan. + observed_hub = api.branch_commit(HUB, "main") + observed_repo = api.public_repository(ENGINE, target_repo["id"]) + observed_base = observed_repo["default_branch"] + observed_target = api.branch_commit(ENGINE, observed_base) + plan["recheck"] = {"hub_commit": observed_hub, "base": observed_base, "base_commit": observed_target} + if observed_hub != hub_commit or observed_base != base or observed_target != target_commit: + plan["status"] = "superseded" + elif changed: + plan["proposed_readme"] = utf8(after) + plan["proposed_pr"] = {"title": "Update engine README facts for " + selection.release_tag, + "body": body, "body_sha256": expected_body_sha} + return plan + + +def write_plan(directory, plan): + """Publish local review files only after every source check has completed.""" + payloads = {} + if "proposed_readme" in plan: + payloads["README.md"] = plan["proposed_readme"].encode("utf-8") + manifest = {key: value for key, value in plan.items() if key != "proposed_readme"} + payloads["plan.json"] = json.dumps(manifest, sort_keys=True, indent=2, + ensure_ascii=True, allow_nan=False).encode("utf-8") + b"\n" + # mkdir and exclusive opens protect every pre-existing output, including symlinks. + directory.mkdir() + created = [] + try: + for name, raw in payloads.items(): + path = directory / name + with path.open("xb") as stream: + created.append(path) + stream.write(raw) + except OSError: + for path in reversed(created): + path.unlink() + directory.rmdir() + raise + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("plan", allow_abbrev=False) + command.add_argument("--output-dir", type=Path, required=True) + command.add_argument("--test-api") + args = parser.parse_args(argv) + try: + require(not os.path.lexists(args.output_dir), "output directory already exists") + plan = build_engine_plan(PublicGitHub(args.test_api), render_with_node) + write_plan(args.output_dir, plan) + print(json.dumps({"schema": SCHEMA, "status": plan["status"], + "plan_identity": plan["plan_identity"], "review_only": True, + "remote_writes": 0}, sort_keys=True)) + return 0 + except (InputError, OSError, ValueError, KeyError, TypeError, RecursionError, OverflowError): + print("engine-readme-plan: input unavailable or refused; no remote changes", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/facts_engine_release.py b/scripts/facts_engine_release.py new file mode 100644 index 0000000..a642106 --- /dev/null +++ b/scripts/facts_engine_release.py @@ -0,0 +1,56 @@ +"""Select the engine README release identified by VERSION and public facts.""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +import facts_validate +from facts_validate import InputError, require + + +@dataclass(frozen=True) +class EngineReleaseSelection: + hub_commit: str + facts_sha256: str + release_version: str + release_tag: str + source_url: str + + +def select_engine_release( + facts_raw: bytes, + version_raw: bytes, + *, + hub_commit: str, + expected_facts_sha256: str, +) -> EngineReleaseSelection: + """Bind the exact VERSION release to the verified original facts bytes.""" + require(type(facts_raw) is bytes, "facts input must be bytes") + require(type(version_raw) is bytes, "VERSION input must be bytes") + require(type(hub_commit) is str + and re.fullmatch(r"[0-9a-f]{40}", hub_commit) is not None, + "invalid hub commit") + require(type(expected_facts_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", expected_facts_sha256) is not None, + "invalid expected facts SHA-256") + + facts_sha256 = facts_validate.digest(facts_raw) + require(facts_sha256 == expected_facts_sha256, "facts SHA-256 mismatch") + facts = facts_validate.validate_facts(facts_validate.decode(facts_raw)) + + require(len(version_raw) <= 256, "VERSION exceeds size limit") + version_bytes = version_raw[:-1] if version_raw.endswith(b"\n") else version_raw + require(re.fullmatch(rb"[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?", + version_bytes) is not None, "invalid VERSION") + version = version_bytes.decode("ascii") + try: + release = facts["releases"][version] + except KeyError: + raise InputError("selected release is missing") from None + return EngineReleaseSelection( + hub_commit=hub_commit, + facts_sha256=facts_sha256, + release_version=version, + release_tag="v" + version, + source_url=release["source"], + ) diff --git a/scripts/facts_engine_render.mjs b/scripts/facts_engine_render.mjs new file mode 100644 index 0000000..b87078b --- /dev/null +++ b/scripts/facts_engine_render.mjs @@ -0,0 +1,49 @@ +// Stdin/stdout bridge to the existing pure README renderer. Requires Node 20. +import { renderDocument } from './facts_markdown.mjs'; + +const LIMIT = 2 * 1024 * 1024; +const fail = () => { throw new Error('invalid engine README render request'); }; +const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +function utf8(text) { + if (typeof text !== 'string') fail(); + const bytes = Buffer.from(text, 'utf8'); + if (bytes.length > LIMIT || bytes.toString('utf8') !== text) fail(); + return bytes; +} + +function jsonTypes(value, depth = 0) { + if (depth > 64) fail(); + if (typeof value === 'string') { utf8(value); return; } + if (value === null || typeof value === 'boolean') return; + if (typeof value === 'number') { if (!Number.isFinite(value)) fail(); return; } + if (!object(value) && !Array.isArray(value)) fail(); + for (const [key, child] of Object.entries(value)) { + utf8(key); + jsonTypes(child, depth + 1); + } +} + +try { + if (process.versions.node.split('.')[0] !== '20') fail(); + const chunks = []; + let length = 0; + for await (const chunk of process.stdin) { + length += chunk.length; + if (length > LIMIT) fail(); + chunks.push(chunk); + } + const raw = Buffer.concat(chunks); + const input = JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(raw)); + if (!object(input) || Object.keys(input).sort().join(',') !== 'facts,text' || !object(input.facts)) fail(); + utf8(input.text); + jsonTypes(input.facts); + const result = renderDocument(input.text, input.facts, 'README.md'); + if (!object(result) || Object.keys(result).sort().join(',') !== 'markers,text' + || !Number.isSafeInteger(result.markers) || result.markers < 1) fail(); + utf8(result.text); + const output = utf8(JSON.stringify({ text: result.text, markers: result.markers })); + process.stdout.write(output); +} catch { + process.stderr.write('engine-readme-render: invalid input or render failure\n'); + process.exitCode = 2; +} diff --git a/scripts/facts_markdown.mjs b/scripts/facts_markdown.mjs new file mode 100644 index 0000000..5ac3c48 --- /dev/null +++ b/scripts/facts_markdown.mjs @@ -0,0 +1,180 @@ +// Pure fact-marker rendering. Callers supply facts and handle document persistence. +const integer = new Intl.NumberFormat('en-US', { maximumFractionDigits: 0 }); +const count = (value, name) => { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`invalid count: ${name}`); + return value; +}; + +export function tokenValue(facts, token) { + const segments = [...token.matchAll(/(?:\[([^\]]+)\]|([^.[\]]+))/g)].map((match) => match[1] ?? match[2]); + let value = facts; + for (const segment of segments) { + if (!value || !Object.hasOwn(value, segment) || ['__proto__', 'prototype', 'constructor'].includes(segment)) throw new Error(`unknown fact token: ${token}`); + value = value[segment]; + } + return value; +} + +function formatToken(value, format = 'text') { + if (format === 'lanes' || format === 'lanes-all') { + const allMarkets = format === 'lanes-all'; + if (allMarkets && (!value || typeof value !== 'object' || Array.isArray(value) || !Array.isArray(value.pairs) + || !value.otherMarkets || typeof value.otherMarkets !== 'object' || Array.isArray(value.otherMarkets))) { + throw new TypeError('lanes-all format requires a scoreboard object with pairs and otherMarkets'); + } + const pairs = allMarkets ? value.pairs : value; + if (!Array.isArray(pairs)) throw new Error('lanes format requires an array'); + const fields = ['graded', 'excellent', 'strong', 'belowStrong']; + const other = allMarkets ? value.otherMarkets : null; + if (allMarkets) { + count(other.markets, 'otherMarkets.markets'); + count(other.lanes, 'otherMarkets.lanes'); + for (const field of fields) { + const aggregate = count(other[field], `otherMarkets.${field}`); + const computed = pairs.reduce((sum, lane) => sum + count(lane[field], field), 0) + aggregate; + if (computed !== count(value[field], field) || (other.lanes === 0 && aggregate !== 0)) { + throw new RangeError(`lanes-all total mismatch: ${field}`); + } + } + } + const header = '| Market · timeframe | Probes graded | Excellent | Strong | Below strong |\n|---|---:|---:|---:|---:|'; + const ordered = [...pairs].sort((left, right) => Number(right.hardProbes > 0) - Number(left.hardProbes > 0)); + const rows = ordered.map((lane) => `| ${lane.symbol} · ${lane.timeframe === '15' ? '15m' : lane.timeframe}${lane.hardProbes > 0 ? ' *(hard lane: zero regression allowed)*' : ''} | ${integer.format(lane.graded)} | ${integer.format(lane.excellent)} | ${integer.format(lane.strong)} | ${integer.format(lane.belowStrong)} |`); + if (!allMarkets && !rows.length) rows.push(''); // Preserve the legacy empty table's bytes. + if (allMarkets && other.lanes > 0) { + rows.push(`| All other markets (${integer.format(other.markets)} markets, ${integer.format(other.lanes)} lanes) | ${fields.map((field) => integer.format(other[field])).join(' | ')} |`); + } + const total = fields.map((field) => `**${integer.format(allMarkets ? value[field] : pairs.reduce((sum, lane) => sum + lane[field], 0))}**`).join(' | '); + rows.push(`| **Total** | ${total} |`); + return `\n\n${header}\n${rows.join('\n')}\n\n`; + } + if (format === 'trades-badge') { count(value, 'TV trades'); return `[![Trades](https://img.shields.io/badge/TradingView%20trades%20graded-${(value / 1_000_000).toFixed(2)}M-brightgreen)](#validation-scoreboard)`; } + if (format === 'parity-badge') { + const label = encodeURIComponent(`${integer.format(value.excellent + value.strong)} / ${integer.format(value.graded)} excellent or strong`); + return `[![Parity](https://img.shields.io/badge/TradingView%20parity-${label}-brightgreen)](#validation-scoreboard)`; + } + if (format === 'int') { count(value, 'token'); return integer.format(value); } + if (format === 'pct') { if (typeof value !== 'number') throw new Error('pct format requires a number'); return value.toFixed(2); } + if (format === 'short') { if (typeof value !== 'string') throw new Error('short format requires text'); return value.slice(0, 8); } + if (format === 'code' || format === 'short-code') { + const content = formatToken(value, format === 'short-code' ? 'short' : 'text'); + if (/[`\r\n]/.test(content)) throw new Error('code format requires single-line text without backticks'); + return `\`${content}\``; + } + if (format !== 'text') throw new Error(`unknown fact format: ${format}`); + if (typeof value !== 'string' && typeof value !== 'number') throw new Error('token requires a scalar or explicit format'); + return String(value); +} + +export function renderText(text, facts) { + let markers = 0; + const rendered = text.replace(/[\s\S]*?/g, + (block, token, separator, format, offset) => { + if (block.slice(block.indexOf('-->') + 3).includes('${formatToken(tokenValue(facts, token), format)}`; + }); + if ((text.match(//g) ?? []).length !== markers) throw new Error('malformed fact markers'); + return { text: rendered, markers }; +} + +const BLOCK_TAGS = new Set(('address article aside base basefont blockquote body caption center col colgroup dd details dialog dir div dl dt ' + + 'fieldset figcaption figure footer form frame frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu menuitem ' + + 'nav noframes ol optgroup option p param section source summary table tbody td tfoot th thead title tr track ul').split(' ')); +const ATTRIBUTE = String.raw`\s+[A-Za-z_:][\w.:-]*(?:\s*=\s*(?:[^\s"'=<>\x60]+|'[^']*'|"[^"]*"))?`; +const LONE_TAG = new RegExp(String.raw`^(?:<[A-Za-z][A-Za-z0-9-]*(?:${ATTRIBUTE})*\s*\/?>|<\/[A-Za-z][A-Za-z0-9-]*\s*>)\s*$`); +function htmlBlockStart(line, paragraph) { + const html = /^ {0,3}(<.*)$/.exec(line)?.[1]; + if (!html) return null; + if (/^<(?:pre|script|style|textarea)(?:[\s>]|$)/i.test(html)) return /<\/(?:pre|script|style|textarea)>/i; + if (html.startsWith('/; + if (html.startsWith('/; + if (/^/; + if (html.startsWith('/; + const tag = /^<\/?([A-Za-z][A-Za-z0-9-]*)(?:[\s>]|\/>|$)/.exec(html); + if (tag && BLOCK_TAGS.has(tag[1].toLowerCase())) return 'blank'; + return !paragraph && LONE_TAG.test(html) ? 'blank' : null; +} +// Leading whitespace with tabs expanded to the next multiple of four columns. +function expandIndent(text) { + return text.replace(/^[ \t]+/, (space) => { + let columns = 0; + for (const char of space) columns = char === '\t' ? columns + 4 - (columns % 4) : columns + 1; + return ' '.repeat(columns); + }); +} + +// A marker whose )/.test(leading)) { + const alone = /^)/.test(leading) && !leading.slice(leading.indexOf('-->') + 3).trim(); + const exempt = alone && (/^/.test(leading) || (lanes && leading.startsWith(''))); + if (!exempt) errors.push(`${file}:${index + 1}: marker starts a line: GitHub renders the line as raw HTML; keep text before it`); + } + for (const [marker, format] of line.matchAll(//g)) lanes = marker.startsWith('old trades.\r\n" + "Commit: old.\r\n" + "Release: old.\r\n" + "\r\nUnmarked **tail** without a final newline" +).encode("utf-8") +# Handwritten values from the public fixture, independent of the production renderer. +EXPECTED = README.replace(b"-->old trades.", b"-->4,776,328 trades.") +EXPECTED = EXPECTED.replace(b"-->old.\r\nRelease:", + b"-->`d1d18867`.\r\nRelease:") +EXPECTED = EXPECTED.replace( + b"-->old.\r\n\r\n", + b"-->https://github.com/pineforge-4pass/pineforge-engine/blob/v1.1.0/README.md#validation-scoreboard" + b".\r\n\r\n") + + +def encode(value): + return json.dumps(value, ensure_ascii=True, separators=(",", ":")).encode("utf-8") + + +def sha256(raw): + return hashlib.sha256(raw).hexdigest() + + +def metadata(name, number, default="trunk"): + owner, short = name.split("/") + return {"id": number, "name": short, "full_name": name, "owner": {"login": owner}, + "private": False, "visibility": "public", "default_branch": default, + "html_url": "https://github.com/" + name, "url": API_ORIGIN + "/repos/" + name} + + +def envelope(name, raw): + return {"type": "file", "path": name, "name": name.rsplit("/", 1)[-1], "size": len(raw), + "encoding": "base64", "content": base64.b64encode(raw).decode("ascii"), + "sha": hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest()} + + +class EnginePlanHTTP(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + self.calls = [] + self.counts = {} + self.outputs = 0 + self.hub = "a" * 40 + self.target = "b" * 40 + self.head = "d" * 40 + self.hub_moved = False + self.target_moved = False + self.default_moved = False + self.repos = {HUB: metadata(HUB, 11, "main"), ENGINE: metadata(ENGINE, 22)} + self.files = {"facts/facts.json": FACTS.read_bytes(), + "facts/facts.schema.json": (ROOT / "facts/facts.schema.json").read_bytes(), + "VERSION": b"1.1.0\n"} + self.readmes = {self.target: README, self.head: EXPECTED} + self.refs = {"v1.0.1": {"type": "commit", "sha": "c" * 40}, + "v1.1.0": {"type": "commit", "sha": "1" * 40}, + "v1.2.0": {"type": "commit", "sha": "2" * 40}} + self.tags = {} + self.prs = [] + self.detail = None + self.proposal_head = None + self.transform = None + self.raw_response = None + self.status_code = 200 + self.omit_length = False + self.extra_headers = {} + test = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + test.calls.append(("GET", self.path, self.headers.get("Authorization"))) + test.counts[self.path] = test.counts.get(self.path, 0) + 1 + try: + data = test.route(self.path) + if test.transform: + data = test.transform(self.path, copy.deepcopy(data)) + raw = test.raw_response if test.raw_response is not None else encode(data) + self.send_response(test.status_code) + self.send_header("Content-Type", "application/json") + if not test.omit_length and "Content-Length" not in test.extra_headers: + self.send_header("Content-Length", str(len(raw))) + for key, value in test.extra_headers.items(): + self.send_header(key, value) + self.end_headers() + try: + self.wfile.write(raw) + except (BrokenPipeError, ConnectionResetError): + pass + except (AssertionError, KeyError): + self.send_error(500) + + def refused_method(self): + test.calls.append((self.command, self.path, self.headers.get("Authorization"))) + self.send_error(405) + + do_POST = do_PATCH = do_PUT = do_DELETE = do_HEAD = refused_method + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + thread = threading.Thread(target=self.server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 5) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.origin = "http://127.0.0.1:" + str(self.server.server_port) + # Registered last, so it checks the recorded requests before server cleanup. + self.addCleanup(self.assert_get_only) + + def assert_get_only(self): + self.assertTrue(all(call[0] == "GET" for call in self.calls), self.calls) + + def route(self, raw_path): + parsed = urlsplit(raw_path) + path = unquote(parsed.path) + query = parse_qs(parsed.query) + if path in ("/repos/" + HUB, "/repos/" + ENGINE): + name = path.removeprefix("/repos/") + result = copy.deepcopy(self.repos[name]) + if name == ENGINE and self.default_moved and self.counts[raw_path] > 1: + result["default_branch"] = "next/default" + return result + if path == f"/repos/{HUB}/git/ref/heads/main": + value = "f" * 40 if self.hub_moved and self.counts[raw_path] > 1 else self.hub + return {"ref": "refs/heads/main", "object": {"type": "commit", "sha": value}} + if path.startswith(f"/repos/{ENGINE}/git/ref/heads/"): + name = path.split("/git/ref/heads/", 1)[1] + assert name in (self.repos[ENGINE]["default_branch"], "next/default", self.proposal_head) + if name == self.proposal_head: + value = self.head + else: + value = "e" * 40 if self.target_moved and self.counts[raw_path] > 1 else self.target + return {"ref": "refs/heads/" + name, "object": {"type": "commit", "sha": value}} + if path.startswith(f"/repos/{ENGINE}/git/ref/tags/"): + name = path.split("/git/ref/tags/", 1)[1] + return {"ref": "refs/tags/" + name, "object": self.refs[name]} + if path.startswith(f"/repos/{ENGINE}/git/tags/"): + return self.tags[path.rsplit("/", 1)[-1]] + if path.startswith(f"/repos/{HUB}/contents/"): + assert query == {"ref": [self.hub]} + name = path.split("/contents/", 1)[1] + return envelope(name, self.files[name]) + if path.startswith(f"/repos/{ENGINE}/contents/"): + assert path.endswith("/contents/README.md") and set(query) == {"ref"} + return envelope("README.md", self.readmes[query["ref"][0]]) + if path == f"/repos/{ENGINE}/pulls": + assert set(query) == {"head", "base", "state", "per_page", "page"} + assert query["state"] == ["open"] and query["per_page"] == ["100"] + assert query["base"] == [self.repos[ENGINE]["default_branch"]] + assert query["head"][0].startswith(OWNER + ":automation/facts-engine-") + return self.prs if query["page"] == ["1"] else [] + if path == f"/repos/{ENGINE}/pulls/7": + return self.detail if self.detail is not None else self.prs[0] + raise AssertionError("unexpected request") + + def environment(self, **extra): + env = dict(os.environ) + env.update(FACTS_READ_TOKEN=TOKEN, GITHUB_SHA="9" * 40, + GITHUB_EVENT_NAME="repository_dispatch", NODE_OPTIONS="--invalid-option-must-not-reach-bridge") + env.update(extra) + return env + + def plan(self): + with patch.dict(os.environ, self.environment(), clear=True): + return build_engine_plan(PublicGitHub(self.origin), render_with_node) + + def invoke(self, env=None, api=None, output=None, file_limit=None): + if output is None: + self.outputs += 1 + output = self.directory / ("output-" + str(self.outputs)) + def limit_files(): + signal.signal(signal.SIGXFSZ, signal.SIG_IGN) + resource.setrlimit(resource.RLIMIT_FSIZE, (file_limit, file_limit)) + + result = subprocess.run([sys.executable, str(CLI), "plan", "--output-dir", str(output), + "--test-api", self.origin if api is None else api], + env=self.environment(**(env or {})), capture_output=True, + preexec_fn=limit_files if file_limit is not None else None) + return result, output + + def assert_refused(self, **arguments): + result, output = self.invoke(**arguments) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, + b"engine-readme-plan: input unavailable or refused; no remote changes\n") + self.assertFalse(output.exists()) + for sensitive in (TOKEN, PRIVATE_TEXT, str(output)): + self.assertNotIn(sensitive.encode(), result.stdout + result.stderr) + + def add_pull(self, plan): + self.proposal_head = plan["target"]["head"] + self.prs = [{"id": 7007, "number": 7, "state": "open", "title": "Title is not an identity", + "html_url": f"https://github.com/{ENGINE}/pull/7", + "url": f"{API_ORIGIN}/repos/{ENGINE}/pulls/7", + "body": plan["proposed_pr"]["body"], + "head": {"ref": self.proposal_head, "label": OWNER + ":" + self.proposal_head, + "sha": self.head, "repo": copy.deepcopy(self.repos[ENGINE])}, + "base": {"ref": "trunk", "label": OWNER + ":trunk", "sha": self.target, + "repo": copy.deepcopy(self.repos[ENGINE])}}] + + def test_create_plan_binds_real_source_renderer_and_unmarked_utf8_bytes(self): + plan = self.plan() + self.assertEqual(plan["schema"], SCHEMA) + self.assertEqual(plan["status"], "create-needed") + self.assertEqual(plan["proposed_readme"].encode(), EXPECTED) + self.assertEqual(plan["readme"], {"path": "README.md", "before_sha256": sha256(README), + "after_sha256": sha256(EXPECTED), "markers": 3, "changed": True}) + self.assertEqual(plan["snapshot"]["commit"], self.hub) + self.assertEqual(plan["snapshot"]["facts_sha256"], sha256(self.files["facts/facts.json"])) + self.assertEqual(base64.b64decode(plan["snapshot"]["version_bytes_base64"]), b"1.1.0\n") + self.assertEqual(plan["release"]["commit"], "1" * 40) + self.assertEqual(plan["target"]["base_commit"], self.target) + self.assertNotEqual(plan["release"]["commit"], plan["target"]["base_commit"]) + identity = {"repository_role": "engine", "repository": ENGINE, "release_tag": "v1.1.0", + "facts_sha256": sha256(self.files["facts/facts.json"])} + expected_id = sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()) + self.assertEqual(plan["plan_identity"], expected_id) + self.assertEqual(plan["target"]["head"], "automation/facts-engine-" + expected_id[:24]) + self.assertEqual(plan["proposed_pr"]["body_sha256"], sha256(plan["proposed_pr"]["body"].encode())) + binding = json.loads(plan["proposed_pr"]["body"].split("```json\n")[1].split("\n```")[0]) + self.assertEqual(binding["tag_commit"], "1" * 40) + self.assertEqual(binding["base_commit"], self.target) + self.assertEqual(binding["readme_after_sha256"], sha256(EXPECTED)) + paths = [path for _, path, _ in self.calls] + self.assertEqual([path for path in paths if "/contents/" in path], [ + f"/repos/{HUB}/contents/facts/facts.json?ref={self.hub}", + f"/repos/{HUB}/contents/facts/facts.schema.json?ref={self.hub}", + f"/repos/{HUB}/contents/VERSION?ref={self.hub}", + f"/repos/{ENGINE}/contents/README.md?ref={self.target}"]) + self.assertNotIn("facts-source", plan["proposed_readme"]) + self.assertTrue(all(auth == "Bearer " + TOKEN for _, _, auth in self.calls)) + self.assertEqual(plan["remote_writes"], 0) + json.dumps(plan, allow_nan=False) + + def test_cli_outputs_review_manifest_and_proposed_bytes_after_rechecks(self): + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + receipt = json.loads(result.stdout) + manifest = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(receipt["status"], "create-needed") + self.assertEqual((output / "README.md").read_bytes(), EXPECTED) + self.assertNotIn("proposed_readme", manifest) + self.assertTrue(manifest["review_only"]) + self.assertEqual(manifest["recheck"]["hub_commit"], manifest["snapshot"]["commit"]) + self.assertEqual(manifest["recheck"]["base_commit"], self.target) + self.assertNotIn(TOKEN, (output / "plan.json").read_text()) + self.assertEqual(set(path.name for path in output.iterdir()), {"plan.json", "README.md"}) + + def test_current_metadata_selects_non_main_default_branch_and_distinct_commits(self): + for base, target, version in (("stable/docs", "3" * 40, b"1.0.1"), + ("readme-next", "4" * 40, b"1.2.0\n")): + with self.subTest(base=base): + self.hub = "5" * 40 + self.target = target + self.files["VERSION"] = version + self.repos[ENGINE]["default_branch"] = base + self.readmes[target] = README + self.calls.clear() + plan = self.plan() + self.assertEqual(plan["snapshot"]["commit"], "5" * 40) + self.assertEqual(plan["target"]["base"], base) + self.assertEqual(plan["target"]["base_commit"], target) + self.assertEqual(plan["release"]["tag"], "v" + version.decode().strip()) + self.assertNotEqual(plan["release"]["commit"], target) + self.assertEqual([path for _, path, _ in self.calls if f"/repos/{ENGINE}/contents/" in path], + [f"/repos/{ENGINE}/contents/README.md?ref={target}"]) + + def test_new_tag_same_facts_fetches_new_tag_and_repairs_drift(self): + first = self.plan() + self.calls.clear() + self.hub = "7" * 40 + self.files["VERSION"] = b"1.2.0" + second = self.plan() + self.assertEqual(second["status"], "create-needed") + self.assertEqual(first["snapshot"]["facts_sha256"], second["snapshot"]["facts_sha256"]) + self.assertNotEqual(first["plan_identity"], second["plan_identity"]) + self.assertNotEqual(first["target"]["head"], second["target"]["head"]) + self.assertIn(f"/repos/{ENGINE}/git/ref/tags/v1.2.0", [path for _, path, _ in self.calls]) + self.assertIn(f"/repos/{ENGINE}/contents/README.md?ref={self.target}", + [path for _, path, _ in self.calls]) + # The existing README's historic token scope is preserved even under a new trigger tag. + self.assertEqual(second["proposed_readme"].encode(), EXPECTED) + + def test_new_tag_without_readme_drift_is_full_identity_noop(self): + before = self.plan() + self.hub = "8" * 40 + self.files["VERSION"] = b"1.2.0\n" + self.readmes[self.target] = EXPECTED + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(plan["status"], "noop") + self.assertEqual(plan["release"], {"tag": "v1.2.0", "commit": "2" * 40, + "source_url": "https://github.com/pineforge-4pass/pineforge-engine/blob/v1.2.0/README.md#validation-scoreboard"}) + self.assertNotEqual(before["plan_identity"], plan["plan_identity"]) + self.assertFalse((output / "README.md").exists()) + self.assertNotIn("proposed_pr", plan) + + def test_changed_facts_same_tag_changes_render_identity_and_branch(self): + first = self.plan() + facts = json.loads(self.files["facts/facts.json"]) + facts["inventory"]["tvTrades"] = 4776345 + self.hub = "7" * 40 + self.files["facts/facts.json"] = encode(facts) + second = self.plan() + self.assertEqual(second["release"]["tag"], first["release"]["tag"]) + self.assertNotEqual(second["plan_identity"], first["plan_identity"]) + self.assertNotEqual(second["target"]["head"], first["target"]["head"]) + self.assertEqual(second["proposed_readme"].encode(), EXPECTED.replace(b"4,776,328", b"4,776,345")) + self.assertIn("superseded open generations", second["limitations"][0]) + + def test_reserialized_identical_facts_bind_original_raw_hash(self): + first = self.plan() + self.hub = "8" * 40 + self.files["facts/facts.json"] = b"\n" + self.files["facts/facts.json"] + b" " + second = self.plan() + self.assertNotEqual(first["snapshot"]["facts_sha256"], second["snapshot"]["facts_sha256"]) + self.assertNotEqual(first["plan_identity"], second["plan_identity"]) + self.assertEqual(first["readme"]["after_sha256"], second["readme"]["after_sha256"]) + + def test_repeat_plan_inspects_current_pr_head_and_body_before_already_planned(self): + first = self.plan() + self.add_pull(first) + self.calls.clear() + second = self.plan() + self.assertEqual(second["status"], "already-planned") + self.assertEqual(second["plan_identity"], first["plan_identity"]) + self.assertEqual(second["target"]["head"], first["target"]["head"]) + self.assertEqual(second["existing_pr"]["url"], f"https://github.com/{ENGINE}/pull/7") + paths = [path for _, path, _ in self.calls] + self.assertIn(f"/repos/{ENGINE}/pulls/7", paths) + self.assertIn(f"/repos/{ENGINE}/contents/README.md?ref={self.head}", paths) + self.assertIn(f"/repos/{ENGINE}/git/ref/heads/" + self.proposal_head.replace("/", "%2F"), paths) + query = parse_qs(urlsplit(next(path for path in paths if "/pulls?" in path)).query) + self.assertEqual(query["head"], [OWNER + ":" + first["target"]["head"]]) + self.assertEqual(query["base"], ["trunk"]) + + def test_existing_pr_readme_or_body_drift_requires_update(self): + first = self.plan() + self.add_pull(first) + for kind in ("readme", "body", "base"): + with self.subTest(kind=kind): + self.readmes[self.head] = README if kind == "readme" else EXPECTED + self.prs[0]["body"] = first["proposed_pr"]["body"] + ("\n" if kind == "body" else "") + self.prs[0]["base"]["sha"] = "3" * 40 if kind == "base" else self.target + plan = self.plan() + self.assertEqual(plan["status"], "update-needed") + self.assertEqual(plan["existing_pr"]["number"], 7) + self.assertEqual(plan["target"]["head"], first["target"]["head"]) + + def test_pr_detail_and_actual_head_ref_are_distinct_evidence(self): + first = self.plan() + self.add_pull(first) + self.detail = copy.deepcopy(self.prs[0]) + self.prs[0]["head"]["sha"] = "4" * 40 + self.assertEqual(self.plan()["status"], "already-planned") + self.head = "5" * 40 + self.readmes[self.head] = EXPECTED + plan = self.plan() + self.assertEqual(plan["status"], "update-needed") + self.assertEqual(plan["existing_pr"]["head_commit"], "5" * 40) + self.assertEqual(plan["existing_pr"]["reported_head_commit"], "d" * 40) + + def test_source_target_or_default_branch_movement_supersedes_without_proposal(self): + for flag in ("hub_moved", "target_moved", "default_moved"): + with self.subTest(flag=flag): + self.counts.clear() + self.hub_moved = self.target_moved = self.default_moved = False + setattr(self, flag, True) + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(plan["status"], "superseded") + self.assertNotIn("proposed_pr", plan) + self.assertNotIn("proposed_readme", plan) + self.assertFalse((output / "README.md").exists()) + + def test_direct_and_eight_annotated_tags_bind_exact_release_commit(self): + for depth in (0, 1, 8): + with self.subTest(depth=depth): + self.tags.clear() + pointers = [f"{20 + i:040x}" for i in range(depth)] + for index, sha in enumerate(pointers): + child = {"type": "tag", "sha": pointers[index + 1]} if index + 1 < depth else { + "type": "commit", "sha": "6" * 40} + self.tags[sha] = {"sha": sha, "tag": "v1.1.0", "object": child} + self.refs["v1.1.0"] = {"type": "tag", "sha": pointers[0]} if depth else { + "type": "commit", "sha": "6" * 40} + plan = self.plan() + self.assertEqual(plan["release"]["commit"], "6" * 40) + self.assertEqual(plan["target"]["base_commit"], "b" * 40) + + def test_annotated_tag_cycles_excess_depth_and_wrong_types_refuse(self): + for kind in ("cycle", "depth", "type", "identity"): + with self.subTest(kind=kind): + self.calls.clear() + pointers = [f"{30 + i:040x}" for i in range(9)] + self.refs["v1.1.0"] = {"type": "tag", "sha": pointers[0]} + self.tags = {sha: {"sha": sha, "tag": "v1.1.0", "object": { + "type": "tag", "sha": pointers[(index + 1) % 9]}} + for index, sha in enumerate(pointers)} + if kind == "cycle": + self.tags[pointers[0]]["object"]["sha"] = pointers[0] + elif kind == "type": + self.tags[pointers[0]]["object"]["type"] = "tree" + elif kind == "identity": + self.tags[pointers[0]]["sha"] = "7" * 40 + self.assert_refused() + tag_reads = [path for _, path, _ in self.calls if f"/repos/{ENGINE}/git/tags/" in path] + self.assertEqual(len(tag_reads), 8 if kind == "depth" else 1) + + def test_private_mismatched_and_malformed_repository_metadata_refuse(self): + for repo in (HUB, ENGINE): + original = copy.deepcopy(self.repos[repo]) + for field, value in (("private", True), ("full_name", "untrusted/repository"), + ("id", True), ("owner", {"login": "foreign"}), + ("default_branch", "unsafe/../name"), ("url", "http://127.0.0.1/private")): + with self.subTest(repo=repo, field=field): + self.repos[repo] = dict(original, **{field: value}) + self.assert_refused() + self.repos[repo] = original + + def test_contents_path_type_name_size_encoding_and_blob_identity_refuse(self): + for name in ("facts/facts.json", "facts/facts.schema.json", "VERSION", "README.md"): + for field, value in (("path", "foreign/path"), ("type", "symlink"), ("name", "foreign"), + ("size", True), ("size", 1), ("sha", "0" * 40), + ("encoding", "none"), ("content", "%%%")): + with self.subTest(name=name, field=field, value=value): + def corrupt(path, data, name=name, field=field, value=value): + if urlsplit(path).path.endswith("/contents/" + name): + data[field] = value + return data + self.transform = corrupt + self.assert_refused() + self.transform = None + + def test_remote_schema_change_and_external_ref_are_refused_before_renderer(self): + original = self.files["facts/facts.schema.json"] + for raw in (b" " + original, encode({"$ref": self.origin + "/private-schema"})): + with self.subTest(raw=raw[:24]): + self.files["facts/facts.schema.json"] = raw + self.calls.clear() + self.assert_refused() + self.assertFalse(any("/git/ref/tags/" in path for _, path, _ in self.calls)) + self.assertFalse(any("private-schema" in path for _, path, _ in self.calls)) + + def test_invalid_actual_facts_and_version_fail_without_selector_fallback(self): + original = dict(self.files) + for filename, raw in (("facts/facts.json", b'{"schema":"a","schema":"b"}'), + ("facts/facts.json", b'{"schema":"wrong"}'), + ("VERSION", b"999.1.0"), ("VERSION", b"1.1.0\r\n"), + ("VERSION", b"1.1.0/unsafe"), ("VERSION", b"1.1.0\n\n")): + with self.subTest(filename=filename, raw=raw): + self.files = dict(original, **{filename: raw}) + self.assert_refused() + + def test_wrong_ref_name_commit_or_tag_target_never_qualifies(self): + for field, value in (("ref", "refs/tags/wrong"), ("sha", "A" * 40), + ("sha", "1" * 39), ("type", "blob")): + with self.subTest(field=field, value=value): + def corrupt(path, data, field=field, value=value): + if path.endswith("/git/ref/tags/v1.1.0"): + if field == "ref": + data[field] = value + else: + data["object"][field] = value + return data + self.transform = corrupt + self.assert_refused() + + def test_malformed_unsafe_zero_and_unknown_markers_fail_real_bridge(self): + for raw in (b"No markers", b"Count: old", + b"old\n", + b"Count: `old`", + b"Count: old", + b"Count: outer " + b"nested", b"\xffx"): + with self.subTest(raw=raw[:50]): + self.readmes[self.target] = raw + self.assert_refused() + + def test_pr_duplicates_ambiguity_and_foreign_ownership_refuse(self): + first = self.plan() + self.add_pull(first) + original = copy.deepcopy(self.prs[0]) + cases = ("duplicate", "ambiguous", "head", "base", "repo", "private", "url", "state", "body", "marker") + for case in cases: + with self.subTest(case=case): + self.prs = [copy.deepcopy(original)] + row = self.prs[0] + if case in ("duplicate", "ambiguous"): + other = copy.deepcopy(row) + if case == "ambiguous": + other.update(id=7008, number=8, html_url=f"https://github.com/{ENGINE}/pull/8", + url=f"{API_ORIGIN}/repos/{ENGINE}/pulls/8") + self.prs.append(other) + elif case in ("head", "base"): + row[case]["ref"] = "foreign" + elif case == "repo": + row["head"]["repo"]["full_name"] = "foreign/engine" + elif case == "private": + row["head"]["repo"]["private"] = True + elif case == "url": + row["html_url"] = self.origin + "/private" + elif case == "state": + row["state"] = "closed" + elif case == "body": + row["body"] = None + else: + row["body"] = row["body"].replace(first["plan_identity"], "0" * 64) + self.assert_refused() + + def test_current_pr_detail_is_validated_even_when_listing_matches(self): + first = self.plan() + self.add_pull(first) + for kind in ("id", "private", "marker"): + with self.subTest(kind=kind): + self.detail = copy.deepcopy(self.prs[0]) + if kind == "id": + self.detail["id"] += 1 + elif kind == "private": + self.detail["base"]["repo"]["private"] = True + else: + self.detail["body"] += "\n" + self.assert_refused() + + def test_pr_page_malformed_truncated_foreign_and_over_bound_links_refuse(self): + for kind in ("malformed", "truncated", "foreign", "bound", "missing-next", "duplicate"): + with self.subTest(kind=kind): + def pagination(path, data, kind=kind): + if "/pulls?" in path: + parsed = urlsplit(path) + query = {key: values[0] for key, values in parse_qs(parsed.query).items()} + query["page"] = "21" if kind == "bound" else "2" + origin = "http://127.0.0.1:1" if kind == "foreign" else self.origin + url = origin + parsed.path + "?" + urlencode(query) + link = f'<{url}>; rel="last"' if kind in ("bound", "missing-next") else f'<{url}>; rel="next"' + if kind == "malformed": + link = "malformed" + elif kind == "duplicate": + link += ", " + link + self.extra_headers["Link"] = link + return data + self.transform = pagination + self.extra_headers.clear() + self.assert_refused() + + def test_full_pr_pages_stop_at_twenty_without_silent_selection(self): + first = self.plan() + self.add_pull(first) + original = copy.deepcopy(self.prs[0]) + + def full_pages(path, data): + if "/pulls?" not in path: + return data + page = int(parse_qs(urlsplit(path).query)["page"][0]) + rows = [] + for offset in range(100): + number = (page - 1) * 100 + offset + 7 + row = copy.deepcopy(original) + row.update(id=10000 + number, number=number, + html_url=f"https://github.com/{ENGINE}/pull/{number}", + url=f"{API_ORIGIN}/repos/{ENGINE}/pulls/{number}") + rows.append(row) + return rows + + self.transform = full_pages + self.calls.clear() + self.assert_refused() + pages = [int(parse_qs(urlsplit(path).query)["page"][0]) + for _, path, _ in self.calls if "/pulls?" in path] + self.assertEqual(pages, list(range(1, 21))) + + def test_pr_page_wrong_shape_and_more_than_100_rows_refuse(self): + for rows in ({"items": []}, [None] * 101): + with self.subTest(shape=type(rows).__name__): + self.transform = lambda path, data, rows=rows: rows if "/pulls?" in path else data + self.assert_refused() + + def test_http_errors_bad_json_and_oversize_are_fixed_safe_refusals(self): + for kind in ("error", "json", "oversize", "undeclared-oversize", "truncated", "encoding"): + with self.subTest(kind=kind): + self.status_code = 503 if kind == "error" else 200 + self.raw_response = (PRIVATE_TEXT.encode() if kind in ("error", "json") else + b" " * (MAX_INPUT_BYTES + 1) if "oversize" in kind else b"{}") + self.extra_headers = {"Content-Encoding": "gzip"} if kind == "encoding" else {} + self.omit_length = kind == "undeclared-oversize" + if kind == "undeclared-oversize": + # Exercise the actual bounded read of an HTTP/1.0 close-delimited body. + self.raw_response = b" " * MAX_INPUT_BYTES + b"!" + elif kind == "truncated": + self.extra_headers["Content-Length"] = "1234" + self.assert_refused() + + def test_redirect_never_sends_token_to_another_origin(self): + foreign_calls = [] + + class Foreign(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + foreign_calls.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + foreign = ThreadingHTTPServer(("127.0.0.1", 0), Foreign) + thread = threading.Thread(target=foreign.serve_forever, daemon=True) + thread.start() + try: + self.status_code = 302 + self.extra_headers["Location"] = f"http://127.0.0.1:{foreign.server_port}/private" + self.raw_response = PRIVATE_TEXT.encode() + self.assert_refused() + self.assertEqual(foreign_calls, []) + finally: + foreign.shutdown() + foreign.server_close() + thread.join(5) + + def test_proxy_environment_is_ignored_by_production_http_transport(self): + result, _ = self.invoke(env={"HTTP_PROXY": "http://127.0.0.1:1", "HTTPS_PROXY": "http://127.0.0.1:1", + "http_proxy": "http://127.0.0.1:1", "https_proxy": "http://127.0.0.1:1", + "NO_PROXY": "", "no_proxy": ""}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.calls) + + def test_test_origin_and_real_token_refuse_before_any_request(self): + for origin in (self.origin + "/", "http://localhost:1234", "https://127.0.0.1:1234", + self.origin + "?query=1", "http://user@127.0.0.1:1234", + "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://127.0.0.1:bad"): + with self.subTest(origin=origin): + self.assert_refused(api=origin) + self.assert_refused(env={"FACTS_READ_TOKEN": "github-real-credential"}) + self.assertEqual(self.calls, []) + + def test_existing_output_directory_file_and_symlink_are_never_overwritten(self): + for kind in ("directory", "file", "symlink"): + with self.subTest(kind=kind): + output = self.directory / kind + if kind == "directory": + output.mkdir() + (output / "keep").write_bytes(b"keep") + elif kind == "file": + output.write_bytes(b"keep") + else: + output.symlink_to(self.directory / "missing") + result, _ = self.invoke(output=output) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, b"") + if kind == "directory": + self.assertEqual((output / "keep").read_bytes(), b"keep") + elif kind == "file": + self.assertEqual(output.read_bytes(), b"keep") + else: + self.assertTrue(output.is_symlink()) + self.assertEqual(self.calls, []) + + def test_native_artifact_write_failure_leaves_no_partial_success_directory(self): + # The real filesystem rejects the larger manifest after the README was written. + self.assert_refused(file_limit=1024) + self.assertTrue(any("/contents/README.md?ref=" in path for _, path, _ in self.calls)) + + +class EngineRenderBridge(unittest.TestCase): + def bridge(self, raw): + return subprocess.run(["node", str(BRIDGE)], input=raw, capture_output=True, + env={"PATH": os.environ.get("PATH", os.defpath)}) + + def test_real_bridge_returns_actual_bytes_and_count_for_changed_public_facts(self): + facts = json.loads(FACTS.read_bytes()) + for count, expected in ((4776328, EXPECTED), + (4776345, EXPECTED.replace(b"4,776,328", b"4,776,345"))): + with self.subTest(count=count): + facts["inventory"]["tvTrades"] = count + result = self.bridge(encode({"text": README.decode(), "facts": facts})) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, b"") + value = json.loads(result.stdout) + self.assertEqual(value["markers"], 3) + self.assertEqual(value["text"].encode(), expected) + + def test_bridge_invalid_input_nonzero_exit_has_only_fixed_stderr(self): + for raw in (b"\xff", b"{", b"[]", encode({"text": "x", "facts": []}), + encode({"text": README.decode(), "facts": {}, "file": PRIVATE_TEXT}), + encode({"text": "No markers", "facts": {}}), + encode({"text": "Count: x", "facts": {}}), + encode({"text": "\ud800", "facts": {}}), b" " * (MAX_INPUT_BYTES + 1)): + with self.subTest(raw=raw[:40]): + result = self.bridge(raw) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"engine-readme-render: invalid input or render failure\n") + + def test_python_rejects_invalid_renderer_protocol_values(self): + # Protocol unit negatives only; every accepted controller plan above executes the real bridge. + for value in (None, {}, {"text": "x", "markers": 0}, {"text": "x", "markers": True}, + {"text": "\ud800", "markers": 1}, {"text": "x", "markers": 1, "extra": 1}, + {"text": "x" * (MAX_INPUT_BYTES + 1), "markers": 1}): + with self.subTest(kind=type(value).__name__): + with self.assertRaises(InputError): + checked_render(value) + + +if __name__ == "__main__": + unittest.main() From 7fd9df1ccc80c94aafaaee1907c46a8af5d577ab Mon Sep 17 00:00:00 2001 From: luisleo526 <20812494+luisleo526@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:52:40 +0800 Subject: [PATCH 2/2] ci: install Node 20 in the facts-validate job The facts-validate job's CLI acceptance step runs every tests/test_facts_*.py module, including the engine README plan tests, which call the Node 20 bridge scripts/facts_engine_render.mjs. That bridge refuses any other Node major, so the job now installs Node 20 with the same pinned actions/setup-node release that python-test.yml uses, under the job's existing docs-only condition. --- .github/workflows/facts-validate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/facts-validate.yml b/.github/workflows/facts-validate.yml index 76b3487..5627053 100644 --- a/.github/workflows/facts-validate.yml +++ b/.github/workflows/facts-validate.yml @@ -128,6 +128,10 @@ jobs: if: steps.paths.outputs.docs-only == 'false' with: python-version: '3.12' + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: steps.paths.outputs.docs-only == 'false' + with: + node-version: "20" - run: python3 -m pip install -r facts/requirements-validation.txt if: steps.paths.outputs.docs-only == 'false' - name: Validate the whole canonical document and descriptions