diff --git a/.github/workflows/facts-validate.yml b/.github/workflows/facts-validate.yml index 76b3487..5627053 100644 --- a/.github/workflows/facts-validate.yml +++ b/.github/workflows/facts-validate.yml @@ -128,6 +128,10 @@ jobs: if: steps.paths.outputs.docs-only == 'false' with: python-version: '3.12' + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: steps.paths.outputs.docs-only == 'false' + with: + node-version: "20" - run: python3 -m pip install -r facts/requirements-validation.txt if: steps.paths.outputs.docs-only == 'false' - name: Validate the whole canonical document and descriptions diff --git a/scripts/facts_engine_plan.py b/scripts/facts_engine_plan.py new file mode 100644 index 0000000..733c8e9 --- /dev/null +++ b/scripts/facts_engine_plan.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Build a public, read-only proposal for the engine README.""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import subprocess +import sys +from http.client import HTTPException +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, quote, urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener + +from facts_engine_release import select_engine_release +from facts_validate import (FACTS_SCHEMA, MAX_INPUT_BYTES, InputError, decode, digest, + read_bytes, require) + +HUB = "pineforge-4pass/pineforge-release" +ENGINE = "pineforge-4pass/pineforge-engine" +OWNER = "pineforge-4pass" +SCHEMA = "pineforge/engine-readme-plan/v1" +API_ORIGIN = "https://api.github.com" +BRIDGE = Path(__file__).with_name("facts_engine_render.mjs") +COMMIT = re.compile(r"[0-9a-f]{40}\Z") +IDENTITY = re.compile(r"[0-9a-f]{64}\Z") +MACHINE_MARKER = re.compile(r"") +GENERATION_LIMIT = ( + "Changed facts under the same release tag produce a different identity and branch. " + "A separately authorized publisher must settle superseded open generations before activation." +) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True, allow_nan=False).encode("utf-8") + + +def commit(value): + require(type(value) is str and COMMIT.fullmatch(value), "invalid commit identity") + return value + + +def branch(value): + require(type(value) is str + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}", value) + and not any(part in value for part in ("..", "//", "@{")) + and all(part and not part.startswith(".") and not part.endswith((".", ".lock")) + for part in value.split("/")), "unsafe Git ref name") + return value + + +def repository(value, expected, expected_id=None): + require(type(value) is dict and expected in (HUB, ENGINE), "invalid repository metadata") + owner, name = expected.split("/") + require(value.get("private") is False and value.get("full_name") == expected + and value.get("name") == name and type(value.get("owner")) is dict + and value["owner"].get("login") == owner + and type(value.get("id")) is int and value["id"] > 0 + and value.get("html_url") == "https://github.com/" + expected + and value.get("url") == API_ORIGIN + "/repos/" + expected + and value.get("visibility", "public") == "public", + "exact public repository was not confirmed") + if expected_id is not None: + require(value["id"] == expected_id, "repository numeric identity changed") + branch(value.get("default_branch")) + return value + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise InputError("GitHub redirect refused") + + +class PublicGitHub: + """A dedicated GET transport; no write credential or mutation interface.""" + + def __init__(self, test_api=None): + self.origin = API_ORIGIN + self.token = os.environ.get("FACTS_READ_TOKEN", "") + require(len(self.token) <= 4096 and (not self.token or + re.fullmatch(r"[A-Za-z0-9_.-]+", self.token)), "invalid read credential") + if test_api is not None: + try: + parsed = urlsplit(test_api) + valid = (parsed.scheme == "http" and parsed.hostname == "127.0.0.1" + and parsed.port is not None and 1 <= parsed.port <= 65535 + and parsed.netloc == "127.0.0.1:" + str(parsed.port) + and not parsed.path and not parsed.query and not parsed.fragment) + except ValueError: + valid = False + require(valid, "test API must be an explicit loopback origin") + require(not self.token or self.token.startswith("test-"), + "real credentials are forbidden on the test transport") + self.origin = test_api + self.opener = build_opener(ProxyHandler({}), NoRedirect()) + + def _get(self, path): + require(type(path) is str and len(path) <= 4096 + and any(path == "/repos/" + repo or path.startswith("/repos/" + repo + "/") + for repo in (HUB, ENGINE)) + and not any(char in path for char in ("\r", "\n", "\\", "#")) + and ".." not in path and "//" not in path, "invalid public API path") + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "pineforge-engine-readme-plan", "Cache-Control": "no-cache", + "Accept-Encoding": "identity"} + if self.token: + headers["Authorization"] = "Bearer " + self.token + try: + with self.opener.open(Request(self.origin + path, headers=headers, method="GET"), + timeout=30) as response: + require(response.status == 200, "unexpected GitHub response status") + require(response.headers.get("Content-Encoding", "identity") == "identity", + "encoded GitHub response refused") + length = response.headers.get("Content-Length") + require(length is None or (re.fullmatch(r"[0-9]{1,10}", length) + and int(length) <= MAX_INPUT_BYTES), "GitHub response exceeds size limit") + raw = response.read(MAX_INPUT_BYTES + 1) + require(length is None or len(raw) == int(length), "truncated GitHub response") + links = response.headers.get("Link", "") + require(len(links) <= 8192, "oversize pagination metadata") + return decode(raw), links + except HTTPError as error: + error.close() + raise InputError("GitHub GET failed") from None + except (URLError, OSError, TimeoutError, HTTPException): + raise InputError("GitHub GET unavailable") from None + + def get(self, path): + return self._get(path)[0] + + def public_repository(self, name, expected_id=None): + require(name in (HUB, ENGINE), "repository is outside the public plan") + return repository(self.get("/repos/" + name), name, expected_id) + + def ref(self, repo, kind, name): + require(repo in (HUB, ENGINE) and kind in ("heads", "tags"), "invalid ref role") + branch(name) + value = self.get(f"/repos/{repo}/git/ref/{kind}/{quote(name, safe='')}") + require(type(value) is dict and value.get("ref") == f"refs/{kind}/{name}", + "Git ref identity differs") + obj = value.get("object") + require(type(obj) is dict and obj.get("type") in ("commit", "tag"), "invalid Git ref object") + commit(obj.get("sha")) + if kind == "heads": + require(obj["type"] == "commit", "branch does not identify a commit") + return obj + + def branch_commit(self, repo, name): + return self.ref(repo, "heads", name)["sha"] + + def tag_commit(self, release_tag): + obj = self.ref(ENGINE, "tags", release_tag) + seen = set() + depth = 0 + while obj["type"] == "tag": + sha = commit(obj["sha"]) + require(sha not in seen and depth < 8, "tag cycle or depth limit") + seen.add(sha) + value = self.get(f"/repos/{ENGINE}/git/tags/{sha}") + require(type(value) is dict and value.get("sha") == sha + and type(value.get("tag")) is str, "annotated tag identity differs") + branch(value["tag"]) + obj = value.get("object") + require(type(obj) is dict and obj.get("type") in ("commit", "tag"), + "invalid annotated tag target") + commit(obj.get("sha")) + depth += 1 + return commit(obj["sha"]) + + def contents(self, repo, sha, name): + commit(sha) + require((repo == HUB and name in ("facts/facts.json", "facts/facts.schema.json", "VERSION")) + or (repo == ENGINE and name == "README.md"), "invalid immutable content role") + value = self.get(f"/repos/{repo}/contents/{name}?ref={sha}") + require(type(value) is dict and value.get("type") == "file" + and value.get("path") == name and value.get("name") == name.rsplit("/", 1)[-1] + and value.get("encoding") == "base64" and type(value.get("content")) is str + and type(value.get("size")) is int and 0 <= value["size"] <= MAX_INPUT_BYTES, + "invalid Contents envelope") + try: + raw = base64.b64decode(value["content"].replace("\n", ""), validate=True) + except (ValueError, UnicodeError): + raise InputError("invalid Contents encoding") from None + require(len(raw) == value["size"] and len(raw) <= MAX_INPUT_BYTES, "Contents size differs") + blob = hashlib.sha1(b"blob " + str(len(raw)).encode("ascii") + b"\0" + raw).hexdigest() + require(value.get("sha") == blob, "Contents Git blob digest differs") + return raw + + def pull_page(self, head, base, page): + require(type(page) is int and 1 <= page <= 20, "pull request pagination limit") + query = {"state": "open", "head": OWNER + ":" + branch(head), "base": branch(base), + "per_page": "100", "page": str(page)} + path = f"/repos/{ENGINE}/pulls" + rows, links = self._get(path + "?" + urlencode(query)) + require(type(rows) is list and len(rows) <= 100, "invalid pull request page") + relations = {} + if links: + for part in links.split(","): + match = re.fullmatch(r'\s*<([^<>]+)>;\s*rel="(next|last|first|prev)"\s*', part) + require(match is not None and match[2] not in relations, "invalid pagination metadata") + url = urlsplit(match[1]) + expected = urlsplit(self.origin) + params = parse_qs(url.query, keep_blank_values=True) + require(url.scheme == expected.scheme and url.netloc == expected.netloc + and url.path == path and not url.fragment + and set(params) == set(query) + and all(params[key] == [value] for key, value in query.items() if key != "page") + and len(params["page"]) == 1 + and re.fullmatch(r"[1-9][0-9]*", params["page"][0]), + "foreign or malformed pagination target") + number = int(params["page"][0]) + require(1 <= number <= 20, "pull request pagination limit") + relations[match[2]] = number + require("next" not in relations or relations["next"] == page + 1, + "discontinuous pull request pagination") + require("first" not in relations or relations["first"] == 1, + "invalid first pagination page") + require("prev" not in relations or relations["prev"] == page - 1, + "invalid previous pagination page") + require("last" not in relations or relations["last"] >= page, + "invalid last pagination page") + require("last" not in relations or relations["last"] <= page or "next" in relations, + "truncated pull request pagination") + require(not (len(rows) < 100 and "next" in relations), "truncated pull request page") + return rows + + def pull(self, number): + require(type(number) is int and number > 0, "invalid pull request number") + return self.get(f"/repos/{ENGINE}/pulls/{number}") + + +def utf8(raw): + require(type(raw) is bytes and len(raw) <= MAX_INPUT_BYTES, "invalid README bytes") + try: + text = raw.decode("utf-8") + require(text.encode("utf-8") == raw, "README UTF-8 roundtrip differs") + return text + except UnicodeError: + raise InputError("README is not UTF-8") from None + + +def checked_render(value): + require(type(value) is dict and set(value) == {"text", "markers"} + and type(value["text"]) is str and type(value["markers"]) is int + and 0 < value["markers"] <= MAX_INPUT_BYTES, "invalid or empty renderer result") + try: + raw = value["text"].encode("utf-8") + except UnicodeError: + raise InputError("invalid renderer UTF-8") from None + utf8(raw) + return raw + + +def render_with_node(text, facts): + """Execute the local Node 20 bridge; renderer errors never qualify as a plan.""" + payload = canonical({"text": text, "facts": facts}) + require(len(payload) <= MAX_INPUT_BYTES, "renderer request exceeds size limit") + try: + result = subprocess.run(["node", "--max-old-space-size=128", str(BRIDGE)], input=payload, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30, + check=False, shell=False, + env={"PATH": os.environ.get("PATH", os.defpath)}) + except (OSError, subprocess.SubprocessError): + raise InputError("engine README renderer unavailable") from None + require(result.returncode == 0 and not result.stderr, "engine README renderer failed") + value = decode(result.stdout) + checked_render(value) + return value + + +def pull_identity(value, repo_id, head, base, plan_identity): + require(type(value) is dict and type(value.get("id")) is int and value["id"] > 0 + and type(value.get("number")) is int and value["number"] > 0 + and value.get("state") == "open", "invalid open pull request") + require(value.get("html_url") == f'https://github.com/{ENGINE}/pull/{value["number"]}' + and value.get("url") == f'{API_ORIGIN}/repos/{ENGINE}/pulls/{value["number"]}', + "pull request public URL differs") + for side, expected in (("head", head), ("base", base)): + endpoint = value.get(side) + require(type(endpoint) is dict and endpoint.get("ref") == expected + and endpoint.get("label") == OWNER + ":" + expected, + "pull request branch differs") + repository(endpoint.get("repo"), ENGINE, repo_id) + commit(endpoint.get("sha")) + body = value.get("body") + require(type(body) is str, "pull request body is unavailable") + try: + require(len(body.encode("utf-8")) <= MAX_INPUT_BYTES, "pull request body exceeds size limit") + except UnicodeError: + raise InputError("invalid pull request body UTF-8") from None + markers = list(MACHINE_MARKER.finditer(body)) + require(len(markers) == 1 and markers[0][1] == plan_identity + and body.count("pineforge:engine-readme-plan") == 1, + "foreign or corrupt pull request machine identity") + return value + + +def existing_pull(api, repo_id, head, base, plan_identity): + matches = [] + seen_ids, seen_numbers = set(), set() + for page in range(1, 21): + rows = api.pull_page(head, base, page) + for row in rows: + pull_identity(row, repo_id, head, base, plan_identity) + require(row["id"] not in seen_ids and row["number"] not in seen_numbers, + "duplicate pull request result") + seen_ids.add(row["id"]) + seen_numbers.add(row["number"]) + matches.append(row) + if len(rows) < 100: + break + else: + raise InputError("pull request pagination limit") + require(len(matches) <= 1, "ambiguous open pull requests") + if not matches: + return None + listed = matches[0] + current = pull_identity(api.pull(listed["number"]), repo_id, head, base, plan_identity) + require(current["id"] == listed["id"] and current["number"] == listed["number"], + "pull request identity changed") + return current + + +def build_engine_plan(api, renderer): + """Return a JSON-safe review artifact. This function has no remote write path.""" + hub_repo = api.public_repository(HUB) + target_repo = api.public_repository(ENGINE) + hub_commit = api.branch_commit(HUB, "main") + facts_raw = api.contents(HUB, hub_commit, "facts/facts.json") + schema_raw = api.contents(HUB, hub_commit, "facts/facts.schema.json") + version_raw = api.contents(HUB, hub_commit, "VERSION") + require(schema_raw == read_bytes(FACTS_SCHEMA), "source schema differs from the local schema") + selection = select_engine_release(facts_raw, version_raw, hub_commit=hub_commit, + expected_facts_sha256=digest(facts_raw)) + tag_commit = api.tag_commit(selection.release_tag) + base = target_repo["default_branch"] + target_commit = api.branch_commit(ENGINE, base) + before = api.contents(ENGINE, target_commit, "README.md") + rendered = renderer(utf8(before), decode(facts_raw)) + after = checked_render(rendered) + identity_input = {"repository_role": "engine", "repository": ENGINE, + "release_tag": selection.release_tag, "facts_sha256": selection.facts_sha256} + plan_identity = digest(canonical(identity_input)) + require(IDENTITY.fullmatch(plan_identity), "invalid plan identity") + head = "automation/facts-engine-" + plan_identity[:24] + require(head != base, "proposal branch equals target branch") + binding = {"plan_identity": plan_identity, "repository": ENGINE, + "hub_commit": hub_commit, "facts_sha256": selection.facts_sha256, + "version": selection.release_version, "version_sha256": digest(version_raw), + "release_tag": selection.release_tag, "tag_commit": tag_commit, + "base": base, "base_commit": target_commit, "head": OWNER + ":" + head, + "readme_before_sha256": digest(before), "readme_after_sha256": digest(after)} + body = (f"\n\n" + "Update the existing engine README fact markers from the selected public release facts.\n\n" + "```json\n" + json.dumps(binding, sort_keys=True, indent=2) + "\n```\n\n" + + GENERATION_LIMIT + "\n") + expected_body_sha = digest(body.encode("utf-8")) + current = existing_pull(api, target_repo["id"], head, base, plan_identity) + existing = None + equal = False + if current is not None: + current_head = api.branch_commit(ENGINE, head) + current_readme = api.contents(ENGINE, current_head, "README.md") + body_sha = digest(current["body"].encode("utf-8")) + equal = (current_head == current["head"]["sha"] and current["base"]["sha"] == target_commit + and digest(current_readme) == digest(after) and body_sha == expected_body_sha) + existing = {"id": current["id"], "number": current["number"], "url": current["html_url"], + "head_commit": current_head, "reported_head_commit": current["head"]["sha"], + "readme_sha256": digest(current_readme), "body_sha256": body_sha} + changed = before != after + status = ("noop" if not changed else "create-needed" if current is None + else "already-planned" if equal else "update-needed") + plan = {"schema": SCHEMA, "status": status, "plan_identity": plan_identity, + "identity_input": identity_input, + "snapshot": {"repository": HUB, "repository_id": hub_repo["id"], "ref": "main", + "commit": hub_commit, "facts_sha256": selection.facts_sha256, + "schema_sha256": digest(schema_raw), "version": selection.release_version, + "version_sha256": digest(version_raw), + "version_bytes_base64": base64.b64encode(version_raw).decode("ascii")}, + "release": {"tag": selection.release_tag, "commit": tag_commit, + "source_url": selection.source_url}, + "target": {"repository": ENGINE, "repository_id": target_repo["id"], "base": base, + "base_commit": target_commit, "head": head}, + "readme": {"path": "README.md", "before_sha256": digest(before), + "after_sha256": digest(after), "markers": rendered["markers"], "changed": changed}, + "limitations": [GENERATION_LIMIT], + "review_only": True, "remote_writes": 0} + if existing is not None: + plan["existing_pr"] = existing + # The last source reads precede emission. A changed default branch also invalidates the plan. + observed_hub = api.branch_commit(HUB, "main") + observed_repo = api.public_repository(ENGINE, target_repo["id"]) + observed_base = observed_repo["default_branch"] + observed_target = api.branch_commit(ENGINE, observed_base) + plan["recheck"] = {"hub_commit": observed_hub, "base": observed_base, "base_commit": observed_target} + if observed_hub != hub_commit or observed_base != base or observed_target != target_commit: + plan["status"] = "superseded" + elif changed: + plan["proposed_readme"] = utf8(after) + plan["proposed_pr"] = {"title": "Update engine README facts for " + selection.release_tag, + "body": body, "body_sha256": expected_body_sha} + return plan + + +def write_plan(directory, plan): + """Publish local review files only after every source check has completed.""" + payloads = {} + if "proposed_readme" in plan: + payloads["README.md"] = plan["proposed_readme"].encode("utf-8") + manifest = {key: value for key, value in plan.items() if key != "proposed_readme"} + payloads["plan.json"] = json.dumps(manifest, sort_keys=True, indent=2, + ensure_ascii=True, allow_nan=False).encode("utf-8") + b"\n" + # mkdir and exclusive opens protect every pre-existing output, including symlinks. + directory.mkdir() + created = [] + try: + for name, raw in payloads.items(): + path = directory / name + with path.open("xb") as stream: + created.append(path) + stream.write(raw) + except OSError: + for path in reversed(created): + path.unlink() + directory.rmdir() + raise + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("plan", allow_abbrev=False) + command.add_argument("--output-dir", type=Path, required=True) + command.add_argument("--test-api") + args = parser.parse_args(argv) + try: + require(not os.path.lexists(args.output_dir), "output directory already exists") + plan = build_engine_plan(PublicGitHub(args.test_api), render_with_node) + write_plan(args.output_dir, plan) + print(json.dumps({"schema": SCHEMA, "status": plan["status"], + "plan_identity": plan["plan_identity"], "review_only": True, + "remote_writes": 0}, sort_keys=True)) + return 0 + except (InputError, OSError, ValueError, KeyError, TypeError, RecursionError, OverflowError): + print("engine-readme-plan: input unavailable or refused; no remote changes", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/facts_engine_release.py b/scripts/facts_engine_release.py new file mode 100644 index 0000000..a642106 --- /dev/null +++ b/scripts/facts_engine_release.py @@ -0,0 +1,56 @@ +"""Select the engine README release identified by VERSION and public facts.""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +import facts_validate +from facts_validate import InputError, require + + +@dataclass(frozen=True) +class EngineReleaseSelection: + hub_commit: str + facts_sha256: str + release_version: str + release_tag: str + source_url: str + + +def select_engine_release( + facts_raw: bytes, + version_raw: bytes, + *, + hub_commit: str, + expected_facts_sha256: str, +) -> EngineReleaseSelection: + """Bind the exact VERSION release to the verified original facts bytes.""" + require(type(facts_raw) is bytes, "facts input must be bytes") + require(type(version_raw) is bytes, "VERSION input must be bytes") + require(type(hub_commit) is str + and re.fullmatch(r"[0-9a-f]{40}", hub_commit) is not None, + "invalid hub commit") + require(type(expected_facts_sha256) is str + and re.fullmatch(r"[0-9a-f]{64}", expected_facts_sha256) is not None, + "invalid expected facts SHA-256") + + facts_sha256 = facts_validate.digest(facts_raw) + require(facts_sha256 == expected_facts_sha256, "facts SHA-256 mismatch") + facts = facts_validate.validate_facts(facts_validate.decode(facts_raw)) + + require(len(version_raw) <= 256, "VERSION exceeds size limit") + version_bytes = version_raw[:-1] if version_raw.endswith(b"\n") else version_raw + require(re.fullmatch(rb"[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?", + version_bytes) is not None, "invalid VERSION") + version = version_bytes.decode("ascii") + try: + release = facts["releases"][version] + except KeyError: + raise InputError("selected release is missing") from None + return EngineReleaseSelection( + hub_commit=hub_commit, + facts_sha256=facts_sha256, + release_version=version, + release_tag="v" + version, + source_url=release["source"], + ) diff --git a/scripts/facts_engine_render.mjs b/scripts/facts_engine_render.mjs new file mode 100644 index 0000000..b87078b --- /dev/null +++ b/scripts/facts_engine_render.mjs @@ -0,0 +1,49 @@ +// Stdin/stdout bridge to the existing pure README renderer. Requires Node 20. +import { renderDocument } from './facts_markdown.mjs'; + +const LIMIT = 2 * 1024 * 1024; +const fail = () => { throw new Error('invalid engine README render request'); }; +const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +function utf8(text) { + if (typeof text !== 'string') fail(); + const bytes = Buffer.from(text, 'utf8'); + if (bytes.length > LIMIT || bytes.toString('utf8') !== text) fail(); + return bytes; +} + +function jsonTypes(value, depth = 0) { + if (depth > 64) fail(); + if (typeof value === 'string') { utf8(value); return; } + if (value === null || typeof value === 'boolean') return; + if (typeof value === 'number') { if (!Number.isFinite(value)) fail(); return; } + if (!object(value) && !Array.isArray(value)) fail(); + for (const [key, child] of Object.entries(value)) { + utf8(key); + jsonTypes(child, depth + 1); + } +} + +try { + if (process.versions.node.split('.')[0] !== '20') fail(); + const chunks = []; + let length = 0; + for await (const chunk of process.stdin) { + length += chunk.length; + if (length > LIMIT) fail(); + chunks.push(chunk); + } + const raw = Buffer.concat(chunks); + const input = JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(raw)); + if (!object(input) || Object.keys(input).sort().join(',') !== 'facts,text' || !object(input.facts)) fail(); + utf8(input.text); + jsonTypes(input.facts); + const result = renderDocument(input.text, input.facts, 'README.md'); + if (!object(result) || Object.keys(result).sort().join(',') !== 'markers,text' + || !Number.isSafeInteger(result.markers) || result.markers < 1) fail(); + utf8(result.text); + const output = utf8(JSON.stringify({ text: result.text, markers: result.markers })); + process.stdout.write(output); +} catch { + process.stderr.write('engine-readme-render: invalid input or render failure\n'); + process.exitCode = 2; +} diff --git a/scripts/facts_markdown.mjs b/scripts/facts_markdown.mjs new file mode 100644 index 0000000..5ac3c48 --- /dev/null +++ b/scripts/facts_markdown.mjs @@ -0,0 +1,180 @@ +// Pure fact-marker rendering. Callers supply facts and handle document persistence. +const integer = new Intl.NumberFormat('en-US', { maximumFractionDigits: 0 }); +const count = (value, name) => { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`invalid count: ${name}`); + return value; +}; + +export function tokenValue(facts, token) { + const segments = [...token.matchAll(/(?:\[([^\]]+)\]|([^.[\]]+))/g)].map((match) => match[1] ?? match[2]); + let value = facts; + for (const segment of segments) { + if (!value || !Object.hasOwn(value, segment) || ['__proto__', 'prototype', 'constructor'].includes(segment)) throw new Error(`unknown fact token: ${token}`); + value = value[segment]; + } + return value; +} + +function formatToken(value, format = 'text') { + if (format === 'lanes' || format === 'lanes-all') { + const allMarkets = format === 'lanes-all'; + if (allMarkets && (!value || typeof value !== 'object' || Array.isArray(value) || !Array.isArray(value.pairs) + || !value.otherMarkets || typeof value.otherMarkets !== 'object' || Array.isArray(value.otherMarkets))) { + throw new TypeError('lanes-all format requires a scoreboard object with pairs and otherMarkets'); + } + const pairs = allMarkets ? value.pairs : value; + if (!Array.isArray(pairs)) throw new Error('lanes format requires an array'); + const fields = ['graded', 'excellent', 'strong', 'belowStrong']; + const other = allMarkets ? value.otherMarkets : null; + if (allMarkets) { + count(other.markets, 'otherMarkets.markets'); + count(other.lanes, 'otherMarkets.lanes'); + for (const field of fields) { + const aggregate = count(other[field], `otherMarkets.${field}`); + const computed = pairs.reduce((sum, lane) => sum + count(lane[field], field), 0) + aggregate; + if (computed !== count(value[field], field) || (other.lanes === 0 && aggregate !== 0)) { + throw new RangeError(`lanes-all total mismatch: ${field}`); + } + } + } + const header = '| Market · timeframe | Probes graded | Excellent | Strong | Below strong |\n|---|---:|---:|---:|---:|'; + const ordered = [...pairs].sort((left, right) => Number(right.hardProbes > 0) - Number(left.hardProbes > 0)); + const rows = ordered.map((lane) => `| ${lane.symbol} · ${lane.timeframe === '15' ? '15m' : lane.timeframe}${lane.hardProbes > 0 ? ' *(hard lane: zero regression allowed)*' : ''} | ${integer.format(lane.graded)} | ${integer.format(lane.excellent)} | ${integer.format(lane.strong)} | ${integer.format(lane.belowStrong)} |`); + if (!allMarkets && !rows.length) rows.push(''); // Preserve the legacy empty table's bytes. + if (allMarkets && other.lanes > 0) { + rows.push(`| All other markets (${integer.format(other.markets)} markets, ${integer.format(other.lanes)} lanes) | ${fields.map((field) => integer.format(other[field])).join(' | ')} |`); + } + const total = fields.map((field) => `**${integer.format(allMarkets ? value[field] : pairs.reduce((sum, lane) => sum + lane[field], 0))}**`).join(' | '); + rows.push(`| **Total** | ${total} |`); + return `\n\n${header}\n${rows.join('\n')}\n\n`; + } + if (format === 'trades-badge') { count(value, 'TV trades'); return `[![Trades](https://img.shields.io/badge/TradingView%20trades%20graded-${(value / 1_000_000).toFixed(2)}M-brightgreen)](#validation-scoreboard)`; } + if (format === 'parity-badge') { + const label = encodeURIComponent(`${integer.format(value.excellent + value.strong)} / ${integer.format(value.graded)} excellent or strong`); + return `[![Parity](https://img.shields.io/badge/TradingView%20parity-${label}-brightgreen)](#validation-scoreboard)`; + } + if (format === 'int') { count(value, 'token'); return integer.format(value); } + if (format === 'pct') { if (typeof value !== 'number') throw new Error('pct format requires a number'); return value.toFixed(2); } + if (format === 'short') { if (typeof value !== 'string') throw new Error('short format requires text'); return value.slice(0, 8); } + if (format === 'code' || format === 'short-code') { + const content = formatToken(value, format === 'short-code' ? 'short' : 'text'); + if (/[`\r\n]/.test(content)) throw new Error('code format requires single-line text without backticks'); + return `\`${content}\``; + } + if (format !== 'text') throw new Error(`unknown fact format: ${format}`); + if (typeof value !== 'string' && typeof value !== 'number') throw new Error('token requires a scalar or explicit format'); + return String(value); +} + +export function renderText(text, facts) { + let markers = 0; + const rendered = text.replace(/[\s\S]*?/g, + (block, token, separator, format, offset) => { + if (block.slice(block.indexOf('-->') + 3).includes('${formatToken(tokenValue(facts, token), format)}`; + }); + if ((text.match(//g) ?? []).length !== markers) throw new Error('malformed fact markers'); + return { text: rendered, markers }; +} + +const BLOCK_TAGS = new Set(('address article aside base basefont blockquote body caption center col colgroup dd details dialog dir div dl dt ' + + 'fieldset figcaption figure footer form frame frameset h1 h2 h3 h4 h5 h6 head header hr html iframe legend li link main menu menuitem ' + + 'nav noframes ol optgroup option p param section source summary table tbody td tfoot th thead title tr track ul').split(' ')); +const ATTRIBUTE = String.raw`\s+[A-Za-z_:][\w.:-]*(?:\s*=\s*(?:[^\s"'=<>\x60]+|'[^']*'|"[^"]*"))?`; +const LONE_TAG = new RegExp(String.raw`^(?:<[A-Za-z][A-Za-z0-9-]*(?:${ATTRIBUTE})*\s*\/?>|<\/[A-Za-z][A-Za-z0-9-]*\s*>)\s*$`); +function htmlBlockStart(line, paragraph) { + const html = /^ {0,3}(<.*)$/.exec(line)?.[1]; + if (!html) return null; + if (/^<(?:pre|script|style|textarea)(?:[\s>]|$)/i.test(html)) return /<\/(?:pre|script|style|textarea)>/i; + if (html.startsWith('/; + if (html.startsWith('/; + if (/^/; + if (html.startsWith('/; + const tag = /^<\/?([A-Za-z][A-Za-z0-9-]*)(?:[\s>]|\/>|$)/.exec(html); + if (tag && BLOCK_TAGS.has(tag[1].toLowerCase())) return 'blank'; + return !paragraph && LONE_TAG.test(html) ? 'blank' : null; +} +// Leading whitespace with tabs expanded to the next multiple of four columns. +function expandIndent(text) { + return text.replace(/^[ \t]+/, (space) => { + let columns = 0; + for (const char of space) columns = char === '\t' ? columns + 4 - (columns % 4) : columns + 1; + return ' '.repeat(columns); + }); +} + +// A marker whose )/.test(leading)) { + const alone = /^)/.test(leading) && !leading.slice(leading.indexOf('-->') + 3).trim(); + const exempt = alone && (/^/.test(leading) || (lanes && leading.startsWith(''))); + if (!exempt) errors.push(`${file}:${index + 1}: marker starts a line: GitHub renders the line as raw HTML; keep text before it`); + } + for (const [marker, format] of line.matchAll(//g)) lanes = marker.startsWith('old trades.\r\n" + "Commit: old.\r\n" + "Release: old.\r\n" + "\r\nUnmarked **tail** without a final newline" +).encode("utf-8") +# Handwritten values from the public fixture, independent of the production renderer. +EXPECTED = README.replace(b"-->old trades.", b"-->4,776,328 trades.") +EXPECTED = EXPECTED.replace(b"-->old.\r\nRelease:", + b"-->`d1d18867`.\r\nRelease:") +EXPECTED = EXPECTED.replace( + b"-->old.\r\n\r\n", + b"-->https://github.com/pineforge-4pass/pineforge-engine/blob/v1.1.0/README.md#validation-scoreboard" + b".\r\n\r\n") + + +def encode(value): + return json.dumps(value, ensure_ascii=True, separators=(",", ":")).encode("utf-8") + + +def sha256(raw): + return hashlib.sha256(raw).hexdigest() + + +def metadata(name, number, default="trunk"): + owner, short = name.split("/") + return {"id": number, "name": short, "full_name": name, "owner": {"login": owner}, + "private": False, "visibility": "public", "default_branch": default, + "html_url": "https://github.com/" + name, "url": API_ORIGIN + "/repos/" + name} + + +def envelope(name, raw): + return {"type": "file", "path": name, "name": name.rsplit("/", 1)[-1], "size": len(raw), + "encoding": "base64", "content": base64.b64encode(raw).decode("ascii"), + "sha": hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest()} + + +class EnginePlanHTTP(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + self.calls = [] + self.counts = {} + self.outputs = 0 + self.hub = "a" * 40 + self.target = "b" * 40 + self.head = "d" * 40 + self.hub_moved = False + self.target_moved = False + self.default_moved = False + self.repos = {HUB: metadata(HUB, 11, "main"), ENGINE: metadata(ENGINE, 22)} + self.files = {"facts/facts.json": FACTS.read_bytes(), + "facts/facts.schema.json": (ROOT / "facts/facts.schema.json").read_bytes(), + "VERSION": b"1.1.0\n"} + self.readmes = {self.target: README, self.head: EXPECTED} + self.refs = {"v1.0.1": {"type": "commit", "sha": "c" * 40}, + "v1.1.0": {"type": "commit", "sha": "1" * 40}, + "v1.2.0": {"type": "commit", "sha": "2" * 40}} + self.tags = {} + self.prs = [] + self.detail = None + self.proposal_head = None + self.transform = None + self.raw_response = None + self.status_code = 200 + self.omit_length = False + self.extra_headers = {} + test = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + test.calls.append(("GET", self.path, self.headers.get("Authorization"))) + test.counts[self.path] = test.counts.get(self.path, 0) + 1 + try: + data = test.route(self.path) + if test.transform: + data = test.transform(self.path, copy.deepcopy(data)) + raw = test.raw_response if test.raw_response is not None else encode(data) + self.send_response(test.status_code) + self.send_header("Content-Type", "application/json") + if not test.omit_length and "Content-Length" not in test.extra_headers: + self.send_header("Content-Length", str(len(raw))) + for key, value in test.extra_headers.items(): + self.send_header(key, value) + self.end_headers() + try: + self.wfile.write(raw) + except (BrokenPipeError, ConnectionResetError): + pass + except (AssertionError, KeyError): + self.send_error(500) + + def refused_method(self): + test.calls.append((self.command, self.path, self.headers.get("Authorization"))) + self.send_error(405) + + do_POST = do_PATCH = do_PUT = do_DELETE = do_HEAD = refused_method + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + thread = threading.Thread(target=self.server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 5) + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.origin = "http://127.0.0.1:" + str(self.server.server_port) + # Registered last, so it checks the recorded requests before server cleanup. + self.addCleanup(self.assert_get_only) + + def assert_get_only(self): + self.assertTrue(all(call[0] == "GET" for call in self.calls), self.calls) + + def route(self, raw_path): + parsed = urlsplit(raw_path) + path = unquote(parsed.path) + query = parse_qs(parsed.query) + if path in ("/repos/" + HUB, "/repos/" + ENGINE): + name = path.removeprefix("/repos/") + result = copy.deepcopy(self.repos[name]) + if name == ENGINE and self.default_moved and self.counts[raw_path] > 1: + result["default_branch"] = "next/default" + return result + if path == f"/repos/{HUB}/git/ref/heads/main": + value = "f" * 40 if self.hub_moved and self.counts[raw_path] > 1 else self.hub + return {"ref": "refs/heads/main", "object": {"type": "commit", "sha": value}} + if path.startswith(f"/repos/{ENGINE}/git/ref/heads/"): + name = path.split("/git/ref/heads/", 1)[1] + assert name in (self.repos[ENGINE]["default_branch"], "next/default", self.proposal_head) + if name == self.proposal_head: + value = self.head + else: + value = "e" * 40 if self.target_moved and self.counts[raw_path] > 1 else self.target + return {"ref": "refs/heads/" + name, "object": {"type": "commit", "sha": value}} + if path.startswith(f"/repos/{ENGINE}/git/ref/tags/"): + name = path.split("/git/ref/tags/", 1)[1] + return {"ref": "refs/tags/" + name, "object": self.refs[name]} + if path.startswith(f"/repos/{ENGINE}/git/tags/"): + return self.tags[path.rsplit("/", 1)[-1]] + if path.startswith(f"/repos/{HUB}/contents/"): + assert query == {"ref": [self.hub]} + name = path.split("/contents/", 1)[1] + return envelope(name, self.files[name]) + if path.startswith(f"/repos/{ENGINE}/contents/"): + assert path.endswith("/contents/README.md") and set(query) == {"ref"} + return envelope("README.md", self.readmes[query["ref"][0]]) + if path == f"/repos/{ENGINE}/pulls": + assert set(query) == {"head", "base", "state", "per_page", "page"} + assert query["state"] == ["open"] and query["per_page"] == ["100"] + assert query["base"] == [self.repos[ENGINE]["default_branch"]] + assert query["head"][0].startswith(OWNER + ":automation/facts-engine-") + return self.prs if query["page"] == ["1"] else [] + if path == f"/repos/{ENGINE}/pulls/7": + return self.detail if self.detail is not None else self.prs[0] + raise AssertionError("unexpected request") + + def environment(self, **extra): + env = dict(os.environ) + env.update(FACTS_READ_TOKEN=TOKEN, GITHUB_SHA="9" * 40, + GITHUB_EVENT_NAME="repository_dispatch", NODE_OPTIONS="--invalid-option-must-not-reach-bridge") + env.update(extra) + return env + + def plan(self): + with patch.dict(os.environ, self.environment(), clear=True): + return build_engine_plan(PublicGitHub(self.origin), render_with_node) + + def invoke(self, env=None, api=None, output=None, file_limit=None): + if output is None: + self.outputs += 1 + output = self.directory / ("output-" + str(self.outputs)) + def limit_files(): + signal.signal(signal.SIGXFSZ, signal.SIG_IGN) + resource.setrlimit(resource.RLIMIT_FSIZE, (file_limit, file_limit)) + + result = subprocess.run([sys.executable, str(CLI), "plan", "--output-dir", str(output), + "--test-api", self.origin if api is None else api], + env=self.environment(**(env or {})), capture_output=True, + preexec_fn=limit_files if file_limit is not None else None) + return result, output + + def assert_refused(self, **arguments): + result, output = self.invoke(**arguments) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, + b"engine-readme-plan: input unavailable or refused; no remote changes\n") + self.assertFalse(output.exists()) + for sensitive in (TOKEN, PRIVATE_TEXT, str(output)): + self.assertNotIn(sensitive.encode(), result.stdout + result.stderr) + + def add_pull(self, plan): + self.proposal_head = plan["target"]["head"] + self.prs = [{"id": 7007, "number": 7, "state": "open", "title": "Title is not an identity", + "html_url": f"https://github.com/{ENGINE}/pull/7", + "url": f"{API_ORIGIN}/repos/{ENGINE}/pulls/7", + "body": plan["proposed_pr"]["body"], + "head": {"ref": self.proposal_head, "label": OWNER + ":" + self.proposal_head, + "sha": self.head, "repo": copy.deepcopy(self.repos[ENGINE])}, + "base": {"ref": "trunk", "label": OWNER + ":trunk", "sha": self.target, + "repo": copy.deepcopy(self.repos[ENGINE])}}] + + def test_create_plan_binds_real_source_renderer_and_unmarked_utf8_bytes(self): + plan = self.plan() + self.assertEqual(plan["schema"], SCHEMA) + self.assertEqual(plan["status"], "create-needed") + self.assertEqual(plan["proposed_readme"].encode(), EXPECTED) + self.assertEqual(plan["readme"], {"path": "README.md", "before_sha256": sha256(README), + "after_sha256": sha256(EXPECTED), "markers": 3, "changed": True}) + self.assertEqual(plan["snapshot"]["commit"], self.hub) + self.assertEqual(plan["snapshot"]["facts_sha256"], sha256(self.files["facts/facts.json"])) + self.assertEqual(base64.b64decode(plan["snapshot"]["version_bytes_base64"]), b"1.1.0\n") + self.assertEqual(plan["release"]["commit"], "1" * 40) + self.assertEqual(plan["target"]["base_commit"], self.target) + self.assertNotEqual(plan["release"]["commit"], plan["target"]["base_commit"]) + identity = {"repository_role": "engine", "repository": ENGINE, "release_tag": "v1.1.0", + "facts_sha256": sha256(self.files["facts/facts.json"])} + expected_id = sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()) + self.assertEqual(plan["plan_identity"], expected_id) + self.assertEqual(plan["target"]["head"], "automation/facts-engine-" + expected_id[:24]) + self.assertEqual(plan["proposed_pr"]["body_sha256"], sha256(plan["proposed_pr"]["body"].encode())) + binding = json.loads(plan["proposed_pr"]["body"].split("```json\n")[1].split("\n```")[0]) + self.assertEqual(binding["tag_commit"], "1" * 40) + self.assertEqual(binding["base_commit"], self.target) + self.assertEqual(binding["readme_after_sha256"], sha256(EXPECTED)) + paths = [path for _, path, _ in self.calls] + self.assertEqual([path for path in paths if "/contents/" in path], [ + f"/repos/{HUB}/contents/facts/facts.json?ref={self.hub}", + f"/repos/{HUB}/contents/facts/facts.schema.json?ref={self.hub}", + f"/repos/{HUB}/contents/VERSION?ref={self.hub}", + f"/repos/{ENGINE}/contents/README.md?ref={self.target}"]) + self.assertNotIn("facts-source", plan["proposed_readme"]) + self.assertTrue(all(auth == "Bearer " + TOKEN for _, _, auth in self.calls)) + self.assertEqual(plan["remote_writes"], 0) + json.dumps(plan, allow_nan=False) + + def test_cli_outputs_review_manifest_and_proposed_bytes_after_rechecks(self): + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + receipt = json.loads(result.stdout) + manifest = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(receipt["status"], "create-needed") + self.assertEqual((output / "README.md").read_bytes(), EXPECTED) + self.assertNotIn("proposed_readme", manifest) + self.assertTrue(manifest["review_only"]) + self.assertEqual(manifest["recheck"]["hub_commit"], manifest["snapshot"]["commit"]) + self.assertEqual(manifest["recheck"]["base_commit"], self.target) + self.assertNotIn(TOKEN, (output / "plan.json").read_text()) + self.assertEqual(set(path.name for path in output.iterdir()), {"plan.json", "README.md"}) + + def test_current_metadata_selects_non_main_default_branch_and_distinct_commits(self): + for base, target, version in (("stable/docs", "3" * 40, b"1.0.1"), + ("readme-next", "4" * 40, b"1.2.0\n")): + with self.subTest(base=base): + self.hub = "5" * 40 + self.target = target + self.files["VERSION"] = version + self.repos[ENGINE]["default_branch"] = base + self.readmes[target] = README + self.calls.clear() + plan = self.plan() + self.assertEqual(plan["snapshot"]["commit"], "5" * 40) + self.assertEqual(plan["target"]["base"], base) + self.assertEqual(plan["target"]["base_commit"], target) + self.assertEqual(plan["release"]["tag"], "v" + version.decode().strip()) + self.assertNotEqual(plan["release"]["commit"], target) + self.assertEqual([path for _, path, _ in self.calls if f"/repos/{ENGINE}/contents/" in path], + [f"/repos/{ENGINE}/contents/README.md?ref={target}"]) + + def test_new_tag_same_facts_fetches_new_tag_and_repairs_drift(self): + first = self.plan() + self.calls.clear() + self.hub = "7" * 40 + self.files["VERSION"] = b"1.2.0" + second = self.plan() + self.assertEqual(second["status"], "create-needed") + self.assertEqual(first["snapshot"]["facts_sha256"], second["snapshot"]["facts_sha256"]) + self.assertNotEqual(first["plan_identity"], second["plan_identity"]) + self.assertNotEqual(first["target"]["head"], second["target"]["head"]) + self.assertIn(f"/repos/{ENGINE}/git/ref/tags/v1.2.0", [path for _, path, _ in self.calls]) + self.assertIn(f"/repos/{ENGINE}/contents/README.md?ref={self.target}", + [path for _, path, _ in self.calls]) + # The existing README's historic token scope is preserved even under a new trigger tag. + self.assertEqual(second["proposed_readme"].encode(), EXPECTED) + + def test_new_tag_without_readme_drift_is_full_identity_noop(self): + before = self.plan() + self.hub = "8" * 40 + self.files["VERSION"] = b"1.2.0\n" + self.readmes[self.target] = EXPECTED + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(plan["status"], "noop") + self.assertEqual(plan["release"], {"tag": "v1.2.0", "commit": "2" * 40, + "source_url": "https://github.com/pineforge-4pass/pineforge-engine/blob/v1.2.0/README.md#validation-scoreboard"}) + self.assertNotEqual(before["plan_identity"], plan["plan_identity"]) + self.assertFalse((output / "README.md").exists()) + self.assertNotIn("proposed_pr", plan) + + def test_changed_facts_same_tag_changes_render_identity_and_branch(self): + first = self.plan() + facts = json.loads(self.files["facts/facts.json"]) + facts["inventory"]["tvTrades"] = 4776345 + self.hub = "7" * 40 + self.files["facts/facts.json"] = encode(facts) + second = self.plan() + self.assertEqual(second["release"]["tag"], first["release"]["tag"]) + self.assertNotEqual(second["plan_identity"], first["plan_identity"]) + self.assertNotEqual(second["target"]["head"], first["target"]["head"]) + self.assertEqual(second["proposed_readme"].encode(), EXPECTED.replace(b"4,776,328", b"4,776,345")) + self.assertIn("superseded open generations", second["limitations"][0]) + + def test_reserialized_identical_facts_bind_original_raw_hash(self): + first = self.plan() + self.hub = "8" * 40 + self.files["facts/facts.json"] = b"\n" + self.files["facts/facts.json"] + b" " + second = self.plan() + self.assertNotEqual(first["snapshot"]["facts_sha256"], second["snapshot"]["facts_sha256"]) + self.assertNotEqual(first["plan_identity"], second["plan_identity"]) + self.assertEqual(first["readme"]["after_sha256"], second["readme"]["after_sha256"]) + + def test_repeat_plan_inspects_current_pr_head_and_body_before_already_planned(self): + first = self.plan() + self.add_pull(first) + self.calls.clear() + second = self.plan() + self.assertEqual(second["status"], "already-planned") + self.assertEqual(second["plan_identity"], first["plan_identity"]) + self.assertEqual(second["target"]["head"], first["target"]["head"]) + self.assertEqual(second["existing_pr"]["url"], f"https://github.com/{ENGINE}/pull/7") + paths = [path for _, path, _ in self.calls] + self.assertIn(f"/repos/{ENGINE}/pulls/7", paths) + self.assertIn(f"/repos/{ENGINE}/contents/README.md?ref={self.head}", paths) + self.assertIn(f"/repos/{ENGINE}/git/ref/heads/" + self.proposal_head.replace("/", "%2F"), paths) + query = parse_qs(urlsplit(next(path for path in paths if "/pulls?" in path)).query) + self.assertEqual(query["head"], [OWNER + ":" + first["target"]["head"]]) + self.assertEqual(query["base"], ["trunk"]) + + def test_existing_pr_readme_or_body_drift_requires_update(self): + first = self.plan() + self.add_pull(first) + for kind in ("readme", "body", "base"): + with self.subTest(kind=kind): + self.readmes[self.head] = README if kind == "readme" else EXPECTED + self.prs[0]["body"] = first["proposed_pr"]["body"] + ("\n" if kind == "body" else "") + self.prs[0]["base"]["sha"] = "3" * 40 if kind == "base" else self.target + plan = self.plan() + self.assertEqual(plan["status"], "update-needed") + self.assertEqual(plan["existing_pr"]["number"], 7) + self.assertEqual(plan["target"]["head"], first["target"]["head"]) + + def test_pr_detail_and_actual_head_ref_are_distinct_evidence(self): + first = self.plan() + self.add_pull(first) + self.detail = copy.deepcopy(self.prs[0]) + self.prs[0]["head"]["sha"] = "4" * 40 + self.assertEqual(self.plan()["status"], "already-planned") + self.head = "5" * 40 + self.readmes[self.head] = EXPECTED + plan = self.plan() + self.assertEqual(plan["status"], "update-needed") + self.assertEqual(plan["existing_pr"]["head_commit"], "5" * 40) + self.assertEqual(plan["existing_pr"]["reported_head_commit"], "d" * 40) + + def test_source_target_or_default_branch_movement_supersedes_without_proposal(self): + for flag in ("hub_moved", "target_moved", "default_moved"): + with self.subTest(flag=flag): + self.counts.clear() + self.hub_moved = self.target_moved = self.default_moved = False + setattr(self, flag, True) + result, output = self.invoke() + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads((output / "plan.json").read_bytes()) + self.assertEqual(plan["status"], "superseded") + self.assertNotIn("proposed_pr", plan) + self.assertNotIn("proposed_readme", plan) + self.assertFalse((output / "README.md").exists()) + + def test_direct_and_eight_annotated_tags_bind_exact_release_commit(self): + for depth in (0, 1, 8): + with self.subTest(depth=depth): + self.tags.clear() + pointers = [f"{20 + i:040x}" for i in range(depth)] + for index, sha in enumerate(pointers): + child = {"type": "tag", "sha": pointers[index + 1]} if index + 1 < depth else { + "type": "commit", "sha": "6" * 40} + self.tags[sha] = {"sha": sha, "tag": "v1.1.0", "object": child} + self.refs["v1.1.0"] = {"type": "tag", "sha": pointers[0]} if depth else { + "type": "commit", "sha": "6" * 40} + plan = self.plan() + self.assertEqual(plan["release"]["commit"], "6" * 40) + self.assertEqual(plan["target"]["base_commit"], "b" * 40) + + def test_annotated_tag_cycles_excess_depth_and_wrong_types_refuse(self): + for kind in ("cycle", "depth", "type", "identity"): + with self.subTest(kind=kind): + self.calls.clear() + pointers = [f"{30 + i:040x}" for i in range(9)] + self.refs["v1.1.0"] = {"type": "tag", "sha": pointers[0]} + self.tags = {sha: {"sha": sha, "tag": "v1.1.0", "object": { + "type": "tag", "sha": pointers[(index + 1) % 9]}} + for index, sha in enumerate(pointers)} + if kind == "cycle": + self.tags[pointers[0]]["object"]["sha"] = pointers[0] + elif kind == "type": + self.tags[pointers[0]]["object"]["type"] = "tree" + elif kind == "identity": + self.tags[pointers[0]]["sha"] = "7" * 40 + self.assert_refused() + tag_reads = [path for _, path, _ in self.calls if f"/repos/{ENGINE}/git/tags/" in path] + self.assertEqual(len(tag_reads), 8 if kind == "depth" else 1) + + def test_private_mismatched_and_malformed_repository_metadata_refuse(self): + for repo in (HUB, ENGINE): + original = copy.deepcopy(self.repos[repo]) + for field, value in (("private", True), ("full_name", "untrusted/repository"), + ("id", True), ("owner", {"login": "foreign"}), + ("default_branch", "unsafe/../name"), ("url", "http://127.0.0.1/private")): + with self.subTest(repo=repo, field=field): + self.repos[repo] = dict(original, **{field: value}) + self.assert_refused() + self.repos[repo] = original + + def test_contents_path_type_name_size_encoding_and_blob_identity_refuse(self): + for name in ("facts/facts.json", "facts/facts.schema.json", "VERSION", "README.md"): + for field, value in (("path", "foreign/path"), ("type", "symlink"), ("name", "foreign"), + ("size", True), ("size", 1), ("sha", "0" * 40), + ("encoding", "none"), ("content", "%%%")): + with self.subTest(name=name, field=field, value=value): + def corrupt(path, data, name=name, field=field, value=value): + if urlsplit(path).path.endswith("/contents/" + name): + data[field] = value + return data + self.transform = corrupt + self.assert_refused() + self.transform = None + + def test_remote_schema_change_and_external_ref_are_refused_before_renderer(self): + original = self.files["facts/facts.schema.json"] + for raw in (b" " + original, encode({"$ref": self.origin + "/private-schema"})): + with self.subTest(raw=raw[:24]): + self.files["facts/facts.schema.json"] = raw + self.calls.clear() + self.assert_refused() + self.assertFalse(any("/git/ref/tags/" in path for _, path, _ in self.calls)) + self.assertFalse(any("private-schema" in path for _, path, _ in self.calls)) + + def test_invalid_actual_facts_and_version_fail_without_selector_fallback(self): + original = dict(self.files) + for filename, raw in (("facts/facts.json", b'{"schema":"a","schema":"b"}'), + ("facts/facts.json", b'{"schema":"wrong"}'), + ("VERSION", b"999.1.0"), ("VERSION", b"1.1.0\r\n"), + ("VERSION", b"1.1.0/unsafe"), ("VERSION", b"1.1.0\n\n")): + with self.subTest(filename=filename, raw=raw): + self.files = dict(original, **{filename: raw}) + self.assert_refused() + + def test_wrong_ref_name_commit_or_tag_target_never_qualifies(self): + for field, value in (("ref", "refs/tags/wrong"), ("sha", "A" * 40), + ("sha", "1" * 39), ("type", "blob")): + with self.subTest(field=field, value=value): + def corrupt(path, data, field=field, value=value): + if path.endswith("/git/ref/tags/v1.1.0"): + if field == "ref": + data[field] = value + else: + data["object"][field] = value + return data + self.transform = corrupt + self.assert_refused() + + def test_malformed_unsafe_zero_and_unknown_markers_fail_real_bridge(self): + for raw in (b"No markers", b"Count: old", + b"old\n", + b"Count: `old`", + b"Count: old", + b"Count: outer " + b"nested", b"\xffx"): + with self.subTest(raw=raw[:50]): + self.readmes[self.target] = raw + self.assert_refused() + + def test_pr_duplicates_ambiguity_and_foreign_ownership_refuse(self): + first = self.plan() + self.add_pull(first) + original = copy.deepcopy(self.prs[0]) + cases = ("duplicate", "ambiguous", "head", "base", "repo", "private", "url", "state", "body", "marker") + for case in cases: + with self.subTest(case=case): + self.prs = [copy.deepcopy(original)] + row = self.prs[0] + if case in ("duplicate", "ambiguous"): + other = copy.deepcopy(row) + if case == "ambiguous": + other.update(id=7008, number=8, html_url=f"https://github.com/{ENGINE}/pull/8", + url=f"{API_ORIGIN}/repos/{ENGINE}/pulls/8") + self.prs.append(other) + elif case in ("head", "base"): + row[case]["ref"] = "foreign" + elif case == "repo": + row["head"]["repo"]["full_name"] = "foreign/engine" + elif case == "private": + row["head"]["repo"]["private"] = True + elif case == "url": + row["html_url"] = self.origin + "/private" + elif case == "state": + row["state"] = "closed" + elif case == "body": + row["body"] = None + else: + row["body"] = row["body"].replace(first["plan_identity"], "0" * 64) + self.assert_refused() + + def test_current_pr_detail_is_validated_even_when_listing_matches(self): + first = self.plan() + self.add_pull(first) + for kind in ("id", "private", "marker"): + with self.subTest(kind=kind): + self.detail = copy.deepcopy(self.prs[0]) + if kind == "id": + self.detail["id"] += 1 + elif kind == "private": + self.detail["base"]["repo"]["private"] = True + else: + self.detail["body"] += "\n" + self.assert_refused() + + def test_pr_page_malformed_truncated_foreign_and_over_bound_links_refuse(self): + for kind in ("malformed", "truncated", "foreign", "bound", "missing-next", "duplicate"): + with self.subTest(kind=kind): + def pagination(path, data, kind=kind): + if "/pulls?" in path: + parsed = urlsplit(path) + query = {key: values[0] for key, values in parse_qs(parsed.query).items()} + query["page"] = "21" if kind == "bound" else "2" + origin = "http://127.0.0.1:1" if kind == "foreign" else self.origin + url = origin + parsed.path + "?" + urlencode(query) + link = f'<{url}>; rel="last"' if kind in ("bound", "missing-next") else f'<{url}>; rel="next"' + if kind == "malformed": + link = "malformed" + elif kind == "duplicate": + link += ", " + link + self.extra_headers["Link"] = link + return data + self.transform = pagination + self.extra_headers.clear() + self.assert_refused() + + def test_full_pr_pages_stop_at_twenty_without_silent_selection(self): + first = self.plan() + self.add_pull(first) + original = copy.deepcopy(self.prs[0]) + + def full_pages(path, data): + if "/pulls?" not in path: + return data + page = int(parse_qs(urlsplit(path).query)["page"][0]) + rows = [] + for offset in range(100): + number = (page - 1) * 100 + offset + 7 + row = copy.deepcopy(original) + row.update(id=10000 + number, number=number, + html_url=f"https://github.com/{ENGINE}/pull/{number}", + url=f"{API_ORIGIN}/repos/{ENGINE}/pulls/{number}") + rows.append(row) + return rows + + self.transform = full_pages + self.calls.clear() + self.assert_refused() + pages = [int(parse_qs(urlsplit(path).query)["page"][0]) + for _, path, _ in self.calls if "/pulls?" in path] + self.assertEqual(pages, list(range(1, 21))) + + def test_pr_page_wrong_shape_and_more_than_100_rows_refuse(self): + for rows in ({"items": []}, [None] * 101): + with self.subTest(shape=type(rows).__name__): + self.transform = lambda path, data, rows=rows: rows if "/pulls?" in path else data + self.assert_refused() + + def test_http_errors_bad_json_and_oversize_are_fixed_safe_refusals(self): + for kind in ("error", "json", "oversize", "undeclared-oversize", "truncated", "encoding"): + with self.subTest(kind=kind): + self.status_code = 503 if kind == "error" else 200 + self.raw_response = (PRIVATE_TEXT.encode() if kind in ("error", "json") else + b" " * (MAX_INPUT_BYTES + 1) if "oversize" in kind else b"{}") + self.extra_headers = {"Content-Encoding": "gzip"} if kind == "encoding" else {} + self.omit_length = kind == "undeclared-oversize" + if kind == "undeclared-oversize": + # Exercise the actual bounded read of an HTTP/1.0 close-delimited body. + self.raw_response = b" " * MAX_INPUT_BYTES + b"!" + elif kind == "truncated": + self.extra_headers["Content-Length"] = "1234" + self.assert_refused() + + def test_redirect_never_sends_token_to_another_origin(self): + foreign_calls = [] + + class Foreign(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + foreign_calls.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + foreign = ThreadingHTTPServer(("127.0.0.1", 0), Foreign) + thread = threading.Thread(target=foreign.serve_forever, daemon=True) + thread.start() + try: + self.status_code = 302 + self.extra_headers["Location"] = f"http://127.0.0.1:{foreign.server_port}/private" + self.raw_response = PRIVATE_TEXT.encode() + self.assert_refused() + self.assertEqual(foreign_calls, []) + finally: + foreign.shutdown() + foreign.server_close() + thread.join(5) + + def test_proxy_environment_is_ignored_by_production_http_transport(self): + result, _ = self.invoke(env={"HTTP_PROXY": "http://127.0.0.1:1", "HTTPS_PROXY": "http://127.0.0.1:1", + "http_proxy": "http://127.0.0.1:1", "https_proxy": "http://127.0.0.1:1", + "NO_PROXY": "", "no_proxy": ""}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.calls) + + def test_test_origin_and_real_token_refuse_before_any_request(self): + for origin in (self.origin + "/", "http://localhost:1234", "https://127.0.0.1:1234", + self.origin + "?query=1", "http://user@127.0.0.1:1234", + "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://127.0.0.1:bad"): + with self.subTest(origin=origin): + self.assert_refused(api=origin) + self.assert_refused(env={"FACTS_READ_TOKEN": "github-real-credential"}) + self.assertEqual(self.calls, []) + + def test_existing_output_directory_file_and_symlink_are_never_overwritten(self): + for kind in ("directory", "file", "symlink"): + with self.subTest(kind=kind): + output = self.directory / kind + if kind == "directory": + output.mkdir() + (output / "keep").write_bytes(b"keep") + elif kind == "file": + output.write_bytes(b"keep") + else: + output.symlink_to(self.directory / "missing") + result, _ = self.invoke(output=output) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, b"") + if kind == "directory": + self.assertEqual((output / "keep").read_bytes(), b"keep") + elif kind == "file": + self.assertEqual(output.read_bytes(), b"keep") + else: + self.assertTrue(output.is_symlink()) + self.assertEqual(self.calls, []) + + def test_native_artifact_write_failure_leaves_no_partial_success_directory(self): + # The real filesystem rejects the larger manifest after the README was written. + self.assert_refused(file_limit=1024) + self.assertTrue(any("/contents/README.md?ref=" in path for _, path, _ in self.calls)) + + +class EngineRenderBridge(unittest.TestCase): + def bridge(self, raw): + return subprocess.run(["node", str(BRIDGE)], input=raw, capture_output=True, + env={"PATH": os.environ.get("PATH", os.defpath)}) + + def test_real_bridge_returns_actual_bytes_and_count_for_changed_public_facts(self): + facts = json.loads(FACTS.read_bytes()) + for count, expected in ((4776328, EXPECTED), + (4776345, EXPECTED.replace(b"4,776,328", b"4,776,345"))): + with self.subTest(count=count): + facts["inventory"]["tvTrades"] = count + result = self.bridge(encode({"text": README.decode(), "facts": facts})) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, b"") + value = json.loads(result.stdout) + self.assertEqual(value["markers"], 3) + self.assertEqual(value["text"].encode(), expected) + + def test_bridge_invalid_input_nonzero_exit_has_only_fixed_stderr(self): + for raw in (b"\xff", b"{", b"[]", encode({"text": "x", "facts": []}), + encode({"text": README.decode(), "facts": {}, "file": PRIVATE_TEXT}), + encode({"text": "No markers", "facts": {}}), + encode({"text": "Count: x", "facts": {}}), + encode({"text": "\ud800", "facts": {}}), b" " * (MAX_INPUT_BYTES + 1)): + with self.subTest(raw=raw[:40]): + result = self.bridge(raw) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"engine-readme-render: invalid input or render failure\n") + + def test_python_rejects_invalid_renderer_protocol_values(self): + # Protocol unit negatives only; every accepted controller plan above executes the real bridge. + for value in (None, {}, {"text": "x", "markers": 0}, {"text": "x", "markers": True}, + {"text": "\ud800", "markers": 1}, {"text": "x", "markers": 1, "extra": 1}, + {"text": "x" * (MAX_INPUT_BYTES + 1), "markers": 1}): + with self.subTest(kind=type(value).__name__): + with self.assertRaises(InputError): + checked_render(value) + + +if __name__ == "__main__": + unittest.main()