From 04660f413ef58a9953fe38773f4a4d8eb1ed0468 Mon Sep 17 00:00:00 2001
From: Jammy2211 <JNightingale2211@gmail.com>
Date: Mon, 12 Oct 2026 12:02:38 +0100
Subject: [PATCH] feat: add resumable local Broca instance setup

---
 README.md           |   6 ++
 broca/__main__.py   |  13 ++++
 broca/setup.py      | 152 ++++++++++++++++++++++++++++++++++++++
 docs/setup.md       |  66 +++++++++++++++++
 tests/test_setup.py | 176 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 413 insertions(+)
 create mode 100644 broca/setup.py
 create mode 100644 docs/setup.md
 create mode 100644 tests/test_setup.py

diff --git a/README.md b/README.md
index c486f7b..60ad723 100644
--- a/README.md
+++ b/README.md
@@ -94,3 +94,9 @@ provenance, freshness and next-action guidance.
 The optional `board_base_url` in `assistants.json` enables the shared organ
 footer for this Scientist instance. It is the hosting root, without an organ
 repository suffix; membership and order come from Brain’s board registry.
+
+## Start an independent instance
+
+Use the [local setup guide](docs/setup.md) to preview and create empty user-owned
+data, resume interrupted setup and validate a local board/feed. Publication and
+Scientist connection follow as explicit steps.
diff --git a/broca/__main__.py b/broca/__main__.py
index b573f06..dc1f93f 100644
--- a/broca/__main__.py
+++ b/broca/__main__.py
@@ -13,6 +13,12 @@ def main(argv=None):
     parser.add_argument("--root", type=Path, default=Path.cwd(), help="Broca data checkout")
     sub = parser.add_subparsers(dest="command", required=True)
     sub.add_parser("check")
+    setup = sub.add_parser("setup", help="preview or resume empty local instance setup")
+    setup.add_argument("--repos", type=Path, required=True)
+    setup.add_argument("--repo", required=True, help="canonical repository key")
+    setup.add_argument("--pages-url", required=True)
+    setup.add_argument("--brain", type=Path, required=True)
+    setup.add_argument("--apply", action="store_true")
     add = sub.add_parser("ingest")
     add.add_argument("record", type=Path)
     collect = sub.add_parser("refresh")
@@ -28,6 +34,13 @@ def main(argv=None):
     board.add_argument("--pages-url", help="override the configured public board directory")
     args = parser.parse_args(argv)
     try:
+        if args.command == "setup":
+            from .setup import prepare, apply
+            prepared = prepare(args.root, args.repos, args.repo, args.pages_url, args.brain)
+            missing = apply(prepared) if args.apply else prepared[2]
+            print(("Created: " if args.apply else "Would create: ") + (", ".join(missing) or "nothing"))
+            print("Local setup only; publication and Scientist connection remain separate.")
+            return 0
         names = config(args.root)["assistants"]
         if args.command == "check":
             records = load_records(args.root, names)
diff --git a/broca/setup.py b/broca/setup.py
new file mode 100644
index 0000000..995b15e
--- /dev/null
+++ b/broca/setup.py
@@ -0,0 +1,152 @@
+"""Offline, create-only initialization of a user-owned Broca data directory."""
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import tempfile
+
+import yaml
+
+from .state import pages_directory
+
+MANIFEST = "setup.json"
+
+
+def encoded(value):
+    return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
+
+
+def plain_path(path):
+    """Reject symlink destinations, including symlink parents, before resolving."""
+    path = Path(os.path.abspath(path))
+    if any(p.is_symlink() for p in (path, *path.parents)):
+        raise ValueError(f"setup path must not use symlinks: {path}")
+    return path
+
+
+def prepare(root, repos, repo, pages_url, brain):
+    root = plain_path(root)
+    if not root.parent.is_dir():
+        raise ValueError("destination parent must already exist")
+    if root.exists() and not root.is_dir():
+        raise ValueError("destination must be a directory")
+    repos, brain = Path(repos).resolve(), Path(brain).resolve()
+    try:
+        catalogue = yaml.safe_load(repos.read_text())
+    except yaml.YAMLError as exc:
+        raise ValueError("invalid canonical repository map") from exc
+    entries = catalogue.get("repos") if isinstance(catalogue, dict) else None
+    row = entries.get(repo) if isinstance(entries, dict) else None
+    identity = row.get("github") if isinstance(row, dict) else None
+    if not isinstance(identity, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", identity):
+        raise ValueError("repo must identify a canonical owner/repository in the supplied map")
+    pages = pages_directory(pages_url)
+    if not all((brain / "board" / name).is_file() for name in ("_state.py", "_theme.py")):
+        raise ValueError("Brain checkout needs the shared board and state contract")
+    manifest = {"schema_version": 1, "repo": repo, "repository": identity,
+                "pages_url": pages, "repos": str(repos), "brain": str(brain),
+                "root": str(root), "framework": str(Path(__file__).resolve().parents[1])}
+    cfg = {"schema_version": 1, "assistants": [], "evidence_max_age_days": 30,
+           "collection_max_age_days": 30, "pages_url": pages}
+    checkin = ("Review this instance's saved communication inventory and assistant evidence.\n"
+               "Read the canonical Mind map and existing task before proposing changes.\n"
+               "Missing collection or evaluation is unknown, never a pass. Preserve records.\n"
+               "Do not collect, evaluate, schedule or publish without explicit authorization.\n")
+    q = shlex.quote
+    prefix = f"python -m broca --root {q(str(root))}"
+    resume = (f"{prefix} setup --repos {q(str(repos))} --repo {q(repo)} "
+              f"--pages-url {q(pages)} --brain {q(str(brain))}")
+    render = (f"{prefix} render --brain {q(str(brain))} --repository {q(identity)} "
+              f"--output {q(str(root / 'site/index.html'))} "
+              f"--state-output {q(str(root / 'site/state.json'))}")
+    handoff = f"""# Local Broca setup handoff
+
+Keep this file and setup.json private: they contain local paths.
+From the framework checkout, with Python and its dependencies available:
+
+```sh
+cd {q(manifest['framework'])}
+{resume}
+# After reviewing the preview, repeat with --apply to finish missing files.
+{prefix} check
+{render}
+```
+
+For a fresh conversation: read setup.json, this file, the canonical map and the
+existing Mind task. Preview setup to identify missing or conflicting files. Resume
+only this setup; never reinitialize or overwrite user edits. If configuration has
+changed intentionally since setup, use check/render rather than restoring defaults.
+The manifest records setup inputs, not task status or evidence of activation.
+
+This prepares local data only. No collection or evaluation has occurred. An empty
+feed is grey with an unknown collection timestamp. Projects, tasks and research
+records remain in their existing homes. Add communication resources using Broca's
+existing inventory procedure; do not copy a maintainer's records or receipts.
+
+Publishing needs a separate approved task using the configured identity and public
+board location. Publish only site/index.html and site/state.json after reviewing
+visibility. Do not publish this directory wholesale. After the real HTTPS feed is
+available, use Scientist's connect_organ.py preview/apply with organ broca, repo
+{repo!r}, and pages URL {pages!r}. Until that validation succeeds, setup is not a
+connected or active Scientist capability. Missing or stale evidence after connection
+remains evidence trouble; it does not mean the organ is uninstalled.
+"""
+    files = {MANIFEST: encoded(manifest), "assistants.json": encoded(cfg),
+             "CHECKIN.md": checkin.encode(), "SETUP.md": handoff.encode()}
+    # Only a manifest from this exact setup grants permission to fill missing files
+    # in an existing directory. Never adopt a pre-existing live data checkout.
+    if root.exists() and any(root.iterdir()) and not (root / MANIFEST).exists():
+        raise ValueError("nonempty destination has no setup manifest; refusing adoption")
+    missing = inspect(root, files)
+    return root, files, missing
+
+
+def inspect(root, files):
+    plain_path(root)
+    missing = []
+    for name, content in files.items():
+        path = root / name
+        if path.is_symlink() or (path.exists() and (not path.is_file() or path.read_bytes() != content)):
+            raise ValueError(f"setup conflict: {name}; preserve it and review manually")
+        if not path.exists():
+            missing.append(name)
+    return missing
+
+
+def create_file(path, content):
+    """Publish a complete file without replacing a concurrently created target."""
+    fd, temporary = tempfile.mkstemp(prefix=".broca-setup-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.link(temporary, path)  # atomic create-only; an existing target wins
+    finally:
+        os.unlink(temporary)
+
+
+def apply(prepared):
+    root, files, _ = prepared
+    root.mkdir(exist_ok=True)
+    if any(root.iterdir()) and not (root / MANIFEST).exists():
+        raise ValueError("destination changed before apply; no setup manifest")
+    missing = inspect(root, files)  # preflight every conflict before writing
+    for name in missing:  # manifest is published first; interruption is resumable
+        inspect(root, files)
+        create_file(root / name, files[name])
+    inspect(root, files)
+    from .collect import check_receipts, config, load_snapshot
+    from .records import load_records
+    from .state import build, shared_contract
+    cfg = config(root)
+    check_receipts(root)
+    load_snapshot(root)
+    load_records(root, cfg["assistants"])
+    manifest = json.loads(files[MANIFEST])
+    state = build(root, manifest["brain"], manifest["repository"])
+    errors = shared_contract(manifest["brain"]).validate_state(state)
+    if errors:
+        raise ValueError(f"owner feed validation failed: {errors}")
+    return missing
diff --git a/docs/setup.md b/docs/setup.md
new file mode 100644
index 0000000..b30e676
--- /dev/null
+++ b/docs/setup.md
@@ -0,0 +1,66 @@
+# Set up your own local Broca data
+
+Use an existing Broca framework checkout with its Python dependencies and a
+compatible Brain checkout. Keep user data in a separate directory. Register your
+chosen repository identity in your canonical Mind `repos.yaml` first; its key and
+GitHub name need not contain “Broca”. No GitHub access is needed for local setup.
+
+From the Broca framework checkout:
+
+```sh
+python -m broca --root /path/to/my-guides setup \
+  --repos /path/to/Mind/repos.yaml --repo my-guides \
+  --pages-url https://research.example/guides/ --brain /path/to/PyAutoBrain
+```
+
+Review the file list, then repeat with `--apply`. The parent directory must exist.
+A new destination or an empty directory is accepted; an existing nonempty directory
+requires the matching setup manifest. Symlink destinations are refused.
+
+Setup creates four files: `setup.json`, `assistants.json`, `CHECKIN.md` and
+`SETUP.md`. The manifest stores the identity and local paths needed to resume.
+Assistant selection starts empty. There are no copied measurements, collection
+receipts, task records or invented resource inventories. Setup does not install
+Python dependencies, clone repositories or configure a harness.
+
+Each file is created atomically without replacing an existing file. After an
+interruption, rerun the same command: matching files remain untouched and missing
+files are filled in. A conflicting setup file stops the operation before writes;
+inspect it manually. A crash before the first manifest was published may leave a
+`.broca-setup-*` temporary file; inspect that new destination before removing only
+that temporary file and retrying. Do not run setup concurrently with manual edits.
+
+Once you customize configuration, validate it with `check` and `render`; setup
+will refuse to restore defaults over your changes. It is an initializer, not an
+upgrade or repair command. Keep the same framework version and paths when resuming
+an unfinished setup. Record-preserving upgrades remain a separate pilot phase.
+
+## Validate and resume
+
+The generated `SETUP.md` has commands with your actual paths for:
+
+- Previewing and resuming setup from a fresh conversation.
+- Checking saved owner records with `python -m broca --root … check`.
+- Rendering a local `site/index.html` and `site/state.json` with canonical identity.
+
+Applying setup validates the local owner state and shared feed contract. Rendering
+remains an explicit command. With no collection evidence, the feed is grey and its
+required timestamp is the unknown-time sentinel, not the setup time. Resource
+inventory remains unavailable until generated through the existing communication
+inventory procedure. Setup success is not evidence of communication quality.
+
+Keep `SETUP.md` and `setup.json` private: they contain local paths. They record
+setup inputs and resumption instructions; the existing Mind task owns task status.
+Project source and research records stay in their existing homes.
+
+## Connect it to Scientist
+
+Publish only the reviewed HTML and feed from `site/` through a separately approved
+publication task. Local setup neither checks remote ownership nor creates a
+repository, publishes content, schedules upkeep or evaluates assistants.
+
+When the real HTTPS feed is available, use Scientist's existing
+`connect_organ.py` preview/apply with organ key `broca`, your canonical repository
+key and the configured Pages directory. Until that check succeeds, this is local
+setup, not an active Scientist connection. Subsequent stale or unavailable evidence
+must remain visible as evidence trouble for the connected organ.
diff --git a/tests/test_setup.py b/tests/test_setup.py
new file mode 100644
index 0000000..ab07159
--- /dev/null
+++ b/tests/test_setup.py
@@ -0,0 +1,176 @@
+"""Recovery and preservation at the independent-instance setup boundary."""
+import json
+from pathlib import Path
+
+import pytest
+
+from broca.__main__ import main
+from broca import setup
+from broca.state import UNKNOWN_TIME, shared_contract
+from broca.records import digest
+
+BRAIN = Path(__file__).resolve().parents[2] / "PyAutoBrain"
+
+
+@pytest.fixture
+def inputs(tmp_path):
+    repos = tmp_path / "repos.yaml"
+    repos.write_text("repos:\n  guides:\n    github: Research/IndependentGuides\n")
+    root = tmp_path / "data"
+    return root, repos
+
+
+def args(inputs):
+    root, repos = inputs
+    return ["--root", str(root), "setup", "--repos", str(repos), "--repo", "guides",
+            "--pages-url", "https://research.example/guides/", "--brain", str(BRAIN)]
+
+
+def prepare(inputs):
+    root, repos = inputs
+    return setup.prepare(root, repos, "guides", "https://research.example/guides/", BRAIN)
+
+
+def contents(root):
+    return {p.relative_to(root): p.read_bytes() for p in root.rglob("*") if p.is_file()}
+
+
+def test_preview_apply_retry_and_render(inputs, tmp_path):
+    root, _ = inputs
+    assert main(args(inputs)) == 0
+    assert not root.exists()
+    assert main(args(inputs) + ["--apply"]) == 0
+    assert set(p.name for p in root.iterdir()) == {"setup.json", "assistants.json", "SETUP.md", "CHECKIN.md"}
+    before = contents(root)
+    assert main(args(inputs) + ["--apply"]) == 0
+    assert contents(root) == before
+    assert main(["--root", str(root), "check"]) == 0
+    site = tmp_path / "preview"
+    assert main(["--root", str(root), "render", "--brain", str(BRAIN),
+                 "--repository", "Research/IndependentGuides", "--output", str(site / "index.html"),
+                 "--state-output", str(site / "state.json")]) == 0
+    feed = json.loads((site / "state.json").read_text())
+    assert shared_contract(BRAIN).validate_state(feed) == []
+    assert feed["repo"] == "IndependentGuides"
+    assert feed["status"] == "grey" and feed["updated"] == UNKNOWN_TIME
+    assert "Collection unavailable" in feed["items"][0]["text"]
+    assert "inventory unavailable" in feed["items"][1]["text"]
+    assert contents(root) == before
+
+
+@pytest.mark.parametrize("after", [1, 2, 3])
+def test_interruption_resumes_without_rewriting_completed_files(inputs, monkeypatch, after):
+    root, _ = inputs
+    original = setup.create_file
+    count = 0
+
+    def interrupt(path, content):
+        nonlocal count
+        if count == after:
+            raise OSError("interrupted")
+        original(path, content)
+        count += 1
+
+    monkeypatch.setattr(setup, "create_file", interrupt)
+    assert main(args(inputs) + ["--apply"]) == 2
+    before = {p: (data, (root / p).stat().st_mtime_ns) for p, data in contents(root).items()}
+    monkeypatch.setattr(setup, "create_file", original)
+    assert main(args(inputs) + ["--apply"]) == 0
+    for p, (data, mtime) in before.items():
+        assert (root / p).read_bytes() == data
+        assert (root / p).stat().st_mtime_ns == mtime
+
+
+def test_conflict_is_preflighted_and_user_records_preserved(inputs):
+    root, _ = inputs
+    assert main(args(inputs) + ["--apply"]) == 0
+    (root / "research.txt").write_text("baseline-A: preserve this research")
+    (root / "records").mkdir()
+    (root / "records" / "note.txt").write_text("existing user record")
+    before = contents(root)
+    assert main(args(inputs) + ["--apply"]) == 0
+    assert contents(root) == before
+    (root / "CHECKIN.md").unlink()
+    (root / "assistants.json").write_text('{"user": "customized"}')
+    before = contents(root)
+    assert main(args(inputs) + ["--apply"]) == 2
+    assert contents(root) == before
+    assert not (root / "CHECKIN.md").exists()
+
+
+@pytest.mark.parametrize("kind", ["identity", "url", "map", "brain", "nonempty", "symlink"])
+def test_invalid_inputs_do_not_initialize(inputs, tmp_path, kind):
+    root, repos = inputs
+    argv = args(inputs)
+    if kind == "identity":
+        argv[argv.index("--repo") + 1] = "missing"
+    elif kind == "url":
+        argv[argv.index("--pages-url") + 1] = "http://research.example/"
+    elif kind == "map":
+        repos.write_text("repos: []")
+    elif kind == "brain":
+        argv[argv.index("--brain") + 1] = str(tmp_path)
+    elif kind == "nonempty":
+        root.mkdir()
+        (root / "research.txt").write_text("preserve")
+    else:
+        other = tmp_path / "other"
+        other.mkdir()
+        root.symlink_to(other, target_is_directory=True)
+    before = contents(root)
+    assert main(argv + ["--apply"]) == 2
+    assert contents(root) == before
+
+
+def test_changed_identity_and_symlink_file_refused(inputs, tmp_path):
+    root, repos = inputs
+    assert main(args(inputs) + ["--apply"]) == 0
+    before = contents(root)
+    repos.write_text("repos:\n  guides:\n    github: Research/OtherGuides\n")
+    assert main(args(inputs) + ["--apply"]) == 2
+    assert contents(root) == before
+    repos.write_text("repos:\n  guides:\n    github: Research/IndependentGuides\n")
+    (root / "CHECKIN.md").unlink()
+    target = tmp_path / "external"
+    target.write_text("untouched")
+    (root / "CHECKIN.md").symlink_to(target)
+    assert main(args(inputs) + ["--apply"]) == 2
+    assert target.read_text() == "untouched"
+
+
+def test_change_between_preview_and_apply_is_not_overwritten(inputs):
+    root, _ = inputs
+    prepared = prepare(inputs)
+    root.mkdir()
+    (root / "research.txt").write_text("new data")
+    with pytest.raises(ValueError, match="changed before apply"):
+        setup.apply(prepared)
+    assert list(root.iterdir()) == [root / "research.txt"]
+
+
+def test_atomic_create_does_not_replace_existing_file(tmp_path):
+    path = tmp_path / "file"
+    path.write_bytes(b"existing")
+    with pytest.raises(FileExistsError):
+        setup.create_file(path, b"replacement")
+    assert path.read_bytes() == b"existing"
+    assert list(tmp_path.iterdir()) == [path]
+
+
+def test_saved_receipt_and_records_survive_failed_owner_validation(inputs):
+    root, _ = inputs
+    assert main(args(inputs) + ["--apply"]) == 0
+    snapshot = {"schema_version": 1, "attempted_at": "2026-10-11T00:00:00Z",
+                "refreshed_at": "2026-10-11T00:00:00Z", "assistants": []}
+    key = digest(snapshot)
+    (root / "receipts").mkdir()
+    (root / "receipts" / (key + ".json")).write_text(json.dumps(snapshot))
+    (root / "latest.json").write_text(json.dumps({"receipt": key}))
+    before = contents(root)
+    assert main(args(inputs) + ["--apply"]) == 0
+    assert contents(root) == before
+    (root / "records").mkdir()
+    (root / "records" / "invalid.json").write_text('{"existing": "do not delete"}')
+    before = contents(root)
+    assert main(args(inputs) + ["--apply"]) == 2
+    assert contents(root) == before