From 467fa1a84156f3c10009d0d38748e8539b1dea29 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 14:47:40 -0300 Subject: [PATCH 01/51] feat(T-1703): strip generated envelopes before transcript distillation --- memorymaster/core/generated_artifacts.py | 67 +++++++++++++++++++++ memorymaster/dreaming/self_observation.py | 17 +++--- memorymaster/profile/renderer.py | 3 +- memorymaster/surfaces/session_end_ingest.py | 29 ++++----- tests/test_dreaming_self_observation.py | 9 +++ tests/test_generated_artifacts.py | 21 +++++++ tests/test_session_end_ingest.py | 21 +++++++ 7 files changed, 140 insertions(+), 27 deletions(-) create mode 100644 memorymaster/core/generated_artifacts.py create mode 100644 tests/test_generated_artifacts.py create mode 100644 tests/test_session_end_ingest.py diff --git a/memorymaster/core/generated_artifacts.py b/memorymaster/core/generated_artifacts.py new file mode 100644 index 00000000..d6d19bfa --- /dev/null +++ b/memorymaster/core/generated_artifacts.py @@ -0,0 +1,67 @@ +"""Recognize generated envelopes without turning separated source spans into evidence.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +GENERATED_OPENER = "" +GENERATED_CLOSER = "" +GENERATED_MARKER = GENERATED_OPENER +_MARKER = re.compile( + r"", re.I, +) + + +@dataclass(frozen=True) +class SourceSpan: + start: int + end: int + text: str + + +@dataclass(frozen=True) +class GeneratedText: + text: str + spans: tuple[SourceSpan, ...] + excluded_spans: tuple[tuple[int, int], ...] = () + + +def contains_generated_marker(text: str) -> bool: + return bool(_MARKER.search(text)) + + +def strip_generated_spans(text: str) -> GeneratedText: + """Keep original character offsets; unclosed/legacy envelopes own their suffix. + + ``text`` is a display convenience. Evidence consumers use ``spans`` rather + than asserting that text across a removed range was one authored statement. + Stray closers are excluded too, so they cannot survive as intake markers. + """ + removed: list[tuple[int, int]] = [] + depth = 0 + start = 0 + for marker in _MARKER.finditer(text): + if marker.group("close"): + if depth: + depth -= 1 + if depth == 0: + removed.append((start, marker.end())) + else: + removed.append(marker.span()) + else: + if depth == 0: + start = marker.start() + depth += 1 + if depth: + removed.append((start, len(text))) + spans: list[SourceSpan] = [] + position = 0 + for start, end in removed: + if position < start: + spans.append(SourceSpan(position, start, text[position:start])) + position = end + if position < len(text): + spans.append(SourceSpan(position, len(text), text[position:])) + spans = [span for span in spans if span.text.strip("\ufeff \t\r\n")] + return GeneratedText("\n".join(span.text for span in spans), tuple(spans), tuple(removed)) diff --git a/memorymaster/dreaming/self_observation.py b/memorymaster/dreaming/self_observation.py index 5cb44127..8281d835 100644 --- a/memorymaster/dreaming/self_observation.py +++ b/memorymaster/dreaming/self_observation.py @@ -6,12 +6,7 @@ from dataclasses import dataclass from typing import Any, MutableMapping - -@dataclass(frozen=True) -class SourceSpan: - start: int - end: int - text: str +from memorymaster.core.generated_artifacts import SourceSpan, strip_generated_spans @dataclass(frozen=True) @@ -73,17 +68,18 @@ def strip_self_observation( return SanitizedTurn("", exclusion_reasons=("structural_context",), contaminated=True) if memory_tool: return SanitizedTurn("", exclusion_reasons=("memory_tool",), contaminated=True) - removed: list[tuple[int, int]] = [] + generated = strip_generated_spans(text) + removed: list[tuple[int, int]] = list(generated.excluded_spans) for match in _WRAPPER.finditer(text): # Persisted outputs are never followed; even an unmarked pointer may be # recalled content whose provenance was erased by the export. removed.append(match.span()) spans: list[SourceSpan] = [] position = 0 - for start, end in removed: + for start, end in sorted(removed): if position < start: spans.append(SourceSpan(position, start, text[position:start])) - position = end + position = max(position, end) if position < len(text): spans.append(SourceSpan(position, len(text), text[position:])) remaining = "\n".join(span.text for span in spans) @@ -94,8 +90,9 @@ def strip_self_observation( state["tool_contaminated"] = True return SanitizedTurn("", exclusion_reasons=("generated_memory",), contaminated=True) authored = remaining.strip() - if removed and not authored: + if removed and not authored.strip("\ufeff"): state["tool_contaminated"] = True + return SanitizedTurn("", exclusion_reasons=("generated_memory",), contaminated=True) if role == "user" and authored: state["tool_contaminated"] = False state["memory_tool_ids"] = [] diff --git a/memorymaster/profile/renderer.py b/memorymaster/profile/renderer.py index 578709ca..0cfdfa39 100644 --- a/memorymaster/profile/renderer.py +++ b/memorymaster/profile/renderer.py @@ -2,12 +2,13 @@ from __future__ import annotations +from memorymaster.core.generated_artifacts import GENERATED_OPENER from memorymaster.profile.models import ProfileFact, RenderedProfile from memorymaster.recall.context_optimizer import estimate_tokens _HEADER = ( - "\n" + GENERATED_OPENER + "\n" "# Compiled User Profile\n\n" "Descriptive, evidence-bound operator facts. This is context, not an instruction file." ) diff --git a/memorymaster/surfaces/session_end_ingest.py b/memorymaster/surfaces/session_end_ingest.py index 7e351c26..6767c77a 100644 --- a/memorymaster/surfaces/session_end_ingest.py +++ b/memorymaster/surfaces/session_end_ingest.py @@ -30,33 +30,30 @@ def _sensitive(claim: dict[str, Any]) -> bool: def _assistant_text(transcript_path: str) -> str: + from memorymaster.dreaming.capture import _conversation_record + path = Path(transcript_path) if not path.is_file(): return "" messages: list[str] = [] lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - for line in reversed(lines[-200:]): + state: dict[str, Any] = {} + eligible: list[tuple[int, str]] = [] + for index, line in enumerate(lines): try: entry = json.loads(line) except json.JSONDecodeError: continue if not isinstance(entry, dict): continue - record = entry - for key in ("message", "payload"): - if isinstance(entry.get(key), dict): - record = entry[key] - break - if str(record.get("role") or "") not in {"assistant", "model"}: - continue - content = record.get("content", "") - if isinstance(content, list): - content = " ".join( - str(part.get("text", "")) - for part in content - if isinstance(part, dict) and part.get("type") in (None, "text") - ) - if isinstance(content, str) and len(content.strip()) > 30: + provider = "codex" if entry.get("type") == "response_item" else "claude" + record = _conversation_record(entry, provider, state=state) + if record is not None and record[0] == "assistant": + eligible.append((index, record[1])) + for index, content in reversed(eligible): + if index < len(lines) - 200: + break + if len(content.strip()) > 30: messages.append(content[:500]) if sum(map(len, messages)) > 3000: break diff --git a/tests/test_dreaming_self_observation.py b/tests/test_dreaming_self_observation.py index a1244276..fed9b754 100644 --- a/tests/test_dreaming_self_observation.py +++ b/tests/test_dreaming_self_observation.py @@ -9,6 +9,15 @@ from memorymaster.dreaming.self_observation import strip_self_observation +def test_balanced_generated_citation_does_not_poison_authored_spans(): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + text = f"Choose WAL.\n{GENERATED_OPENER}\nmm-12345678 old memory\n{GENERATED_CLOSER}\nUse transactions." + clean = strip_self_observation(text, role="assistant") + assert "Choose WAL." in clean.text and "Use transactions." in clean.text + assert "old memory" not in clean.text + assert all(text[span.start:span.end] == span.text for span in clean.spans) + + @pytest.mark.parametrize("text", [ "[MemoryMaster recall]\nPrior claim", "# Memory Context\nPrior claim", diff --git a/tests/test_generated_artifacts.py b/tests/test_generated_artifacts.py new file mode 100644 index 00000000..84fa07e7 --- /dev/null +++ b/tests/test_generated_artifacts.py @@ -0,0 +1,21 @@ +import pytest + +from memorymaster.core.generated_artifacts import ( + GENERATED_CLOSER, GENERATED_OPENER, strip_generated_spans, +) + + +@pytest.mark.parametrize("opener", [GENERATED_OPENER, ""]) +def test_mixed_generated_spans_keep_original_offsets(opener): + raw = f"Authored prefix.\n{opener}\nmm-12345678 recalled\n{GENERATED_CLOSER}\nAuthored suffix." + clean = strip_generated_spans(raw) + assert [span.text.strip() for span in clean.spans] == ["Authored prefix.", "Authored suffix."] + assert all(raw[span.start:span.end] == span.text for span in clean.spans) + assert "recalled" not in clean.text + + +def test_nested_repeated_and_unclosed_markers_quarantine_only_generated_ranges(): + raw = f"prefix{GENERATED_OPENER}outer{GENERATED_OPENER}inner{GENERATED_CLOSER}outer{GENERATED_CLOSER}suffix{GENERATED_OPENER}unclosed" + clean = strip_generated_spans(raw) + assert [span.text for span in clean.spans] == ["prefix", "suffix"] + assert strip_generated_spans("\ufeff" + GENERATED_OPENER + "generated").text.strip("\ufeff \n") == "" diff --git a/tests/test_session_end_ingest.py b/tests/test_session_end_ingest.py new file mode 100644 index 00000000..4d1e0975 --- /dev/null +++ b/tests/test_session_end_ingest.py @@ -0,0 +1,21 @@ +import json + +from memorymaster.surfaces.session_end_ingest import _assistant_text + + +def test_generated_turn_filtered_before_distillation(tmp_path): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + raw = f"{GENERATED_OPENER}\n" + "excluded fact " * 100 + f"\n{GENERATED_CLOSER}\n" + "A newly authored durable decision about transactions. " * 3 + path = tmp_path / "transcript.jsonl" + path.write_text(json.dumps({"role": "assistant", "content": raw}), encoding="utf-8") + result = _assistant_text(str(path)) + assert "excluded fact" not in result + assert "newly authored" in result + + +def test_chronological_tool_state_precedes_last_two_hundred_slice(tmp_path): + rows = [{"role": "assistant", "content": [{"type": "tool_use", "id": "recall", "name": "mcp__memorymaster__query_memory"}]}] + rows += [{"role": "assistant", "content": "Unmarked recalled echo should never be distilled into a new claim."}] * 205 + path = tmp_path / "transcript.jsonl" + path.write_text("\n".join(map(json.dumps, rows)), encoding="utf-8") + assert _assistant_text(str(path)) == "" From b71b415bb13cad4d44748baa27878694e366fdbd Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 14:49:59 -0300 Subject: [PATCH 02/51] feat(T-1703): record bounded session start baselines and supported deltas --- memorymaster/profile/project_sessions.py | 276 +++++++++++++++++++++++ tests/test_project_sessions.py | 120 ++++++++++ 2 files changed, 396 insertions(+) create mode 100644 memorymaster/profile/project_sessions.py create mode 100644 tests/test_project_sessions.py diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py new file mode 100644 index 00000000..bf65785e --- /dev/null +++ b/memorymaster/profile/project_sessions.py @@ -0,0 +1,276 @@ +"""Bounded derived continuity receipts; never acknowledge changes at session end.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Callable + +SCHEMA = "memorymaster.project-sessions.v1" +MAX_STATE_BYTES = 8 * 1024 * 1024 +MAX_BASELINE_BYTES = 128 * 1024 +ACTIVE_TTL = timedelta(hours=12) + + +def _encoded(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_encoded(value)).hexdigest() + + +def _time(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("timezone required") + return parsed.astimezone(timezone.utc) + + +def session_state_path(config) -> Path: + slug = config.scope.removeprefix("project:") + if not all(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part) for part in (slug, config.context_key)): + raise ValueError("invalid project identity") + return Path(config.state_root) / config.context_key / slug / "sessions.json" + + +def _empty(config) -> dict: + return {"schema": SCHEMA, "activation_epoch": config.epoch, "scope": config.scope, + "context_key": config.context_key, "seq": 0, "latest_end": None, + "active": {}, "completed": [], "baselines": {}} + + +def read_session_state(config) -> dict: + """Read one atomic state file without initializing directories or source stores.""" + if not config.enabled: + return _empty(config) + path = session_state_path(config) + if not path.exists(): + return _empty(config) + with path.open("rb") as handle: + raw = handle.read(MAX_STATE_BYTES + 1) + if len(raw) > MAX_STATE_BYTES: + raise ValueError("state_capacity") + state = json.loads(raw) + if (state.get("schema") != SCHEMA or state.get("scope") != config.scope + or state.get("context_key") != config.context_key): + raise ValueError("state_identity") + if state.get("activation_epoch") != config.epoch: + return _empty(config) + if (len(state["active"]) > 16 or len(state["completed"]) > 4 + or len(state["baselines"]) > 36): + raise ValueError("state_capacity") + for key, baseline in state["baselines"].items(): + if len(_encoded(baseline)) > MAX_BASELINE_BYTES or _digest(baseline) != key: + raise ValueError("baseline_integrity") + for record in [*state["active"].values(), *state["completed"]]: + if record["baseline"] not in state["baselines"]: + raise ValueError("baseline_unavailable") + latest = state["latest_end"] + if latest is not None and latest not in state["completed"]: + raise ValueError("end_pointer_integrity") + return state + + +@dataclass(frozen=True) +class SessionDelta: + text: str + status: str + supports: list[dict] = field(default_factory=list) + omitted: int = 0 + + +def _short(text: str, size: int) -> str: + raw = " ".join(text.split()).encode("utf-8") + return raw.decode("utf-8") if len(raw) <= size else raw[:size - 3].decode("utf-8", "ignore") + "..." + + +def build_session_delta(baseline: dict | None, projection_coverage: dict | None, + current: dict, *, diagnostic: str = "") -> SessionDelta: + """Union start and projection differences, with current authorized supports only.""" + coverage = current.get("coverage", {}) + items = current.get("items", {}) + comparisons = [value for value in (baseline, projection_coverage) if value is not None] + changed = {key for value in comparisons for key in coverage if value.get(key) != coverage[key]} + removed = {key for value in comparisons for key in value if key not in coverage} + # A currently authorized closed card may be carried in items outside open-card coverage. + changed.update(key for key in removed if key in items) + withdrawn = len(removed - changed) + complete = bool(current.get("complete")) and projection_coverage is not None + status = "partial" if not complete else "first_recorded_session" if baseline is None else "ok" + heading = "Prior-session changes and pending projection changes; current-state comparison." + if not comparisons: + status = "partial" + label = "first recorded session" if status == "first_recorded_session" else status + if diagnostic: + label += "; " + diagnostic + ordered = sorted(changed) + chosen: list[tuple[str, dict]] = [] + omitted = 0 + for key in ordered: + item = items.get(key) + if not item or not item.get("support"): + omitted += 1 + continue + support = item["support"] + reference = f"#{support['id']}" if support.get("kind") == "claim" else str(support["id"]) + chosen.append((f"- {_short(item.get('text', ''), 110)} [{reference}]", support)) + # Always reserve explicit omission/withdrawal counts before selecting lines. + def render(): + lines = [heading, label, *(line for line, _ in chosen)] + if not changed and not withdrawn and complete and baseline is not None: + lines.append("No changes.") + lines.append(f"withdrawn={withdrawn}; omitted={omitted}") + return "\n".join(lines) + while len(render().encode("utf-8")) > 500 and chosen: + chosen.pop() + omitted += 1 + text = render() + if len(text.encode("utf-8")) > 500: + raise ValueError("delta_budget") + supports = [] + for line, support in chosen: + start = text.index(line) + supports.append({"support": support, "start": start, "end": start + len(line), + "display_sha256": hashlib.sha256(line.encode()).hexdigest()}) + return SessionDelta(text, status, supports, omitted) + + +def _compact(state: dict) -> None: + state["completed"] = sorted(state["completed"], key=lambda r: (r["end_at"], r["seq"]))[-4:] + used = set() + for record in [*state["active"].values(), *state["completed"]]: + used.add(record["baseline"]) + if record.get("prior_baseline"): + used.add(record["prior_baseline"]) + state["baselines"] = {key: val for key, val in state["baselines"].items() if key in used} + if len(state["baselines"]) > 36 or len(_encoded(state)) > MAX_STATE_BYTES: + raise ValueError("state_capacity") + + +def _select_prior(state: dict, now: datetime) -> tuple[dict | None, str]: + latest = state["latest_end"] + stale = [record for record in state["active"].values() + if now - _time(record["start_at"]) >= ACTIVE_TTL] + fallback = max(stale, key=lambda record: record["start_at"], default=None) + if fallback and (latest is None or fallback["start_at"] > latest["start_at"]): + return fallback, "prior_session_unclosed" + return latest, "" + + +def record_project_session_start(config, session_id: str, *, projection: dict | None = None, + snapshot_loader: Callable[[], dict] | None = None, + now: datetime | None = None) -> dict: + if not config.enabled: + return {"status": "inactive"} + if not session_id or not config.epoch: + return {"status": "baseline_unavailable"} + from memorymaster.profile.project_projection import atomic_json, project_lock + + now = now or datetime.now(timezone.utc) + sid = hashlib.sha256(session_id.encode()).hexdigest() + path = session_state_path(config) + lock = path.with_suffix(".lock") + for _ in range(2): + try: + with project_lock(lock, timeout=0.05) as acquired: + if not acquired: + return {"status": "continuity_pending"} + state = read_session_state(config) + seq = state["seq"] + duplicate = state["active"].get(sid) + if any(record["session"] == sid for record in state["completed"]): + return {"status": "already_completed"} + prior, diagnostic = _select_prior(state, now) + if duplicate: + prior_id = duplicate.get("prior_baseline") + diagnostic = duplicate.get("diagnostic", "") + else: + prior_id = prior["baseline"] if prior else None + baseline = copy.deepcopy(state["baselines"].get(prior_id)) + if snapshot_loader is None: + if projection is None: + return {"status": "baseline_unavailable"} + from memorymaster.profile.project_repository import snapshot_since + + current = snapshot_since(config, projection["source_coverage"], projection["read_at"]) + else: + current = snapshot_loader() + if not current.get("complete"): + return {"status": "baseline_unavailable", + "delta": build_session_delta(baseline, None, current)} + coverage = current["coverage"] + if len(_encoded(coverage)) > MAX_BASELINE_BYTES: + return {"status": "baseline_capacity"} + delta = build_session_delta(baseline, projection.get("source_coverage") if projection else None, + current, diagnostic=diagnostic) + if duplicate: + return {"status": "duplicate", "delta": delta} + with project_lock(lock, timeout=0.05) as acquired: + if not acquired: + return {"status": "continuity_pending"} + state = read_session_state(config) + if state["seq"] != seq: + continue + # Stale starts never fabricate ends. Reclaim their slots after retaining the selected baseline. + state["active"] = {key: val for key, val in state["active"].items() + if now - _time(val["start_at"]) < ACTIVE_TTL} + if len(state["active"]) >= 16: + return {"status": "active_capacity"} + baseline_id = _digest(coverage) + state["baselines"][baseline_id] = coverage + state["seq"] += 1 + state["active"][sid] = {"session": sid, "start_at": now.isoformat(), + "read_at": current["read_at"], "baseline": baseline_id, + "prior_baseline": prior_id, "prior_receipt": prior.get("session") if prior else None, + "diagnostic": diagnostic, "seq": state["seq"]} + _compact(state) + atomic_json(path, state) + return {"status": "ok", "delta": delta} + except (OSError, ValueError, KeyError, TypeError): + return {"status": "continuity_pending"} + return {"status": "continuity_pending"} + + +def record_project_session_end(config, session_id: str, *, end_at: str | None = None, + now: datetime | None = None) -> bool: + if not config.enabled or not session_id or not session_state_path(config).is_file(): + return False + from memorymaster.profile.project_projection import atomic_json, project_lock + + now = now or datetime.now(timezone.utc) + try: + end = _time(end_at) if end_at else now + if end > now + timedelta(minutes=1): + return False + path = session_state_path(config) + sid = hashlib.sha256(session_id.encode()).hexdigest() + with project_lock(path.with_suffix(".lock"), timeout=0.05) as acquired: + if not acquired: + return False + state = read_session_state(config) + record = state["active"].get(sid) + if record is None or end < _time(record["start_at"]): + return False + state["seq"] += 1 + record = {**state["active"].pop(sid), "end_at": end.isoformat(), "seq": state["seq"]} + # Completed baselines do not recursively retain their own prior baseline. + record.pop("prior_baseline", None) + state["completed"].append(record) + latest = state["latest_end"] + if latest is None or (end, record["seq"]) > (_time(latest["end_at"]), latest["seq"]): + state["latest_end"] = record + _compact(state) + atomic_json(path, state) + return True + except (OSError, ValueError, KeyError, TypeError): + return False + + +def last_project_session_end(config) -> dict | None: + return read_session_state(config)["latest_end"] if config.enabled else None diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py new file mode 100644 index 00000000..cf849a48 --- /dev/null +++ b/tests/test_project_sessions.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from memorymaster.profile import project_sessions as sessions + + +NOW = datetime(2026, 10, 22, 9, tzinfo=timezone.utc) + + +def config(tmp_path): + return SimpleNamespace(enabled=True, state_root=tmp_path, context_key="fixture", + scope="project:sample", epoch="pilot") + + +def snapshot(**items): + return {"coverage": {key: item["support"] for key, item in items.items()}, + "items": items, "complete": True, "read_at": NOW.isoformat()} + + +def claim(text, version=1): + return {"key": "claim:1", "text": text, "section": "Decisions", + "support": {"kind": "claim", "id": 1, "version": version, "hash": text}} + + +def test_session_gap_no_observable_change_lost(tmp_path): + cfg = config(tmp_path) + original = snapshot(**{"claim:1": claim("Original decision")}) + projection = {"source_coverage": original["coverage"], "read_at": NOW.isoformat()} + first = sessions.record_project_session_start( + cfg, "S", projection=projection, snapshot_loader=lambda: original, now=NOW) + assert first["delta"].status == "first_recorded_session" + changed = snapshot(**{"claim:1": claim("Changed decision", 2)}) + assert sessions.record_project_session_end(cfg, "S", now=NOW + timedelta(minutes=20)) + result = sessions.record_project_session_start( + cfg, "T", projection=projection, snapshot_loader=lambda: changed, + now=NOW + timedelta(minutes=21)) + assert "Changed decision" in result["delta"].text + assert result["delta"].supports + assert "no changes" not in result["delta"].text.lower() + # A beat rendering that change does not suppress prior-session work. + new_projection = {"source_coverage": changed["coverage"], "read_at": changed["read_at"]} + delta = sessions.build_session_delta(original["coverage"], new_projection["source_coverage"], changed) + assert "Changed decision" in delta.text + + +def test_session_end_state_is_atomic_bounded_idempotent(tmp_path): + cfg = config(tmp_path) + current = snapshot(**{"claim:1": claim("decision")}) + projection = {"source_coverage": current["coverage"], "read_at": NOW.isoformat()} + for index in range(100): + now = NOW + timedelta(minutes=index) + sessions.record_project_session_start(cfg, str(index), projection=projection, + snapshot_loader=lambda: current, now=now) + assert sessions.record_project_session_end(cfg, str(index), now=now + timedelta(seconds=1)) + path = sessions.session_state_path(cfg) + state = json.loads(path.read_text(encoding="utf-8")) + assert len(state["completed"]) == 4 + assert len(state["baselines"]) == 1 + assert not state["active"] + previous = path.read_bytes() + assert not sessions.record_project_session_end(cfg, "0", now=NOW + timedelta(days=2)) + assert path.read_bytes() == previous + assert "99" not in state["latest_end"]["session"] + + +def test_duplicate_start_does_not_reset_baseline(tmp_path): + cfg = config(tmp_path) + original = snapshot(**{"claim:1": claim("original")}) + projection = {"source_coverage": original["coverage"], "read_at": NOW.isoformat()} + sessions.record_project_session_start(cfg, "S", projection=projection, + snapshot_loader=lambda: original, now=NOW) + before = sessions.session_state_path(cfg).read_bytes() + sessions.record_project_session_start(cfg, "S", projection=projection, + snapshot_loader=lambda: snapshot(**{"claim:1": claim("new", 2)}), + now=NOW + timedelta(minutes=1)) + assert sessions.session_state_path(cfg).read_bytes() == before + + +def test_unclosed_session_ttl_fallback_preserves_closures(tmp_path): + cfg = config(tmp_path) + card = {"key": "card:T-X", "text": "A visible card", "section": "Open cards", + "support": {"kind": "card", "id": "T-X", "hash": "open"}} + original = snapshot(**{"card:T-X": card}) + projection = {"source_coverage": original["coverage"], "read_at": NOW.isoformat()} + sessions.record_project_session_start(cfg, "killed", projection=projection, + snapshot_loader=lambda: original, now=NOW) + closed = {**card, "text": "Completed visible card", "closed": True, + "support": {**card["support"], "hash": "closed"}} + current = snapshot(**{"card:T-X": closed}) + result = sessions.record_project_session_start(cfg, "next", projection={ + "source_coverage": current["coverage"], "read_at": current["read_at"]}, + snapshot_loader=lambda: current, now=NOW + timedelta(hours=13)) + assert "prior_session_unclosed" in result["delta"].text + assert "Completed visible card" in result["delta"].text + assert sessions.last_project_session_end(cfg) is None + + +def test_delta_cap_withdrawal_and_no_false_empty(): + current = snapshot(**{f"claim:{i}": {**claim("é" * 300), "key": f"claim:{i}"} + for i in range(30)}) + delta = sessions.build_session_delta({"claim:private": {"hash": "old-private"}}, {}, current) + assert len(delta.text.encode("utf-8")) <= 500 + assert "omitted=" in delta.text + assert "old-private" not in delta.text + assert "withdrawn=" in delta.text + assert "private" not in delta.text + partial = sessions.build_session_delta({}, None, snapshot()) + assert partial.status == "partial" + assert "no changes" not in partial.text.lower() + + +def test_disabled_state_never_reads_or_writes(tmp_path): + cfg = config(tmp_path) + cfg.enabled = False + assert sessions.record_project_session_start(cfg, "S")["status"] == "inactive" + assert not sessions.record_project_session_end(cfg, "S") + assert not list(tmp_path.iterdir()) From 00e57069a8b9219e2b9b7d1231f5d1ac7ad38a83 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 14:51:56 -0300 Subject: [PATCH 03/51] feat(T-1703): gate project hook delivery and independent beat refresh --- .../hooks/memorymaster-session-end.py | 12 +- .../hooks/memorymaster-session-start.py | 14 + memorymaster/profile/project_runtime.py | 243 ++++++++++++++++++ memorymaster/surfaces/scheduled_task.py | 20 ++ tests/test_compiled_profile_surfaces.py | 15 ++ tests/test_project_runtime.py | 71 +++++ tests/test_scheduled_task_runtime.py | 15 ++ tests/test_session_end_capture_hook.py | 17 ++ 8 files changed, 405 insertions(+), 2 deletions(-) create mode 100644 memorymaster/profile/project_runtime.py create mode 100644 tests/test_project_runtime.py diff --git a/memorymaster/config_templates/hooks/memorymaster-session-end.py b/memorymaster/config_templates/hooks/memorymaster-session-end.py index 049431f9..451f4838 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-end.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-end.py @@ -19,12 +19,20 @@ def _enabled(name, default=True): def main(): - if not _enabled("MEMORYMASTER_SESSION_END_CAPTURE", True): - return try: data = json.loads(sys.stdin.read() or "{}") + if not isinstance(data, dict): + data = {} except Exception: data = {} + try: + from memorymaster.profile.project_runtime import record_session_end + + record_session_end(data) + except Exception: + pass # Lifecycle failure must not suppress independently authorized capture. + if not _enabled("MEMORYMASTER_SESSION_END_CAPTURE", True): + return transcript_path = data.get("transcript_path", "") session_id = str(data.get("session_id") or "unknown") cwd = str(data.get("cwd") or PROJECT_ROOT) diff --git a/memorymaster/config_templates/hooks/memorymaster-session-start.py b/memorymaster/config_templates/hooks/memorymaster-session-start.py index 71d7e1d3..3cba2733 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-start.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-start.py @@ -365,6 +365,20 @@ def main(): except Exception: pass + try: + from memorymaster.profile.project_runtime import session_start_context + + context = session_start_context(data, profile_loader=_load_compiled_profile) + if context is not None: + sys.stdout.write(json.dumps({"hookSpecificOutput": { + "hookEventName": "SessionStart", "additionalContext": context}})) + sys.stdout.flush() + sys.exit(0) + except Exception: + # An enabled project consumer must never fall through to less restrictive legacy SQL. + if os.environ.get("MEMORYMASTER_PROJECT_PROJECTION_ENABLED", "").lower() in ("1", "true", "yes", "on"): + sys.exit(0) + if not DB_PATH.exists(): _log("skip", reason="db-missing") sys.exit(0) diff --git a/memorymaster/profile/project_runtime.py b/memorymaster/profile/project_runtime.py new file mode 100644 index 00000000..2425546e --- /dev/null +++ b/memorymaster/profile/project_runtime.py @@ -0,0 +1,243 @@ +"""Explicit opt-in configuration and bounded project projection consumers.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import time +from datetime import datetime, timezone +from pathlib import Path + +from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + +PREFIX = "MEMORYMASTER_PROJECT_PROJECTION_" +_refresh_cache: dict[tuple, tuple] = {} +_connections: dict[str, sqlite3.Connection] = {} +_rotation = 0 + + +def active(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def load_project_configs() -> list: + """Parse an explicit pilot allowlist. No path discovery or store initialization.""" + if not active(PREFIX + "ENABLED"): + return [] + from memorymaster.profile.project_projection import ProjectProjectionConfig + + try: + raw = os.environ.get(PREFIX + "PROJECTS", "[]") + if len(raw.encode()) > 32768: + return [] + mappings = json.loads(raw) + epoch = os.environ.get(PREFIX + "EPOCH", "").strip() + output = os.environ.get(PREFIX + "DIR", "") + state = os.environ.get("MEMORYMASTER_PROJECT_SESSION_STATE_DIR", "") + ledger = os.environ.get("MEMORYMASTER_PROJECT_LEDGER_TASKS_DIR", "") + if (not isinstance(mappings, list) or not 1 <= len(mappings) <= 3 or not epoch + or len(epoch) > 128 or not output or not state + or not Path(output).is_absolute() or not Path(state).is_absolute() + or (ledger and not Path(ledger).is_absolute())): + return [] + configs = [] + seen = set() + for row in mappings: + # Null tenant is a deliberate legacy boundary, not a default across tenants. + if "tenant_id" not in row or not Path(row["db_path"]).is_absolute(): + return [] + if not Path(row["project_root"]).is_absolute(): + return [] + identity = (row["context_key"], row["scope"]) + if identity in seen: + return [] + seen.add(identity) + configs.append(ProjectProjectionConfig( + db_path=Path(row["db_path"]), scope=row["scope"], tenant_id=row["tenant_id"], + context_key=row["context_key"], epoch=epoch, project_root=Path(row["project_root"]), + output_root=Path(output), state_root=Path(state), + ledger_root=Path(ledger) if ledger else None, ledger_repo=row.get("ledger_repo"), + ledger_filtered=row.get("ledger_filtered") is True, + aliases=tuple(row.get("aliases", ())), enabled=True)) + return configs + except (ValueError, KeyError, TypeError): + return [] + + +def resolve_projection_scope(config, cwd) -> str | None: + """Both scope resolvers and the actual repository owner must agree.""" + from memorymaster.core.scope_utils import ( + _repository_owner, + resolve_transcript_workspace_scope, + scope_from_cwd_or_user, + ) + + try: + path = Path(cwd).resolve(strict=True) + if not path.is_dir(): + return None + strict = resolve_transcript_workspace_scope(str(path)) + generic = scope_from_cwd_or_user(str(path)) + if strict != generic or strict not in (config.scope, *config.aliases): + return None + for parent in (path, *list(path.parents)[:12]): + marker = parent / ".git" + if marker.exists(): + owner = _repository_owner(parent) + if owner is None or owner.resolve() != config.project_root.resolve(strict=True): + return None + # Linked worktrees must have a valid backlink; generic resolution alone is insufficient. + if marker.is_file(): + if marker.stat().st_size > 4096: + return None + pointer = marker.read_text(encoding="utf-8").strip() + if not pointer.startswith("gitdir:"): + return None + gitdir = (parent / pointer.partition(":")[2].strip()).resolve() + backlink = gitdir / "gitdir" + if (not backlink.is_file() or backlink.stat().st_size > 4096 + or Path(backlink.read_text(encoding="utf-8").strip()).resolve() != marker.resolve()): + return None + return config.scope + return None + except (OSError, ValueError, RuntimeError): + return None + + +def config_for_cwd(cwd): + matches = [cfg for cfg in load_project_configs() if resolve_projection_scope(cfg, cwd)] + return matches[0] if len(matches) == 1 else None + + +def compose_session_context(payload: str, delta: str = "", profile: str = "") -> str: + """Embed the shared bytes intact, with one envelope and complete optional lines.""" + if len(payload.encode("utf-8")) > 2200 or len(delta.encode("utf-8")) > 500: + raise ValueError("consumer_budget") + if GENERATED_OPENER in payload or GENERATED_CLOSER in payload: + raise ValueError("nested_envelope") + parts = [payload] + if delta: + parts.append(delta) + profile_lines = [line for line in profile.splitlines() + if line.strip() and not line.lstrip().startswith("" +CLOSER = "" +SCHEMA = "memorymaster.project-projection.v1" +SECTIONS = ("Purpose", "Constraints", "Decisions", "Open cards", "Gotchas") +MAX_MANIFEST_BYTES = 256 * 1024 + + +@dataclass(frozen=True) +class ProjectProjectionConfig: + db_path: Path + scope: str + tenant_id: str | None + context_key: str + epoch: str + project_root: Path + output_root: Path + state_root: Path + ledger_root: Path | None = None + ledger_repo: str | None = None + ledger_filtered: bool = False + aliases: tuple[str, ...] = () + enabled: bool = False + budget_bytes: int = 2200 + + def __post_init__(self) -> None: + if str(self.db_path).startswith(("postgres:", "postgresql:")): + raise ValueError("SQLite required") + for name in ("context_key", "epoch"): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,95}", getattr(self, name)): + raise ValueError("invalid context identity") + if not re.fullmatch(r"project:[a-z0-9][a-z0-9_.-]{0,95}", self.scope): + raise ValueError("invalid project scope") + if ( + any(not re.fullmatch(r"project:[a-z0-9][a-z0-9_.-]{0,95}", alias) for alias in self.aliases) + or len(self.aliases) > 8 + ): + raise ValueError("invalid aliases") + if not 400 <= self.budget_bytes <= 2200: + raise ValueError("project cap must be 400..2200 UTF-8 bytes") + for name in ("db_path", "project_root", "output_root", "state_root", "ledger_root"): + value = getattr(self, name) + if value is not None: + object.__setattr__(self, name, Path(value).resolve()) + object.__setattr__(self, "aliases", tuple(self.aliases)) + + @property + def slug(self) -> str: + return self.scope.split(":", 1)[1] + + +def projection_paths(config: ProjectProjectionConfig) -> tuple[Path, Path]: + root = config.output_root / config.context_key + return root / f"project-{config.slug}.md", root / f"project-{config.slug}.json" + + +@contextmanager +def project_lock(path: Path, timeout: float = 0.05) -> Iterator[bool]: + """Nonblocking kernel lock; the OS releases ownership after process death.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + stream = path.open("a+b") + acquired = False + try: + if stream.tell() == 0: + stream.write(b"\0") + stream.flush() + deadline = time.monotonic() + max(0, timeout) + while True: + try: + stream.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + break + except OSError: + if time.monotonic() >= deadline: + break + time.sleep(0.005) + yield acquired + finally: + if acquired: + stream.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + stream.close() + + +def _atomic_write(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def atomic_json(path: Path, value: dict) -> None: + _atomic_write(path, json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()) + + +@dataclass(frozen=True) +class ProjectionItem: + key: str + section: str + text: str + support: dict + sort: list = field(default_factory=list) + + +@dataclass +class ProjectProjection: + payload: str + manifest: dict + items: list[dict] + status: str + + @property + def markdown(self) -> str: + return f"{OPENER}\n{self.payload}\n{CLOSER}\n" + + @property + def public_manifest(self) -> dict: + return { + key: value + for key, value in self.manifest.items() + if key not in ("source_coverage", "source_fingerprints", "tenant_id", "aliases", "context_key") + } + + +def _line(item: dict) -> str: + support = item["support"] + reference = f"claim:{support['id']}" if support["kind"] == "claim" else str(support["id"]) + return f"- {item['text']} [{reference}]" + + +def render_project_projection( + config: ProjectProjectionConfig, + items: list[dict], + *, + generation: str, + read_at: str, + coverage: dict | None = None, + diagnostics: list[str] | None = None, + omissions: dict | None = None, + fingerprints: dict | None = None, + source_complete: bool = True, + next_temporal_boundary: str | None = None, +) -> ProjectProjection: + selected = [dict(item) for item in items] + diagnostics = list(diagnostics or []) + omissions = dict(omissions or {}) + decisions = [item for item in selected if item["section"] == "Decisions"] + mandatory_decisions = {item["key"] for item in decisions[:3]} + if len(decisions) < 3: + diagnostics.append("m2_not_ready:decisions_incomplete") + # Purpose and pinned safety prose share a strict byte allocation. Shortening + # literal excerpts keeps references and support identities intact. + safety = [item for item in selected if item["section"] == "Purpose" or item.get("pinned")] + remaining = 400 + for item in safety: + cap = min(remaining, 400 // max(1, len(safety))) + if cap < 4: + selected.remove(item) + omissions["pinned_constraints"] = omissions.get("pinned_constraints", 0) + 1 + diagnostics.append("m2_not_ready:pinned_constraints_trimmed") + continue + text = item["text"] + while len(text.encode()) > cap: + text = text[:-1] + if text != item["text"]: + while text and len((text + "…").encode()) > cap: + text = text[:-1] + text += "…" + item["text"] = text + remaining -= len(item["text"].encode()) + diagnostics = list(dict.fromkeys(diagnostics)) + max_bytes = config.budget_bytes + status = "partial" if diagnostics or any(omissions.values()) else "ready" + + def compose() -> tuple[str, str]: + nonlocal status + status = "partial" if diagnostics or any(omissions.values()) else "ready" + identity = digest( + [ + generation, + [(item["key"], item["support"], item["text"]) for item in selected], + status, + omissions, + diagnostics, + ] + ) + lines = [f"Project {config.slug}", f"View {identity[:16]} | {status}"] + for section in SECTIONS: + lines.append(f"## {section}") + matches = [item for item in selected if item["section"] == section] + if section == "Purpose" and not matches: + lines.append("Purpose not recorded in confirmed memory.") + lines.extend(_line(item) for item in matches) + if diagnostics: + lines.append("Status: " + ", ".join(dict.fromkeys(diagnostics))) + if any(omissions.values()): + lines.append("Omitted: " + ", ".join(f"{key}={value}" for key, value in sorted(omissions.items()) if value)) + return "\n".join(lines), identity + + def size(payload: str) -> int: + return len(f"{OPENER}\n{payload}\n{CLOSER}\n".encode()) + + # Reserve the complete compact card set before optional prose; exact support + # records are generated only after the final trim. + compacted = False + while True: + payload, identity = compose() + if ( + size(payload) <= max_bytes + and len(selected) <= 64 + and sum(len(item["support"].get("citations", [])) for item in selected) <= 256 + ): + break + if not compacted: + for item in selected: + text = item["text"] + minimum = 48 if item["section"] == "Open cards" else 80 + if len(text) > minimum: + item["text"] = text[: minimum - 1] + "…" + compacted = True + continue + optional = ( + lambda item: item["section"] == "Gotchas", + lambda item: item["section"] == "Decisions" and item["key"] not in mandatory_decisions, + lambda item: item["section"] == "Constraints" and not item.get("pinned"), + lambda item: item["section"] == "Purpose", + lambda item: item["section"] == "Constraints", + lambda item: item["section"] == "Decisions", + lambda item: item["section"] == "Open cards", + ) + removable = next((item for predicate in optional for item in reversed(selected) if predicate(item)), None) + if removable is None: + raise ValueError("mandatory_projection_framing_does_not_fit") + selected.remove(removable) + key = removable["section"].lower().replace(" ", "_") + omissions[key] = omissions.get(key, 0) + 1 + if removable["section"] == "Open cards" and "m2_not_ready:open_cards_trimmed" not in diagnostics: + diagnostics.append("m2_not_ready:open_cards_trimmed") + if removable["section"] == "Decisions" and "m2_not_ready:decisions_trimmed" not in diagnostics: + if removable["key"] in mandatory_decisions: + diagnostics.append("m2_not_ready:decisions_trimmed") + if removable.get("pinned") and "m2_not_ready:pinned_constraints_trimmed" not in diagnostics: + diagnostics.append("m2_not_ready:pinned_constraints_trimmed") + records = [] + lines = payload.splitlines() + for item in selected: + rendered = _line(item) + index = lines.index(rendered) + start = len(("\n".join(lines[:index]) + "\n").encode()) if index else 0 + records.append( + { + "key": item["key"], + "section": item["section"], + "text": item["text"], + "line": index + 1, + "span": [start, start + len(rendered.encode())], + "text_hash": digest(item["text"]), + "supports": [item["support"]], + "pinned": item.get("pinned", False), + } + ) + markdown = f"{OPENER}\n{payload}\n{CLOSER}\n" + manifest = { + "schema": SCHEMA, + "algorithm_version": 1, + "scope": config.scope, + "aliases": list(config.aliases), + "tenant_id": config.tenant_id, + "context_key": config.context_key, + "activation_epoch": config.epoch, + "generation_id": generation, + "view_id": identity, + "read_at": read_at, + "checked_at": utc_now(), + "output_sha256": hashlib.sha256(markdown.encode()).hexdigest(), + "payload_sha256": hashlib.sha256(payload.encode()).hexdigest(), + "output_bytes": len(markdown.encode()), + "payload_bytes": len(payload.encode()), + "budget_bytes": max_bytes, + "items": records, + "source_coverage": coverage or {}, + "source_fingerprints": fingerprints or {}, + "omissions": omissions, + "diagnostics": list(dict.fromkeys(diagnostics)), + "status": status, + "complete": source_complete, + "source_complete": source_complete, + "next_temporal_boundary": next_temporal_boundary, + } + return ProjectProjection(payload, manifest, selected, status) + + +def build_project_projection(config: ProjectProjectionConfig) -> ProjectProjection: + if not config.enabled: + raise ValueError("projection inactive") + snapshot = load_project_snapshot(config) + ordered = [] + for section in SECTIONS: + matches = [item for item in snapshot["items"].values() if item["section"] == section and item.get("open", True)] + matches.sort(key=lambda item: item["sort"], reverse=section == "Decisions") + ordered.extend(matches[:1] if section == "Purpose" else matches[:10] if section == "Decisions" else matches) + diagnostics = list(snapshot["diagnostics"]) + if not snapshot["complete"] and not diagnostics: + diagnostics.append("source_incomplete") + if any(diagnostic.startswith("ledger_") for diagnostic in diagnostics): + diagnostics.append("m2_not_ready:ledger_incomplete") + generation = digest([config.scope, config.context_key, config.epoch, snapshot["read_at"], snapshot["coverage"]]) + return render_project_projection( + config, + ordered, + generation=generation, + read_at=snapshot["read_at"], + coverage=snapshot["coverage"], + diagnostics=diagnostics, + fingerprints=snapshot["fingerprints"], + source_complete=snapshot["complete"], + next_temporal_boundary=snapshot["next_temporal_boundary"], + ) + + +def publish_project_projection(config: ProjectProjectionConfig, projection: ProjectProjection) -> bool: + if not config.enabled: + return False + md, path = projection_paths(config) + with project_lock(path.with_suffix(".lock"), timeout=0) as acquired: + if not acquired: + return False + if len(json.dumps(projection.manifest).encode()) > MAX_MANIFEST_BYTES: + return False + _atomic_write(md, projection.markdown.encode()) + atomic_json(path, projection.manifest) + return True + + +def read_manifest(config: ProjectProjectionConfig) -> dict | None: + if not config.enabled: + return None + md, path = projection_paths(config) + try: + if path.stat().st_size > MAX_MANIFEST_BYTES or md.stat().st_size > config.budget_bytes: + return None + with path.open("rb") as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + with md.open("rb") as stream: + output = stream.read(config.budget_bytes + 1) + manifest = json.loads(raw) + if ( + manifest["schema"] != SCHEMA + or manifest["scope"] != config.scope + or manifest["tenant_id"] != config.tenant_id + or manifest["context_key"] != config.context_key + or manifest["activation_epoch"] != config.epoch + or manifest["aliases"] != list(config.aliases) + or manifest["budget_bytes"] != config.budget_bytes + or hashlib.sha256(output).hexdigest() != manifest["output_sha256"] + or len(output) != manifest["output_bytes"] + or len(output) > config.budget_bytes + ): + return None + text = output.decode() + if not text.startswith(OPENER + "\n") or not text.endswith("\n" + CLOSER + "\n"): + return None + payload = text[len(OPENER) + 1 : -len(CLOSER) - 2] + if ( + hashlib.sha256(payload.encode()).hexdigest() != manifest["payload_sha256"] + or len(payload.encode()) != manifest["payload_bytes"] + ): + return None + records = manifest["items"] + if ( + len(records) > 64 + or len(json.dumps(manifest["source_coverage"]).encode()) > 128 * 1024 + or len(manifest["source_coverage"]) > 768 + ): + return None + citations = 0 + keys = set() + for record in records: + if record["key"] in keys or len(record["supports"]) != 1 or record["section"] not in SECTIONS: + return None + keys.add(record["key"]) + support = record["supports"][0] + if support["kind"] == "claim": + if type(support["id"]) is not int or support["id"] <= 0: + return None + citations += len(support["citations"]) + elif support["kind"] != "card" or not isinstance(support["id"], str): + return None + if record["text_hash"] != digest(record["text"]): + return None + rendered = _line({"text": record["text"], "support": support}) + line_index = record["line"] - 1 + lines = payload.splitlines() + if line_index < 0 or line_index >= len(lines) or lines[line_index] != rendered: + return None + start = len(("\n".join(lines[:line_index]) + "\n").encode()) if line_index else 0 + if record["span"] != [start, start + len(rendered.encode())]: + return None + if citations > 256: + return None + expected_generation = digest( + [config.scope, config.context_key, config.epoch, manifest["read_at"], manifest["source_coverage"]] + ) + expected_view = digest( + [ + manifest["generation_id"], + [(record["key"], record["supports"][0], record["text"]) for record in records], + manifest["status"], + manifest["omissions"], + manifest["diagnostics"], + ] + ) + if ( + manifest["generation_id"] != expected_generation + or manifest["view_id"] != expected_view + or f"View {expected_view[:16]} | {manifest['status']}" not in payload.splitlines() + ): + return None + return manifest + except (OSError, ValueError, KeyError, TypeError, UnicodeError): + return None + + +def read_project_projection(config: ProjectProjectionConfig) -> ProjectProjection | None: + manifest = read_manifest(config) + if manifest is None: + return None + try: + checked = datetime.fromisoformat(manifest["checked_at"]) + if checked.tzinfo is None or (datetime.now(timezone.utc) - checked).total_seconds() > 3600: + return None + except (ValueError, TypeError, KeyError): + return None + ids = [record["supports"][0]["id"] for record in manifest["items"] if record["supports"][0]["kind"] == "claim"] + current = load_project_snapshot(config, claim_ids=ids, validation=True) + selected = [] + diagnostics = list(manifest["diagnostics"]) + omissions = dict(manifest["omissions"]) + for record in manifest["items"]: + item = current["items"].get(record["key"]) + if record["supports"][0]["kind"] == "card": + # Modified still-open cards are safe to render from the authorized + # ledger snapshot and retain their reserved place in the payload. + if item and item.get("open"): + if item["support"] == record["supports"][0]: + item = {**item, "text": record["text"]} + selected.append(item) + elif "m2_not_ready:cards_changed" not in diagnostics: + diagnostics.append("m2_not_ready:cards_changed") + elif item and item["support"] == record["supports"][0]: + # Keep an unrelated bullet byte-identical even when the original + # renderer shortened it to fit the shared allocation. + selected.append({**item, "text": record["text"]}) + else: + diagnostics.append("m2_not_ready:context_withdrawn") + diagnostics.extend(current["diagnostics"]) + if any(key.startswith("card:") and key not in manifest["source_coverage"] for key in current["coverage"]): + diagnostics.append("m2_not_ready:cards_changed") + if not current["complete"] and "source_incomplete" not in diagnostics: + diagnostics.append("source_incomplete") + result = render_project_projection( + config, + selected, + generation=manifest["generation_id"], + read_at=manifest["read_at"], + coverage=manifest["source_coverage"], + diagnostics=list(dict.fromkeys(diagnostics)), + omissions=omissions, + fingerprints=manifest["source_fingerprints"], + source_complete=current["complete"], + next_temporal_boundary=manifest.get("next_temporal_boundary"), + ) + result.manifest["checked_at"] = manifest["checked_at"] + return result diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py new file mode 100644 index 00000000..38f7eb84 --- /dev/null +++ b/memorymaster/profile/project_repository.py @@ -0,0 +1,430 @@ +"""Bounded read-only project evidence, never a writable profile repository.""" + +from __future__ import annotations + +from contextlib import closing +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import re +import sqlite3 +import time +from types import SimpleNamespace +from typing import Any + +from memorymaster.core.security import scan_persisted_value +from memorymaster.core.temporal_policy import claim_is_temporally_current +from memorymaster.profile.egress import prepare_claim_egress + +OPEN_STATES = frozenset({"queued", "ready", "running", "review", "blocked"}) +MAX_CLAIMS = 512 +MAX_CITATIONS = 2048 +MAX_FIELD_BYTES = 16384 +MAX_SOURCE_BYTES = 8 * 1024 * 1024 +GENERATED_SOURCES = frozenset( + {"project-projection", "compiled-profile", "cockpit", "operational-review", "beat-narrative", "graph-summary"} +) + + +def digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + ).hexdigest() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _safe(value: Any) -> bool: + text = json.dumps(value, ensure_ascii=False) + lowered = text.lower() + return ( + len(text.encode()) <= MAX_FIELD_BYTES + and not any(marker in lowered for marker in ("generated by memorymaster", "[redacted", "[erased_claim_text]")) + and not scan_persisted_value(value) + and not prepare_claim_egress(text).blocked + ) + + +def _display(text: Any, limit: int = 240) -> str: + # One inert line; never permit source headings or markers to create instructions. + text = prepare_claim_egress(str(text or "")).text + text = re.sub(r"[\x00-\x1f\x7f]", " ", text) + text = text.translate(str.maketrans({"<": "‹", ">": "›", "[": "（", "]": "）", "`": "′"})) + literal = " ".join(text.split()) + return literal if len(literal) <= limit else literal[: limit - 1] + "…" + + +def cheap_fingerprints(config: Any) -> dict[str, Any]: + """Filesystem change hints only: callers must also honor temporal boundaries.""" + if not config.enabled: + return {} + paths = [Path(config.db_path), Path(str(config.db_path) + "-wal")] + if config.ledger_root: + paths.extend(_ledger_paths(config)) + result: dict[str, Any] = {} + for index, path in enumerate(paths): + try: + stat = path.stat() + result[str(index)] = [stat.st_mtime_ns, stat.st_size] + except OSError: + result[str(index)] = None + return result + + +def _ledger_paths(config: Any) -> list[Path]: + root = Path(config.ledger_root) + if root.is_file(): + return [root] + # Enumerate only the explicit approved input directory, with a hard count cap. + paths = [] + for path in root.iterdir(): + if path.suffix == ".json" and path.is_file(): + paths.append(path) + if len(paths) > 1024: + raise ValueError("ledger_file_limit") + return sorted(paths) + + +def load_ledger(config: Any, *, max_bytes: int = MAX_SOURCE_BYTES) -> tuple[dict, bool, list[str]]: + if not config.ledger_root or not config.ledger_repo or not config.ledger_filtered: + return {}, False, ["ledger_unavailable"] + for attempt in range(2): + try: + paths = _ledger_paths(config) + if not paths: + raise ValueError("no_ledger_input") + before = [(path.stat().st_mtime_ns, path.stat().st_size) for path in paths] + if sum(size for _, size in before) > max_bytes: + raise ValueError("ledger_byte_limit") + records = [] + complete = True + for path in paths: + with path.open("rb") as stream: + raw = stream.read(max_bytes + 1) + if len(raw) > max_bytes: + raise ValueError("ledger_byte_limit") + data = json.loads(raw) + if isinstance(data, dict) and "tasks" in data: + complete = complete and data.get("complete") is True + tasks = data["tasks"] + else: + tasks = data if isinstance(data, list) else [data] + if not isinstance(tasks, list): + raise ValueError("invalid_ledger") + records.extend(tasks) + if len(records) > 1024: + raise ValueError("ledger_record_limit") + cards = {} + for row in records: + if not isinstance(row, dict): + raise ValueError("invalid_ledger_record") + if row.get("repo") != config.ledger_repo: + continue + if row.get("blind") or row.get("is_blind") or row.get("visibility", "public") != "public": + continue + selected = {name: row.get(name) for name in ("id", "repo", "state", "title", "version")} + if not all( + isinstance(selected[name], str) and selected[name].strip() + for name in ("id", "repo", "state", "title") + ) or not _safe(selected): + complete = False + continue + card_id = selected["id"] + if card_id in cards: + raise ValueError("duplicate_card") + support = { + "kind": "card", + "id": card_id, + "hash": digest(selected), + "version": selected["version"], + "state": selected["state"], + } + cards[card_id] = { + "key": "card:" + card_id, + "section": "Open cards", + "text": f"{_display(card_id, 48)} [{_display(selected['state'], 24)}] {_display(selected['title'], 80)}", + "support": support, + "open": selected["state"] in OPEN_STATES, + "sort": [selected["state"], card_id], + } + if len(cards) > 256: + raise ValueError("ledger_card_limit") + after = [(path.stat().st_mtime_ns, path.stat().st_size) for path in paths] + if before != after or paths != _ledger_paths(config): + if attempt == 0: + continue + raise ValueError("ledger_changed_during_read") + return cards, complete, [] if complete else ["ledger_incomplete"] + except (OSError, ValueError, TypeError, KeyError): + return {}, False, ["ledger_unavailable"] + return {}, False, ["ledger_unavailable"] + + +def _section(row: dict) -> str | None: + mapping = { + "project_purpose": "Purpose", + "purpose": "Purpose", + "constraint": "Constraints", + "rule": "Constraints", + "invariant": "Constraints", + "decision": "Decisions", + "architecture_decision": "Decisions", + "gotcha": "Gotchas", + "pitfall": "Gotchas", + "workaround": "Gotchas", + } + return mapping.get(row.get("predicate")) or mapping.get(row.get("claim_type")) + + +def _claim_item(row: dict, citations: list[dict]) -> dict | None: + if ( + row.get("status") != "confirmed" + or (row.get("visibility") or "public") != "public" + or row.get("replaced_by_claim_id") is not None + or row.get("source_agent") in GENERATED_SOURCES + or not citations + or not _section(row) + ): + return None + relevant = { + name: row.get(name) + for name in ( + "text", + "subject", + "predicate", + "object_value", + "claim_type", + "status", + "visibility", + "replaced_by_claim_id", + "source_agent", + "valid_from", + "valid_until", + "scope", + "tenant_id", + "event_time", + "created_at", + "pinned", + ) + } + if not _safe(relevant) or not all(_safe(citation) for citation in citations): + return None + if not claim_is_temporally_current( + SimpleNamespace(valid_from=row.get("valid_from"), valid_until=row.get("valid_until")) + ): + return None + if not str(row.get("text") or "").strip(): + return None + if not all(citation.get("source") and citation.get("locator") for citation in citations): + return None + support = { + "kind": "claim", + "id": int(row["id"]), + "version": int(row.get("version") or 1), + "hash": digest(relevant), + "citations": [{"id": int(c["id"]), "hash": digest(c)} for c in citations], + } + section = _section(row) + date = row.get("event_time") or row.get("created_at") or "" + text = _display(row["text"]) + if section == "Decisions": + label = "event" if row.get("event_time") else "recorded" + text = f"{str(date)[:10]} ({label}): {text}" + return { + "key": f"claim:{row['id']}", + "section": section, + "text": text, + "support": support, + "pinned": bool(row.get("pinned")), + "sort": [0 if row.get("pinned") else 1, str(date), int(row["id"])], + } + + +def _read_claims( + config: Any, *, ids: list[int] | None = None, changed_since: str | None = None, validation: bool = False +) -> tuple[dict, bool, dict]: + limit = 64 if validation else MAX_CLAIMS + citation_limit = 256 if validation else MAX_CITATIONS + byte_limit = 2 * 1024 * 1024 if validation else MAX_SOURCE_BYTES + if ids is not None and len(ids) > MAX_CLAIMS: + raise ValueError("claim_limit") + deadline = time.monotonic() + 0.1 + with closing( + sqlite3.connect(Path(config.db_path).resolve().as_uri() + "?mode=ro", uri=True, timeout=0.025) + ) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA query_only=ON") + conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000) + conn.execute("BEGIN") + scopes = (config.scope, *config.aliases) + params: list[Any] = [config.tenant_id, *scopes] + where = "tenant_id IS ? AND scope IN (" + ",".join("?" for _ in scopes) + ")" + if ids is not None: + clauses = [] + if ids: + clauses.append("id IN (" + ",".join("?" for _ in ids) + ")") + params.extend(ids) + if changed_since: + # Inclusive changes catch old-candidate confirmation and equal timestamps. + clauses.extend( + [ + "updated_at >= ?", + "valid_from >= ?", + "EXISTS (SELECT 1 FROM citations c WHERE c.claim_id=claims.id AND c.created_at >= ?)", + ] + ) + params.extend([changed_since] * 3) + if not clauses: + return {}, True, {"data_version": conn.execute("PRAGMA data_version").fetchone()[0]} + where += " AND (" + " OR ".join(clauses) + ")" + else: + where += ( + " AND status='confirmed' AND COALESCE(visibility,'public')='public' AND replaced_by_claim_id IS NULL" + ) + fields = [row[1] for row in conn.execute("PRAGMA table_info(claims)")] + names = [ + name + for name in fields + if name + in { + "id", + "version", + "text", + "subject", + "predicate", + "object_value", + "claim_type", + "status", + "visibility", + "replaced_by_claim_id", + "source_agent", + "valid_from", + "valid_until", + "scope", + "tenant_id", + "event_time", + "created_at", + "updated_at", + "pinned", + } + ] + # Reject oversized rows before materializing their payload in Python. + size = "+".join(f"length(CAST(COALESCE({name},'') AS BLOB))" for name in names) + columns = ",".join( + f"substr({name},1,{MAX_FIELD_BYTES + 1}) AS {name}" + if name not in ("id", "version", "pinned", "replaced_by_claim_id") + else name + for name in names + ) + rows = conn.execute( + f"SELECT {columns}, ({size}) AS _source_bytes FROM claims WHERE {where} ORDER BY id LIMIT ?", + [*params, limit + 1], + ).fetchall() + complete = len(rows) <= limit + rows = rows[:limit] + oversized = {row["id"] for row in rows if row["_source_bytes"] > MAX_FIELD_BYTES} + complete = complete and not oversized + ids_read = [row["id"] for row in rows] + citations: dict[int, list] = {} + raw_citations = [] + if ids_read: + placeholders = ",".join("?" for _ in ids_read) + raw_citations = conn.execute( + "SELECT id,claim_id," + + ",".join( + f"substr({name},1,{MAX_FIELD_BYTES + 1}) AS {name}" for name in ("source", "locator", "excerpt") + ) + + ",length(CAST(COALESCE(source,'')||COALESCE(locator,'')||COALESCE(excerpt,'') AS BLOB)) AS _source_bytes " + f"FROM citations WHERE claim_id IN ({placeholders}) ORDER BY id LIMIT ?", + [*ids_read, citation_limit + 1], + ).fetchall() + complete = complete and len(raw_citations) <= citation_limit + if len(raw_citations) > citation_limit: + return {}, False, {} + for citation in raw_citations[:citation_limit]: + if citation["_source_bytes"] > MAX_FIELD_BYTES: + oversized.add(citation["claim_id"]) + complete = False + value = dict(citation) + value.pop("_source_bytes") + citations.setdefault(citation["claim_id"], []).append(value) + total = sum(len(json.dumps(dict(row)).encode()) for row in [*rows, *raw_citations]) + if total > byte_limit: + return {}, False, {} + items = {} + for row in rows: + if row["id"] in oversized: + continue + item = _claim_item(dict(row), citations.get(row["id"], [])) + if item: + items[item["key"]] = item + now = utc_now() + boundaries = [ + str(row[name]) + for row in rows + for name in ("valid_from", "valid_until") + if row[name] and str(row[name]) > now + ] + fingerprint = { + "data_version": conn.execute("PRAGMA data_version").fetchone()[0], + "next_temporal_boundary": min(boundaries) if boundaries else None, + "source_bytes": total, + } + return items, complete, fingerprint + + +def snapshot_since(config: Any, coverage: dict, read_at: str) -> dict: + return load_project_snapshot(config, coverage=coverage, changed_since=read_at) + + +def load_project_snapshot( + config: Any, + *, + coverage: dict | None = None, + changed_since: str | None = None, + claim_ids: list[int] | None = None, + validation: bool = False, +) -> dict: + result: dict[str, Any] = { + "read_at": utc_now(), + "coverage": {}, + "items": {}, + "complete": False, + "diagnostics": [], + "fingerprints": {}, + "next_temporal_boundary": None, + } + if not config.enabled: + result["diagnostics"] = ["inactive"] + return result + try: + ids = claim_ids + if coverage is not None: + ids = [int(key.split(":", 1)[1]) for key in coverage if key.startswith("claim:")] + items, complete, fingerprints = _read_claims( + config, ids=ids, changed_since=changed_since, validation=validation + ) + except (sqlite3.Error, OSError, ValueError, TypeError): + items, complete, fingerprints = {}, False, {} + result["diagnostics"].append("claims_unavailable") + byte_budget = 2 * 1024 * 1024 if validation else MAX_SOURCE_BYTES + cards, ledger_complete, diagnostics = load_ledger( + config, max_bytes=max(0, byte_budget - int(fingerprints.get("source_bytes", 0))) + ) + items.update((card["key"], card) for card in cards.values()) + result.update(items=items, complete=complete and ledger_complete, fingerprints=fingerprints) + result["next_temporal_boundary"] = fingerprints.get("next_temporal_boundary") + result["coverage"] = {key: item["support"] for key, item in items.items() if item.get("open", True)} + result["diagnostics"].extend(diagnostics) + if len(json.dumps(result["coverage"]).encode()) > 128 * 1024: + result.update(coverage={}, complete=False) + result["diagnostics"].append("coverage_limit") + return result + + +def load_confirmed_project_claims(config: Any) -> list[dict]: + snapshot = load_project_snapshot(config) + return [item for key, item in snapshot["items"].items() if key.startswith("claim:")] diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py new file mode 100644 index 00000000..0dbc1b2a --- /dev/null +++ b/tests/test_project_projection.py @@ -0,0 +1,160 @@ +from dataclasses import replace +import json +import sqlite3 + +from test_project_repository import claim + + +from memorymaster.profile.project_projection import ( + build_project_projection, + publish_project_projection, + read_project_projection, + projection_paths, + project_lock, +) + +pytest_plugins = ["test_project_repository"] + + +def test_exact_manifest_and_per_bullet_invalidation(project_config): + config = project_config + for i in range(1, 4): + claim(config, i, text=f"Decision number {i}") + built = build_project_projection(config) + assert publish_project_projection(config, built) + before = read_project_projection(config) + with sqlite3.connect(config.db_path) as conn: + conn.execute("UPDATE claims SET visibility='private',version=2 WHERE id=2") + after = read_project_projection(config) + assert "Decision number 1" in after.payload and "Decision number 3" in after.payload + assert "Decision number 2" not in after.payload + assert before.manifest["view_id"] != after.manifest["view_id"] + assert all(s["id"] != 2 for item in after.manifest["items"] for s in item["supports"]) + assert "source_coverage" not in after.public_manifest + + +def test_modified_still_open_cards_are_rerendered_without_trim_warning(project_config): + config = project_config + path = config.ledger_root / "tasks.json" + path.write_text( + json.dumps({"complete": True, "tasks": [{"id": "T-1", "repo": "demo", "state": "ready", "title": "Before"}]}) + ) + assert publish_project_projection(config, build_project_projection(config)) + path.write_text( + json.dumps({"complete": True, "tasks": [{"id": "T-1", "repo": "demo", "state": "running", "title": "After"}]}) + ) + result = read_project_projection(config) + assert "After" in result.payload and "Before" not in result.payload + assert "open_cards_trimmed" not in result.payload + + +def test_utf8_cap_card_reservation_and_m2_warning(project_config): + config = project_config + claim(config, text="Verbose constraint " * 300, claim_type="constraint", pinned=1) + path = config.ledger_root / "tasks.json" + path.write_text( + json.dumps( + { + "complete": True, + "tasks": [{"id": f"T-{i}", "repo": "demo", "state": "ready", "title": "Á😀" * 40} for i in range(12)], + } + ) + ) + built = build_project_projection(config) + assert len(built.markdown.encode()) <= 2200 + assert built.markdown.count("") == 1 + assert built.markdown.count("") == 1 + assert len(built.manifest["items"]) <= 64 + tiny = build_project_projection(replace(config, budget_bytes=700)) + assert len(tiny.markdown.encode()) <= 700 + assert "m2_not_ready:open_cards_trimmed" in tiny.payload + + +def test_pair_publication_and_validation_limits(project_config): + config = project_config + claim(config) + assert publish_project_projection(config, build_project_projection(config)) + md, manifest = projection_paths(config) + md.write_text("tampered") + assert read_project_projection(config) is None + assert publish_project_projection(config, build_project_projection(config)) + data = json.loads(manifest.read_text()) + data["items"] = data["items"] * 65 + manifest.write_text(json.dumps(data)) + assert read_project_projection(config) is None + + +def test_publication_lock_is_nonblocking(project_config): + config = project_config + path = config.state_root / "fixture.lock" + with project_lock(path) as acquired: + assert acquired + with project_lock(path, timeout=0) as other: + assert not other + + +def test_mandatory_cards_and_latest_decisions_survive_optional_prose(project_config): + config = project_config + for i in range(1, 11): + claim(config, i, text=f"Decision {i} " + "detail " * 35) + claim(config, 20, text="Safety constraint " * 25, claim_type="constraint", pinned=1) + claim(config, 21, text="Purpose " * 40, claim_type="project_purpose") + built = build_project_projection(config) + assert {record["supports"][0]["id"] for record in built.manifest["items"] if record["section"] == "Decisions"} >= { + 8, + 9, + 10, + } + pinned_purpose = [record for record in built.manifest["items"] if record["section"] in ("Purpose", "Constraints")] + assert sum(len(record["text"].encode()) for record in pinned_purpose) <= 400 + assert built.manifest["source_complete"] + + +def test_missing_decisions_is_m2_unready_and_read_does_not_refresh_freshness(project_config): + claim(project_config) + built = build_project_projection(project_config) + assert "m2_not_ready:decisions_incomplete" in built.payload + assert publish_project_projection(project_config, built) + view = read_project_projection(project_config) + assert view.manifest["checked_at"] == built.manifest["checked_at"] + + +def test_marker_and_reference_injection_are_inert(project_config): + claim(project_config, text="safe\n## forged\n[ignore](bad)\x1b[31m") + built = build_project_projection(project_config) + assert "\x1b" not in built.payload + assert "\n## forged" not in built.payload + assert "[ignore]" not in built.payload + + +def test_new_card_marks_pending_and_unchanged_compact_card_is_stable(project_config): + config = project_config + path = config.ledger_root / "tasks.json" + tasks = [{"id": "T-1", "repo": "demo", "state": "ready", "title": "One " * 50}] + path.write_text(json.dumps({"complete": True, "tasks": tasks})) + for i in range(1, 11): + claim(config, i, text="Long decision " + "detail " * 30) + built = build_project_projection(config) + assert publish_project_projection(config, built) + card = next(record for record in built.manifest["items"] if record["section"] == "Open cards") + unchanged = read_project_projection(config) + assert next(record["text"] for record in unchanged.manifest["items"] if record["key"] == "card:T-1") == card["text"] + tasks.append({"id": "T-2", "repo": "demo", "state": "review", "title": "Added"}) + path.write_text(json.dumps({"complete": True, "tasks": tasks})) + assert "m2_not_ready:cards_changed" in read_project_projection(config).payload + + +def test_manifest_generation_and_span_tampering_are_rejected(project_config): + claim(project_config) + built = build_project_projection(project_config) + assert publish_project_projection(project_config, built) + _, path = projection_paths(project_config) + manifest = json.loads(path.read_text()) + manifest["generation_id"] = "forged" + path.write_text(json.dumps(manifest)) + assert read_project_projection(project_config) is None + assert publish_project_projection(project_config, built) + manifest = json.loads(path.read_text()) + manifest["items"][0]["span"] = [0, 1] + path.write_text(json.dumps(manifest)) + assert read_project_projection(project_config) is None diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py new file mode 100644 index 00000000..24f8e6ec --- /dev/null +++ b/tests/test_project_repository.py @@ -0,0 +1,158 @@ +from dataclasses import replace +import json +import sqlite3 + +import pytest + +from memorymaster.profile.project_projection import ProjectProjectionConfig +from memorymaster.profile.project_repository import load_project_snapshot, snapshot_since + + +@pytest.fixture +def project_config(tmp_path): + root = tmp_path / "demo" + root.mkdir() + (root / ".git").mkdir() + (root / ".git" / "HEAD").write_text("ref: refs/heads/main\n") + db = tmp_path / "fixture.sqlite" + with sqlite3.connect(db) as conn: + conn.executescript("""CREATE TABLE claims(id INTEGER PRIMARY KEY,text TEXT, + scope TEXT,tenant_id TEXT,status TEXT,visibility TEXT,source_agent TEXT, + replaced_by_claim_id INTEGER,version INTEGER DEFAULT 1,claim_type TEXT, + subject TEXT,predicate TEXT,object_value TEXT,pinned INTEGER DEFAULT 0, + created_at TEXT,updated_at TEXT,event_time TEXT,valid_from TEXT,valid_until TEXT); + CREATE TABLE citations(id INTEGER PRIMARY KEY,claim_id INTEGER,source TEXT, + locator TEXT,excerpt TEXT,created_at TEXT);""") + ledger = tmp_path / "ledger" + ledger.mkdir() + (ledger / "tasks.json").write_text(json.dumps({"complete": True, "tasks": []})) + return ProjectProjectionConfig( + db_path=db, + scope="project:demo", + tenant_id=None, + context_key="fixture", + epoch="pilot", + project_root=root, + output_root=tmp_path / "out", + state_root=tmp_path / "state", + ledger_root=ledger, + ledger_repo="demo", + ledger_filtered=True, + enabled=True, + ) + + +def claim(config, id=1, text="Use bounded reads", **changes): + values = dict( + id=id, + text=text, + scope="project:demo", + tenant_id=None, + status="confirmed", + visibility="public", + source_agent="operator", + version=1, + claim_type="decision", + created_at="2026-10-01T00:00:00+00:00", + updated_at="2026-10-01T00:00:00+00:00", + ) + values.update(changes) + with sqlite3.connect(config.db_path) as conn: + conn.execute( + f"INSERT INTO claims ({','.join(values)}) VALUES ({','.join('?' for _ in values)})", list(values.values()) + ) + conn.execute( + "INSERT INTO citations(claim_id,source,locator,excerpt) VALUES (?,?,?,?)", + (id, "session", f"session:{id}", "Authored evidence"), + ) + + +def test_confirmed_authorized_current_selection(project_config): + config = project_config + claim(config) + for i, changes in enumerate( + [ + {"status": "candidate"}, + {"scope": "project:other"}, + {"tenant_id": "other"}, + {"visibility": "private"}, + {"valid_until": "2001-01-01"}, + {"valid_from": "2999-01-01"}, + {"replaced_by_claim_id": 1}, + {"source_agent": "project-projection"}, + {"subject": "password=secret-password"}, + ], + 2, + ): + claim(config, i, **changes) + claim(config, 20, subject="engine", predicate="uses", object_value=None) + result = load_project_snapshot(config) + assert set(result["items"]) == {"claim:1", "claim:20"} + assert result["complete"] + assert result["coverage"]["claim:1"]["kind"] == "claim" + + +def test_ledger_actual_schema_visibility_and_completeness(project_config): + config = project_config + path = config.ledger_root / "tasks.json" + path.write_text( + json.dumps( + { + "complete": True, + "tasks": [ + {"id": "T-1", "repo": "demo", "state": "ready", "title": "Visible"}, + {"id": "T-2", "repo": "demo", "state": "blocked", "title": "Hidden", "blind": True}, + {"id": "T-3", "repo": "other", "state": "ready", "title": "Other"}, + {"id": "T-4", "repo": "demo", "state": "done", "title": "Closed"}, + ], + } + ) + ) + result = load_project_snapshot(config) + assert set(result["items"]) == {"card:T-1", "card:T-4"} + assert set(result["coverage"]) == {"card:T-1"} + path.write_text("{") + result = load_project_snapshot(config) + assert not result["complete"] + assert "ledger_unavailable" in result["diagnostics"] + + +def test_incremental_baseline_rehydrates_coverage_and_changed_rows(project_config): + config = project_config + claim(config) + baseline = load_project_snapshot(config) + claim(config, 2, status="candidate") + with sqlite3.connect(config.db_path) as conn: + conn.execute("UPDATE claims SET status='confirmed',updated_at=? WHERE id=2", (baseline["read_at"],)) + conn.execute("UPDATE citations SET excerpt='Edited evidence' WHERE claim_id=1") + current = snapshot_since(config, baseline["coverage"], baseline["read_at"]) + assert set(current["coverage"]) == {"claim:1", "claim:2"} + assert current["coverage"]["claim:1"] != baseline["coverage"]["claim:1"] + + +def test_explicit_context_and_disabled_have_no_source_reads(project_config): + config = replace(project_config, enabled=False, db_path=project_config.db_path / "missing") + assert load_project_snapshot(config)["diagnostics"] == ["inactive"] + with pytest.raises(ValueError): + replace(project_config, context_key="../escape") + with pytest.raises(ValueError): + replace(project_config, db_path="postgresql://localhost/db") + + +def test_oversized_citation_cannot_hide_sensitive_or_generated_payload(project_config): + claim(project_config) + with sqlite3.connect(project_config.db_path) as conn: + conn.execute( + "INSERT INTO citations(claim_id,source,locator,excerpt) VALUES (1,?,?,?)", + ("session", "extra", "x" * 17000 + ""), + ) + snapshot = load_project_snapshot(project_config) + assert "claim:1" not in snapshot["items"] + assert not snapshot["complete"] + + +def test_incremental_includes_temporal_boundary_without_row_edit(project_config): + claim(project_config, valid_from="2026-10-01T00:00:00+00:00") + snapshot = snapshot_since(project_config, {}, "2026-09-30T00:00:00+00:00") + assert "claim:1" in snapshot["coverage"] + assert "next_temporal_boundary" in snapshot From 96bc276e75c47c1b50bb198d031c83b92491302b Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:01:13 -0300 Subject: [PATCH 07/51] fix(T-1703): preserve continuity and cheap refresh across missing caches --- memorymaster/profile/project_runtime.py | 95 +++++++++++--- memorymaster/profile/project_sessions.py | 41 ++++-- tests/test_project_integration.py | 152 +++++++++++++++++++++++ tests/test_project_runtime.py | 28 ++++- tests/test_project_sessions.py | 52 ++++++++ 5 files changed, 342 insertions(+), 26 deletions(-) create mode 100644 tests/test_project_integration.py diff --git a/memorymaster/profile/project_runtime.py b/memorymaster/profile/project_runtime.py index 2425546e..e4e28c0a 100644 --- a/memorymaster/profile/project_runtime.py +++ b/memorymaster/profile/project_runtime.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +import hashlib import os import sqlite3 import time +from itertools import islice from datetime import datetime, timezone from pathlib import Path @@ -141,22 +143,21 @@ def session_start_context(data: dict, *, profile_loader=None) -> str | None: return None config = config_for_cwd(data.get("cwd") or os.getcwd()) if config is None: - return None + return compose_session_context("Project context unavailable; project authorization unresolved.") from memorymaster.profile.project_projection import read_project_projection projection = read_project_projection(config) - if projection is None: - return compose_session_context("Project context unavailable; refresh pending.") delta = "" if active("MEMORYMASTER_PROJECT_SESSION_DELTA"): from memorymaster.profile.project_sessions import record_project_session_start result = record_project_session_start(config, str(data.get("session_id") or ""), - projection=projection.manifest) + projection=projection.manifest if projection else None) delta = (result["delta"].text if result.get("delta") else "Prior-session continuity unavailable: " + result["status"]) profile = profile_loader() if profile_loader is not None else "" - return compose_session_context(projection.payload, delta, profile) + return compose_session_context(projection.payload if projection else + "Project context unavailable; refresh pending.", delta, profile) def record_session_end(data: dict) -> bool: @@ -189,25 +190,80 @@ def stat(path): except FileNotFoundError: return None files = [stat(config.db_path), stat(Path(str(config.db_path) + "-wal"))] - if config.ledger_root: + if config.ledger_root and config.ledger_root.is_file(): + files.append(stat(config.ledger_root)) + elif config.ledger_root: # Only configured JSON records, bounded before materializing a directory list. - for index, path in enumerate(sorted(config.ledger_root.glob("*.json"))): - if index >= 1024: - raise ValueError("ledger_capacity") + paths = list(islice(config.ledger_root.glob("*.json"), 1025)) + if len(paths) > 1024: + raise ValueError("ledger_capacity") + for path in sorted(paths): files.append((path.name, stat(path))) # An elapsed second can cross valid_from/valid_until even without a database writer. # Cache stores the next temporal boundary separately after each full source read. - return (data_version, *files) + return (data_version, repr(config), *files) + + +def _cheap_hash(config) -> str: + return hashlib.sha256(json.dumps(cheap_fingerprint(config), sort_keys=True).encode()).hexdigest() + + +def _persisted_skip_hint(config, cheap: tuple) -> bool: + from memorymaster.profile.project_projection import read_manifest + + manifest = read_manifest(config) + if not manifest: + return False + hint = manifest.get("refresh_hint") + now = datetime.now(timezone.utc) + if not hint or hint.get("sha256") != hashlib.sha256(json.dumps(cheap, sort_keys=True).encode()).hexdigest(): + return False + checked = datetime.fromisoformat(manifest["checked_at"].replace("Z", "+00:00")) + boundary = manifest.get("next_temporal_boundary") + return (0 <= (now - checked).total_seconds() < 3600 and + (not boundary or now < datetime.fromisoformat(boundary.replace("Z", "+00:00")))) -def _refresh_one(config) -> dict: - from memorymaster.profile.project_projection import build_project_projection, publish_project_projection +def _refresh_one(config, *, skip: bool = False) -> dict: + from memorymaster.profile.project_projection import ( + atomic_json, + build_project_projection, + project_lock, + projection_paths, + publish_project_projection, + read_manifest, + ) + previous = read_manifest(config) + before = _cheap_hash(config) projection = build_project_projection(config) - if not projection.manifest.get("complete", projection.manifest.get("source_complete", False)): + manifest = projection.manifest + if not manifest.get("source_complete", False): return {"status": "pending"} + after = _cheap_hash(config) + if before != after: + return {"status": "pending"} + manifest["refresh_hint"] = {"sha256": after} + boundary = manifest.get("next_temporal_boundary") + def item_keys(value): + return [(item["key"], item["text"], item["supports"]) for item in value["items"]] + if (previous and previous["source_coverage"] == manifest["source_coverage"] + and item_keys(previous) == item_keys(manifest) + and previous["diagnostics"] == manifest["diagnostics"] + and previous["budget_bytes"] == manifest["budget_bytes"]): + if not skip or previous.get("refresh_hint") != manifest["refresh_hint"]: + _, path = projection_paths(config) + with project_lock(path.with_suffix(".lock"), timeout=0) as acquired: + if not acquired or read_manifest(config) != previous: + return {"status": "pending"} + if not skip: + previous["checked_at"] = datetime.now(timezone.utc).isoformat() + previous["refresh_hint"] = manifest["refresh_hint"] + previous["next_temporal_boundary"] = boundary + atomic_json(path, previous) + return {"status": "ok", "next_temporal_boundary": boundary} published = publish_project_projection(config, projection) - return {"status": "ok" if published else "pending"} + return {"status": "ok" if published else "pending", "next_temporal_boundary": boundary} def refresh_project_projections(*, skip: bool = False) -> dict: @@ -233,9 +289,16 @@ def refresh_project_projections(*, skip: bool = False) -> dict: if skip and cached and cached[0] == cheap and time.monotonic() < cached[1]: results.append({"status": "unchanged"}) continue - result = _refresh_one(config) + if skip and not cached and hasattr(config, "db_path") and _persisted_skip_hint(config, cheap): + results.append({"status": "unchanged"}) + continue + result = _refresh_one(config, skip=skip) if result["status"] == "ok": - _refresh_cache[key] = (cheap, time.monotonic() + 60) + seconds = 60.0 + if result.get("next_temporal_boundary"): + boundary = datetime.fromisoformat(result["next_temporal_boundary"].replace("Z", "+00:00")) + seconds = min(seconds, max(0, (boundary - datetime.now(timezone.utc)).total_seconds())) + _refresh_cache[key] = (cheap, time.monotonic() + seconds) results.append(result) except (OSError, ValueError, sqlite3.Error): results.append({"status": "pending", "reason": "source_unavailable"}) diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py index bf65785e..6f1c54ee 100644 --- a/memorymaster/profile/project_sessions.py +++ b/memorymaster/profile/project_sessions.py @@ -26,6 +26,8 @@ def _digest(value: object) -> str: def _time(value: str) -> datetime: + if not isinstance(value, str): + raise ValueError("invalid timestamp") parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: raise ValueError("timezone required") @@ -34,7 +36,7 @@ def _time(value: str) -> datetime: def session_state_path(config) -> Path: slug = config.scope.removeprefix("project:") - if not all(re.fullmatch(r"[A-Za-z0-9_-]{1,128}", part) for part in (slug, config.context_key)): + if not all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,95}", part) for part in (slug, config.context_key)): raise ValueError("invalid project identity") return Path(config.state_root) / config.context_key / slug / "sessions.json" @@ -57,20 +59,38 @@ def read_session_state(config) -> dict: if len(raw) > MAX_STATE_BYTES: raise ValueError("state_capacity") state = json.loads(raw) + if not isinstance(state, dict): + raise ValueError("invalid_state") if (state.get("schema") != SCHEMA or state.get("scope") != config.scope or state.get("context_key") != config.context_key): raise ValueError("state_identity") if state.get("activation_epoch") != config.epoch: return _empty(config) + if (type(state.get("seq")) is not int or state["seq"] < 0 + or not isinstance(state.get("active"), dict) + or not isinstance(state.get("completed"), list) + or not isinstance(state.get("baselines"), dict)): + raise ValueError("invalid_state") if (len(state["active"]) > 16 or len(state["completed"]) > 4 or len(state["baselines"]) > 36): raise ValueError("state_capacity") for key, baseline in state["baselines"].items(): - if len(_encoded(baseline)) > MAX_BASELINE_BYTES or _digest(baseline) != key: + if not isinstance(baseline, dict) or len(_encoded(baseline)) > MAX_BASELINE_BYTES or _digest(baseline) != key: raise ValueError("baseline_integrity") for record in [*state["active"].values(), *state["completed"]]: + if (not isinstance(record, dict) or not isinstance(record.get("session"), str) + or not re.fullmatch(r"[0-9a-f]{64}", record["session"]) + or type(record.get("seq")) is not int or record["seq"] > state["seq"]): + raise ValueError("invalid_record") + _time(record["start_at"]) + _time(record["read_at"]) + if record.get("end_at"): + if _time(record["end_at"]) < _time(record["start_at"]): + raise ValueError("invalid_end") if record["baseline"] not in state["baselines"]: raise ValueError("baseline_unavailable") + if record.get("prior_baseline") and record["prior_baseline"] not in state["baselines"]: + raise ValueError("baseline_unavailable") latest = state["latest_end"] if latest is not None and latest not in state["completed"]: raise ValueError("end_pointer_integrity") @@ -93,9 +113,11 @@ def _short(text: str, size: int) -> str: def build_session_delta(baseline: dict | None, projection_coverage: dict | None, current: dict, *, diagnostic: str = "") -> SessionDelta: """Union start and projection differences, with current authorized supports only.""" - coverage = current.get("coverage", {}) + def factual(value): + return {key: support for key, support in value.items() if key.startswith(("claim:", "card:"))} + coverage = factual(current.get("coverage", {})) items = current.get("items", {}) - comparisons = [value for value in (baseline, projection_coverage) if value is not None] + comparisons = [factual(value) for value in (baseline, projection_coverage) if value is not None] changed = {key for value in comparisons for key in coverage if value.get(key) != coverage[key]} removed = {key for value in comparisons for key in value if key not in coverage} # A currently authorized closed card may be carried in items outside open-card coverage. @@ -172,7 +194,7 @@ def record_project_session_start(config, session_id: str, *, projection: dict | return {"status": "baseline_unavailable"} from memorymaster.profile.project_projection import atomic_json, project_lock - now = now or datetime.now(timezone.utc) + now = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) sid = hashlib.sha256(session_id.encode()).hexdigest() path = session_state_path(config) lock = path.with_suffix(".lock") @@ -194,11 +216,12 @@ def record_project_session_start(config, session_id: str, *, projection: dict | prior_id = prior["baseline"] if prior else None baseline = copy.deepcopy(state["baselines"].get(prior_id)) if snapshot_loader is None: - if projection is None: + if projection is None and baseline is None: return {"status": "baseline_unavailable"} from memorymaster.profile.project_repository import snapshot_since - current = snapshot_since(config, projection["source_coverage"], projection["read_at"]) + current = snapshot_since(config, projection["source_coverage"] if projection else baseline, + projection["read_at"] if projection else prior["read_at"]) else: current = snapshot_loader() if not current.get("complete"): @@ -243,10 +266,10 @@ def record_project_session_end(config, session_id: str, *, end_at: str | None = return False from memorymaster.profile.project_projection import atomic_json, project_lock - now = now or datetime.now(timezone.utc) + now = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) try: end = _time(end_at) if end_at else now - if end > now + timedelta(minutes=1): + if end > now: return False path = session_state_path(config) sid = hashlib.sha256(session_id.encode()).hexdigest() diff --git a/tests/test_project_integration.py b/tests/test_project_integration.py new file mode 100644 index 00000000..ba1cdc55 --- /dev/null +++ b/tests/test_project_integration.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +import importlib.util +import io +import json +from pathlib import Path +import shutil +import sqlite3 +import sys + +from test_project_repository import claim + +pytest_plugins = ["test_project_repository"] + +from memorymaster.operations.operational_review import ReviewConfig, Verdict, check_project_projection +from memorymaster.profile import project_repository, project_runtime +from memorymaster.profile.project_projection import ( + build_project_projection, projection_paths, publish_project_projection, read_project_projection, +) +from memorymaster.profile.project_sessions import record_project_session_start + + +def _hook(name): + path = Path(__file__).resolve().parents[1] / "memorymaster/config_templates/hooks" / name + spec = importlib.util.spec_from_file_location("ao3_" + name.replace("-", "_"), path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_m2_three_repo_probe(project_config, tmp_path, monkeypatch, capsys): + import pytest + + configs = [] + for index in range(3): + root = tmp_path / f"repo{index}" + root.mkdir() + (root / ".git").mkdir() + (root / ".git/HEAD").write_text("ref: refs/heads/main\n") + db = tmp_path / f"repo{index}.sqlite" + shutil.copyfile(project_config.db_path, db) + ledger = tmp_path / f"ledger{index}" + ledger.mkdir() + cfg = replace(project_config, project_root=root, scope=f"project:repo{index}", + db_path=db, ledger_root=ledger, ledger_repo=f"repo{index}") + for number in range(1, 4): + claim(cfg, number, scope=cfg.scope, text=f"Repository {index} decision {number}") + claim(cfg, 4, scope=cfg.scope, status="candidate", text=f"Late confirmed choice {index}") + (ledger / "tasks.json").write_text(json.dumps({"complete": True, "tasks": [ + {"id": f"T-{index}", "repo": cfg.ledger_repo, "state": "ready", "title": f"Visible work {index}"}]})) + assert publish_project_projection(cfg, build_project_projection(cfg)) + configs.append(cfg) + for suffix in ("ENABLED", "SESSION_START"): + monkeypatch.setenv(project_runtime.PREFIX + suffix, "1") + monkeypatch.setenv("MEMORYMASTER_PROJECT_SESSION_DELTA", "1") + monkeypatch.setenv("MEMORYMASTER_SESSION_END_CAPTURE", "0") + monkeypatch.setattr(project_runtime, "load_project_configs", lambda: configs) + start = _hook("memorymaster-session-start.py") + end = _hook("memorymaster-session-end.py") + monkeypatch.setattr(start, "_load_compiled_profile", lambda: "") + monkeypatch.setattr(start, "_log", lambda *args, **kwargs: None) + # The enabled branch has no independent claim/wiki SQL or recall/model fallback. + monkeypatch.setattr(start, "_query_recent_claims", lambda *args: pytest.fail("legacy SQL used")) + for cfg in configs: + payload = {"cwd": str(cfg.project_root), "session_id": cfg.scope + "-S"} + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(payload))) + with pytest.raises(SystemExit): + start.main() + first = json.loads(capsys.readouterr().out)["hookSpecificOutput"]["additionalContext"] + for number in range(1, 4): + assert f"decision {number}" in first + with sqlite3.connect(cfg.db_path) as connection: + connection.execute("UPDATE claims SET status='confirmed',updated_at=? WHERE id=4", + (datetime.now(timezone.utc).isoformat(),)) + path = cfg.ledger_root / "tasks.json" + current = json.loads(path.read_text()) + current["tasks"][0].update(state="running", title="Edited visible work") + path.write_text(json.dumps(current)) + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(payload))) + end.main() + payload["session_id"] += "-next" + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(payload))) + with pytest.raises(SystemExit): + start.main() + context = json.loads(capsys.readouterr().out)["hookSpecificOutput"]["additionalContext"] + shared = read_project_projection(cfg) + assert shared.payload in context # Same reader contract used by the external Cockpit owner. + assert "Late confirmed choice" in context + assert "Edited visible work" in context + assert "open_cards_trimmed" not in context + assert "prior-session" in context.lower() + assert len(context) <= 3000 + assert context.count(project_runtime.GENERATED_OPENER) == context.count(project_runtime.GENERATED_CLOSER) == 1 + + +def test_session_start_baseline_never_full_source_scan(project_config, monkeypatch): + claim(project_config) + projection = build_project_projection(project_config) + observed = [] + original = project_repository._read_claims + def bounded(config, **kwargs): + observed.append(kwargs) + assert kwargs["ids"] is not None + assert kwargs["changed_since"] == projection.manifest["read_at"] + return original(config, **kwargs) + monkeypatch.setattr(project_repository, "_read_claims", bounded) + result = record_project_session_start(project_config, "fixture", projection=projection.manifest) + assert result["status"] == "ok" + assert len(observed) == 1 + + +def test_skip_unchanged_refresh_keeps_files_and_checked_at(project_config): + claim(project_config) + assert project_runtime._refresh_one(project_config)["status"] == "ok" + paths = projection_paths(project_config) + before = [(path.read_bytes(), path.stat().st_mtime_ns) for path in paths] + assert project_runtime._refresh_one(project_config, skip=True)["status"] == "ok" + assert before == [(path.read_bytes(), path.stat().st_mtime_ns) for path in paths] + + +def test_skip_refresh_survives_a_fresh_scheduler_process(project_config, monkeypatch): + claim(project_config) + assert project_runtime._refresh_one(project_config)["status"] == "ok" + project_runtime._refresh_cache.clear() + for connection in project_runtime._connections.values(): + connection.close() + project_runtime._connections.clear() + monkeypatch.setattr(project_runtime, "active", lambda name: True) + monkeypatch.setattr(project_runtime, "load_project_configs", lambda: [project_config]) + def forbidden(*args, **kwargs): + raise AssertionError("unchanged SKIP performed full scan in fresh process") + monkeypatch.setattr(project_runtime, "_refresh_one", forbidden) + assert project_runtime.refresh_project_projections(skip=True)["status"] == "ok" + + +def test_projection_review_never_fabricates_real_m2_receipt(project_config, monkeypatch): + for number in range(1, 4): + claim(project_config, number) + assert publish_project_projection(project_config, build_project_projection(project_config)) + monkeypatch.setenv(project_runtime.PREFIX + "ENABLED", "1") + result = check_project_projection(ReviewConfig(db=project_config.db_path, + project_projection_configs=(project_config,))) + assert result.verdict == Verdict.WARN + assert result.counts["m2_passed"] == 0 + assert "real_m2_probe_pending" in result.detail + _, manifest_path = projection_paths(project_config) + manifest = json.loads(manifest_path.read_text()) + manifest["checked_at"] = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat() + manifest_path.write_text(json.dumps(manifest)) + assert read_project_projection(project_config) is None diff --git a/tests/test_project_runtime.py b/tests/test_project_runtime.py index d25570aa..a09503dd 100644 --- a/tests/test_project_runtime.py +++ b/tests/test_project_runtime.py @@ -54,13 +54,32 @@ def test_skip_refresh_prechecks_cheap_fingerprints_without_full_scan(tmp_path, m monkeypatch.setattr(runtime, "load_project_configs", lambda: [cfg]) monkeypatch.setattr(runtime, "cheap_fingerprint", lambda config: ("unchanged",)) calls = [] - monkeypatch.setattr(runtime, "_refresh_one", lambda config: calls.append(config) or {"status": "ok"}) + monkeypatch.setattr(runtime, "_refresh_one", lambda config, **kwargs: calls.append(config) or {"status": "ok"}) runtime._refresh_cache.clear() runtime.refresh_project_projections(skip=True) runtime.refresh_project_projections(skip=True) assert calls == [cfg] +def test_cheap_cache_respects_temporal_boundary(monkeypatch): + from datetime import datetime, timedelta, timezone + + cfg = SimpleNamespace(scope="project:fixture", context_key="fixture", epoch="test") + monkeypatch.setattr(runtime, "active", lambda name: True) + monkeypatch.setattr(runtime, "load_project_configs", lambda: [cfg]) + monkeypatch.setattr(runtime, "cheap_fingerprint", lambda config: ("unchanged",)) + calls = [] + def refresh(config, **kwargs): + calls.append(config) + return {"status": "ok", "next_temporal_boundary": + (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat()} + monkeypatch.setattr(runtime, "_refresh_one", refresh) + runtime._refresh_cache.clear() + runtime.refresh_project_projections(skip=True) + runtime.refresh_project_projections(skip=True) + assert calls == [cfg, cfg] + + def test_scope_disagreement_and_wrong_repository_denied(tmp_path, monkeypatch): from memorymaster.core import scope_utils @@ -69,3 +88,10 @@ def test_scope_disagreement_and_wrong_repository_denied(tmp_path, monkeypatch): monkeypatch.setattr(scope_utils, "scope_from_cwd_or_user", lambda cwd: "project:fixture") monkeypatch.setattr(scope_utils, "resolve_transcript_workspace_scope", lambda cwd: "project:wrong") assert runtime.resolve_projection_scope(cfg, tmp_path) is None + + +def test_enabled_unresolved_scope_never_requests_legacy_sql(monkeypatch): + monkeypatch.setattr(runtime, "active", lambda name: True) + monkeypatch.setattr(runtime, "config_for_cwd", lambda cwd: None) + result = runtime.session_start_context({"cwd": "unresolved"}) + assert result is not None and "unavailable" in result.lower() diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py index cf849a48..f7f5c3f3 100644 --- a/tests/test_project_sessions.py +++ b/tests/test_project_sessions.py @@ -118,3 +118,55 @@ def test_disabled_state_never_reads_or_writes(tmp_path): assert sessions.record_project_session_start(cfg, "S")["status"] == "inactive" assert not sessions.record_project_session_end(cfg, "S") assert not list(tmp_path.iterdir()) + + +def test_missing_projection_keeps_completed_baseline_comparison(tmp_path, monkeypatch): + from memorymaster.profile import project_repository + + cfg = config(tmp_path) + original = snapshot(**{"claim:1": claim("original")}) + projection = {"source_coverage": original["coverage"], "read_at": NOW.isoformat()} + sessions.record_project_session_start(cfg, "S", projection=projection, + snapshot_loader=lambda: original, now=NOW) + sessions.record_project_session_end(cfg, "S", now=NOW + timedelta(minutes=1)) + monkeypatch.setattr(project_repository, "snapshot_since", lambda config, coverage, read_at: + snapshot(**{"claim:1": claim("changed without projection", 2)})) + result = sessions.record_project_session_start(cfg, "T", now=NOW + timedelta(minutes=2)) + assert "changed without projection" in result["delta"].text + assert result["delta"].status == "partial" + + +def test_dotted_project_slug_has_bounded_state(tmp_path): + cfg = config(tmp_path) + cfg.scope = "project:my.repo" + assert sessions.session_state_path(cfg).parent.name == "my.repo" + + +def test_malformed_state_and_end_time_fail_closed(tmp_path): + cfg = config(tmp_path) + path = sessions.session_state_path(cfg) + path.parent.mkdir(parents=True) + for invalid in ([], None, {"schema": sessions.SCHEMA, "scope": cfg.scope, + "context_key": cfg.context_key, "activation_epoch": cfg.epoch, + "active": [], "completed": [], "baselines": {}, "seq": 0}): + path.write_text(json.dumps(invalid)) + assert sessions.record_project_session_start(cfg, "S")["status"] == "continuity_pending" + assert not sessions.record_project_session_end(cfg, "S", end_at=123) + + +def test_future_end_is_rejected_and_start_remains(tmp_path): + cfg = config(tmp_path) + current = snapshot() + sessions.record_project_session_start(cfg, "S", projection={"source_coverage": {}, "read_at": NOW.isoformat()}, + snapshot_loader=lambda: current, now=NOW) + assert not sessions.record_project_session_end(cfg, "S", end_at=(NOW + timedelta(seconds=30)).isoformat(), now=NOW) + assert sessions.last_project_session_end(cfg) is None + + +def test_internal_scan_fence_never_becomes_delta_support_or_omission(): + current = snapshot() + current["coverage"]["__scan_fence__"] = {"watched_ids": [42], "max_seen_id": 42} + delta = sessions.build_session_delta({}, {}, current) + assert delta.omitted == 0 + assert not delta.supports + assert "No changes" in delta.text From fffe1b7665ed65e8acffb729caee8f9e8ef512c3 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:04:06 -0300 Subject: [PATCH 08/51] fix(T-1703): fence incremental coverage and verify ledger snapshots --- memorymaster/profile/project_projection.py | 8 +- memorymaster/profile/project_repository.py | 81 +++++++++--- tests/test_project_projection.py | 1 + tests/test_project_repository.py | 140 ++++++++++++++++++++- 4 files changed, 211 insertions(+), 19 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index ae2da96e..d6ec5bec 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -14,14 +14,16 @@ import time from typing import Iterator +from memorymaster.core.generated_artifacts import ( + GENERATED_OPENER as OPENER, + GENERATED_CLOSER as CLOSER, +) from memorymaster.profile.project_repository import ( digest, load_project_snapshot, utc_now, ) -OPENER = "" -CLOSER = "" SCHEMA = "memorymaster.project-projection.v1" SECTIONS = ("Purpose", "Constraints", "Decisions", "Open cards", "Gotchas") MAX_MANIFEST_BYTES = 256 * 1024 @@ -414,7 +416,7 @@ def read_manifest(config: ProjectProjectionConfig) -> dict | None: if ( len(records) > 64 or len(json.dumps(manifest["source_coverage"]).encode()) > 128 * 1024 - or len(manifest["source_coverage"]) > 768 + or len(manifest["source_coverage"]) > 769 ): return None citations = 0 diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index 38f7eb84..cd58cec0 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -91,6 +91,20 @@ def _ledger_paths(config: Any) -> list[Path]: def load_ledger(config: Any, *, max_bytes: int = MAX_SOURCE_BYTES) -> tuple[dict, bool, list[str]]: if not config.ledger_root or not config.ledger_repo or not config.ledger_filtered: return {}, False, ["ledger_unavailable"] + consumed = 0 + + def read_bytes(path: Path) -> bytes: + nonlocal consumed + remaining = max_bytes - consumed + if remaining < 0: + raise ValueError("ledger_byte_limit") + with path.open("rb") as stream: + raw = stream.read(remaining + 1) + consumed += len(raw) + if consumed > max_bytes: + raise ValueError("ledger_byte_limit") + return raw + for attempt in range(2): try: paths = _ledger_paths(config) @@ -101,11 +115,10 @@ def load_ledger(config: Any, *, max_bytes: int = MAX_SOURCE_BYTES) -> tuple[dict raise ValueError("ledger_byte_limit") records = [] complete = True + content_hashes = [] for path in paths: - with path.open("rb") as stream: - raw = stream.read(max_bytes + 1) - if len(raw) > max_bytes: - raise ValueError("ledger_byte_limit") + raw = read_bytes(path) + content_hashes.append(hashlib.sha256(raw).digest()) data = json.loads(raw) if isinstance(data, dict) and "tasks" in data: complete = complete and data.get("complete") is True @@ -152,8 +165,9 @@ def load_ledger(config: Any, *, max_bytes: int = MAX_SOURCE_BYTES) -> tuple[dict } if len(cards) > 256: raise ValueError("ledger_card_limit") + after_hashes = [hashlib.sha256(read_bytes(path)).digest() for path in paths] after = [(path.stat().st_mtime_ns, path.stat().st_size) for path in paths] - if before != after or paths != _ledger_paths(config): + if before != after or content_hashes != after_hashes or paths != _ledger_paths(config): if attempt == 0: continue raise ValueError("ledger_changed_during_read") @@ -244,7 +258,12 @@ def _claim_item(row: dict, citations: list[dict]) -> dict | None: def _read_claims( - config: Any, *, ids: list[int] | None = None, changed_since: str | None = None, validation: bool = False + config: Any, + *, + ids: list[int] | None = None, + changed_since: str | None = None, + validation: bool = False, + max_seen_id: int | None = None, ) -> tuple[dict, bool, dict]: limit = 64 if validation else MAX_CLAIMS citation_limit = 256 if validation else MAX_CITATIONS @@ -268,7 +287,12 @@ def _read_claims( clauses.append("id IN (" + ",".join("?" for _ in ids) + ")") params.extend(ids) if changed_since: - # Inclusive changes catch old-candidate confirmation and equal timestamps. + # Store writes use whole UTC seconds; read_at retains microseconds. + # A bare second prefix includes every same-second timestamp suffix. + instant = datetime.fromisoformat(changed_since.replace("Z", "+00:00")) + if instant.tzinfo is None: + raise ValueError("unqualified_baseline_time") + lower_bound = instant.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") clauses.extend( [ "updated_at >= ?", @@ -276,14 +300,16 @@ def _read_claims( "EXISTS (SELECT 1 FROM citations c WHERE c.claim_id=claims.id AND c.created_at >= ?)", ] ) - params.extend([changed_since] * 3) + params.extend([lower_bound] * 3) + if max_seen_id is not None: + clauses.append("id > ?") + params.append(max_seen_id) if not clauses: return {}, True, {"data_version": conn.execute("PRAGMA data_version").fetchone()[0]} where += " AND (" + " OR ".join(clauses) + ")" - else: - where += ( - " AND status='confirmed' AND COALESCE(visibility,'public')='public' AND replaced_by_claim_id IS NULL" - ) + # Builder scans watch bounded all-status IDs. Ineligible rows can become + # eligible without a reliable post-commit timestamp; their IDs are only + # protected continuity metadata, never factual support or public output. fields = [row[1] for row in conn.execute("PRAGMA table_info(claims)")] names = [ name @@ -373,6 +399,15 @@ def _read_claims( "next_temporal_boundary": min(boundaries) if boundaries else None, "source_bytes": total, } + if not validation: + watched_ids = sorted(set(ids or ()) | {int(row["id"]) for row in rows}) + fingerprint["scan_fence"] = { + "kind": "scan_fence", + "watched_ids": watched_ids[:MAX_CLAIMS], + "max_seen_id": max([max_seen_id or 0, *watched_ids]), + "complete": complete and len(watched_ids) <= MAX_CLAIMS, + } + complete = complete and len(watched_ids) <= MAX_CLAIMS return items, complete, fingerprint @@ -402,11 +437,25 @@ def load_project_snapshot( return result try: ids = claim_ids + prior_fence = None if coverage is not None: - ids = [int(key.split(":", 1)[1]) for key in coverage if key.startswith("claim:")] + prior_fence = coverage.get("__scan_fence__") + ids = sorted( + set(int(key.split(":", 1)[1]) for key in coverage if key.startswith("claim:")) + | set(prior_fence.get("watched_ids", ()) if isinstance(prior_fence, dict) else ()) + ) + if any(type(id) is not int or id <= 0 for id in ids): + raise ValueError("invalid_scan_fence") items, complete, fingerprints = _read_claims( - config, ids=ids, changed_since=changed_since, validation=validation + config, + ids=ids, + changed_since=changed_since, + validation=validation, + max_seen_id=prior_fence.get("max_seen_id") if isinstance(prior_fence, dict) else None, ) + if coverage is not None and (not prior_fence or not prior_fence.get("complete")): + complete = False + result["diagnostics"].append("incremental_scan_fence_incomplete") except (sqlite3.Error, OSError, ValueError, TypeError): items, complete, fingerprints = {}, False, {} result["diagnostics"].append("claims_unavailable") @@ -418,6 +467,10 @@ def load_project_snapshot( result.update(items=items, complete=complete and ledger_complete, fingerprints=fingerprints) result["next_temporal_boundary"] = fingerprints.get("next_temporal_boundary") result["coverage"] = {key: item["support"] for key, item in items.items() if item.get("open", True)} + fence = fingerprints.pop("scan_fence", None) + if fence is not None: + fence["complete"] = fence["complete"] and complete + result["coverage"]["__scan_fence__"] = fence result["diagnostics"].extend(diagnostics) if len(json.dumps(result["coverage"]).encode()) > 128 * 1024: result.update(coverage={}, complete=False) diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 0dbc1b2a..25be900a 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -31,6 +31,7 @@ def test_exact_manifest_and_per_bullet_invalidation(project_config): assert before.manifest["view_id"] != after.manifest["view_id"] assert all(s["id"] != 2 for item in after.manifest["items"] for s in item["supports"]) assert "source_coverage" not in after.public_manifest + assert "__scan_fence__" not in json.dumps(after.public_manifest) def test_modified_still_open_cards_are_rerendered_without_trim_warning(project_config): diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index 24f8e6ec..18dc11b7 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -1,5 +1,6 @@ from dataclasses import replace import json +import os import sqlite3 import pytest @@ -110,7 +111,7 @@ def test_ledger_actual_schema_visibility_and_completeness(project_config): ) result = load_project_snapshot(config) assert set(result["items"]) == {"card:T-1", "card:T-4"} - assert set(result["coverage"]) == {"card:T-1"} + assert set(result["coverage"]) == {"card:T-1", "__scan_fence__"} path.write_text("{") result = load_project_snapshot(config) assert not result["complete"] @@ -126,7 +127,7 @@ def test_incremental_baseline_rehydrates_coverage_and_changed_rows(project_confi conn.execute("UPDATE claims SET status='confirmed',updated_at=? WHERE id=2", (baseline["read_at"],)) conn.execute("UPDATE citations SET excerpt='Edited evidence' WHERE claim_id=1") current = snapshot_since(config, baseline["coverage"], baseline["read_at"]) - assert set(current["coverage"]) == {"claim:1", "claim:2"} + assert set(current["coverage"]) == {"claim:1", "claim:2", "__scan_fence__"} assert current["coverage"]["claim:1"] != baseline["coverage"]["claim:1"] @@ -156,3 +157,138 @@ def test_incremental_includes_temporal_boundary_without_row_edit(project_config) snapshot = snapshot_since(project_config, {}, "2026-09-30T00:00:00+00:00") assert "claim:1" in snapshot["coverage"] assert "next_temporal_boundary" in snapshot + + +def test_incremental_same_second_promotions_and_new_rows_use_inclusive_floor(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + claim(project_config, status="candidate") + monkeypatch.setattr(repository, "utc_now", lambda: "2026-10-22T15:00:00.750000+00:00") + baseline = load_project_snapshot(project_config) + assert not any(key.startswith("claim:") for key in baseline["coverage"]) + # Store writes have whole-second UTC timestamps, while projection read_at + # retains microseconds. Both commits happen after the baseline read. + store_time = "2026-10-22T15:00:00+00:00" + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("UPDATE claims SET status='confirmed',updated_at=? WHERE id=1", (store_time,)) + claim(project_config, 2, created_at=store_time, updated_at=store_time) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert set(current["coverage"]) == {"claim:1", "claim:2", "__scan_fence__"} + + +@pytest.mark.parametrize("rewrite_count", [1, 2]) +def test_ledger_digest_detects_equal_stat_rewrites(project_config, monkeypatch, rewrite_count): + from memorymaster.profile import project_repository as repository + + path = project_config.ledger_root / "tasks.json" + + def encoded(title): + return json.dumps( + {"complete": True, "tasks": [{"id": "T-1", "repo": "demo", "state": "ready", "title": title}]} + ).encode() + + path.write_bytes(encoded("Before")) + original_stat = path.stat() + original_loads = json.loads + calls = [] + + def racing_loads(raw, *args, **kwargs): + result = original_loads(raw, *args, **kwargs) + if isinstance(result, dict) and "tasks" in result: + calls.append(True) + if len(calls) <= rewrite_count: + path.write_bytes(encoded("After!" if len(calls) == 1 else "Before")) + os.utime(path, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) + return result + + monkeypatch.setattr(repository.json, "loads", racing_loads) + current = load_project_snapshot(project_config) + assert len(calls) == 2 + if rewrite_count == 1: + assert current["complete"] + assert "After!" in current["items"]["card:T-1"]["text"] + else: + assert not current["complete"] + assert "card:T-1" not in current["items"] + + +def test_incremental_fence_finds_late_old_timestamp_promotions_and_inserts(project_config): + claim(project_config, status="candidate") + baseline = load_project_snapshot(project_config) + assert baseline["coverage"]["__scan_fence__"]["watched_ids"] == [1] + # These timestamps can precede read_at in a long transaction, despite the + # commits becoming visible only after the projection snapshot. + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("UPDATE claims SET status='confirmed' WHERE id=1") + claim(project_config, 2, updated_at="2001-01-01T00:00:00+00:00") + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert {"claim:1", "claim:2"} <= set(current["coverage"]) + assert current["complete"] + assert current["coverage"]["__scan_fence__"]["watched_ids"] == [1, 2] + + +def test_incremental_fence_finds_first_citation_without_claim_timestamp_edit(project_config): + claim(project_config) + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("DELETE FROM citations") + baseline = load_project_snapshot(project_config) + assert "claim:1" not in baseline["coverage"] + with sqlite3.connect(project_config.db_path) as conn: + conn.execute( + "INSERT INTO citations(claim_id,source,locator,excerpt) VALUES (1,'session','authored','evidence')" + ) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert "claim:1" in current["coverage"] + assert current["complete"] + + +def test_incomplete_scan_fence_never_claims_complete_incremental_coverage(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + monkeypatch.setattr(repository, "MAX_CLAIMS", 1) + claim(project_config, status="candidate") + claim(project_config, 2, status="candidate") + baseline = load_project_snapshot(project_config) + assert not baseline["complete"] + assert not baseline["coverage"]["__scan_fence__"]["complete"] + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert not current["complete"] + + +def test_incremental_fence_survives_real_wal_late_commit_without_full_rescan(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + claim(project_config, status="candidate") + writer = sqlite3.connect(project_config.db_path) + try: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("UPDATE claims SET status='confirmed',updated_at='2001-01-01T00:00:00+00:00' WHERE id=1") + writer.execute( + "INSERT INTO claims(id,text,scope,status,visibility,source_agent,claim_type,created_at,updated_at) " + "VALUES (2,'Late insert','project:demo','confirmed','public','operator','decision'," + "'2001-01-01T00:00:00+00:00','2001-01-01T00:00:00+00:00')" + ) + writer.execute( + "INSERT INTO citations(claim_id,source,locator,excerpt) VALUES (2,'session','authored','evidence')" + ) + baseline = load_project_snapshot(project_config) + assert baseline["coverage"]["__scan_fence__"]["watched_ids"] == [1] + assert "claim:1" not in baseline["coverage"] + writer.commit() + finally: + writer.close() + statements = [] + original_connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + conn = original_connect(*args, **kwargs) + conn.set_trace_callback(statements.append) + return conn + + monkeypatch.setattr(repository.sqlite3, "connect", traced_connect) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert {"claim:1", "claim:2"} <= set(current["coverage"]) + assert current["complete"] + row_reads = [statement for statement in statements if "FROM claims WHERE" in statement] + assert len(row_reads) == 1 + assert "id IN (1)" in row_reads[0] and "id > 1" in row_reads[0] From 25d5d61cebb9e5268976c5ca0efe5b08dc419519 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:04:43 -0300 Subject: [PATCH 09/51] feat(T-1703): accept explicit lifecycle receipts before CLI distillation --- memorymaster/surfaces/session_end_ingest.py | 8 ++++++++ tests/test_session_end_ingest.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/memorymaster/surfaces/session_end_ingest.py b/memorymaster/surfaces/session_end_ingest.py index b6f3669f..780f70a0 100644 --- a/memorymaster/surfaces/session_end_ingest.py +++ b/memorymaster/surfaces/session_end_ingest.py @@ -139,7 +139,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--transcript", required=True) parser.add_argument("--source-agent", default="codex-session") parser.add_argument("--cwd", default=None) + parser.add_argument("--session-id", help="explicit project lifecycle identity; requires --end-at") + parser.add_argument("--end-at", help="observed UTC end time; requires --session-id") args = parser.parse_args(argv) + if bool(args.session_id) != bool(args.end_at): + parser.error("--session-id and --end-at must be supplied together") + if args.session_id: + from memorymaster.profile.project_runtime import record_session_end + + record_session_end({"cwd": args.cwd, "session_id": args.session_id, "end_at": args.end_at}) try: count = run( args.db, diff --git a/tests/test_session_end_ingest.py b/tests/test_session_end_ingest.py index bf7a0075..d888c424 100644 --- a/tests/test_session_end_ingest.py +++ b/tests/test_session_end_ingest.py @@ -25,3 +25,19 @@ def test_generic_payload_model_turn_preserves_legacy_distillation(tmp_path): path = tmp_path / "transcript.jsonl" path.write_text(json.dumps({"payload": {"role": "model", "content": "A durable authored finding about SQLite transaction boundaries."}}), encoding="utf-8") assert "durable authored" in _assistant_text(str(path)) + + +def test_cli_records_explicit_end_before_distillation_failure(tmp_path, monkeypatch): + from memorymaster.profile import project_runtime + from memorymaster.surfaces import session_end_ingest as ingest + + seen = [] + monkeypatch.setattr(project_runtime, "record_session_end", lambda data: seen.append(data)) + monkeypatch.setattr(ingest, "run", lambda *args, **kwargs: (_ for _ in ()).throw(ValueError("fixture failure"))) + assert ingest.main(["--db", str(tmp_path / "fixture.db"), "--transcript", str(tmp_path / "missing.jsonl"), + "--cwd", str(tmp_path), "--session-id", "explicit", "--end-at", "2026-10-22T09:00:00Z"]) == 1 + assert seen == [{"cwd": str(tmp_path), "session_id": "explicit", "end_at": "2026-10-22T09:00:00Z"}] + seen.clear() + monkeypatch.setattr(ingest, "run", lambda *args, **kwargs: 0) + assert ingest.main(["--db", str(tmp_path / "fixture.db"), "--transcript", "unused"]) == 0 + assert not seen From 63e125f09c6dd39bac9735e218d17859bdb72e77 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:09:03 -0300 Subject: [PATCH 10/51] fix(T-1703): bound SQLite source materialization before allocation --- memorymaster/profile/project_repository.py | 57 +++++++++++------ tests/test_project_repository.py | 74 ++++++++++++++++++++++ 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index cd58cec0..5c3348a4 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -340,46 +340,65 @@ def _read_claims( # Reject oversized rows before materializing their payload in Python. size = "+".join(f"length(CAST(COALESCE({name},'') AS BLOB))" for name in names) columns = ",".join( - f"substr({name},1,{MAX_FIELD_BYTES + 1}) AS {name}" - if name not in ("id", "version", "pinned", "replaced_by_claim_id") - else name + f"CASE WHEN ({size}) <= {MAX_FIELD_BYTES} THEN {name} ELSE NULL END AS {name}" if name != "id" else name for name in names ) - rows = conn.execute( + cursor = conn.execute( f"SELECT {columns}, ({size}) AS _source_bytes FROM claims WHERE {where} ORDER BY id LIMIT ?", [*params, limit + 1], - ).fetchall() - complete = len(rows) <= limit - rows = rows[:limit] + ) + rows = [] + total = 0 + for row in cursor: + if len(rows) >= limit: + fingerprint = {"source_bytes": total} + if not validation: + watched = sorted(set(ids or ()) | {int(value["id"]) for value in rows}) + fingerprint["scan_fence"] = { + "kind": "scan_fence", + "watched_ids": watched[:MAX_CLAIMS], + "max_seen_id": max([max_seen_id or 0, *watched]), + "complete": False, + } + return {}, False, fingerprint + row_bytes = len(json.dumps(dict(row)).encode()) + if total + row_bytes > byte_limit: + # Exhaust the shared source allowance so ledger reads cannot + # restart that budget after an interrupted SQLite stream. + return {}, False, {"source_bytes": byte_limit} + total += row_bytes + rows.append(row) + complete = True oversized = {row["id"] for row in rows if row["_source_bytes"] > MAX_FIELD_BYTES} complete = complete and not oversized ids_read = [row["id"] for row in rows] citations: dict[int, list] = {} - raw_citations = [] if ids_read: placeholders = ",".join("?" for _ in ids_read) - raw_citations = conn.execute( + citation_size = "length(CAST(COALESCE(source,'')||COALESCE(locator,'')||COALESCE(excerpt,'') AS BLOB))" + citation_cursor = conn.execute( "SELECT id,claim_id," + ",".join( - f"substr({name},1,{MAX_FIELD_BYTES + 1}) AS {name}" for name in ("source", "locator", "excerpt") + f"CASE WHEN {citation_size} <= {MAX_FIELD_BYTES} THEN {name} ELSE NULL END AS {name}" + for name in ("source", "locator", "excerpt") ) - + ",length(CAST(COALESCE(source,'')||COALESCE(locator,'')||COALESCE(excerpt,'') AS BLOB)) AS _source_bytes " + + f",{citation_size} AS _source_bytes " f"FROM citations WHERE claim_id IN ({placeholders}) ORDER BY id LIMIT ?", [*ids_read, citation_limit + 1], - ).fetchall() - complete = complete and len(raw_citations) <= citation_limit - if len(raw_citations) > citation_limit: - return {}, False, {} - for citation in raw_citations[:citation_limit]: + ) + for index, citation in enumerate(citation_cursor): + citation_bytes = len(json.dumps(dict(citation)).encode()) + if total + citation_bytes > byte_limit: + return {}, False, {"source_bytes": byte_limit} + total += citation_bytes + if index >= citation_limit: + return {}, False, {"source_bytes": total} if citation["_source_bytes"] > MAX_FIELD_BYTES: oversized.add(citation["claim_id"]) complete = False value = dict(citation) value.pop("_source_bytes") citations.setdefault(citation["claim_id"], []).append(value) - total = sum(len(json.dumps(dict(row)).encode()) for row in [*rows, *raw_citations]) - if total > byte_limit: - return {}, False, {} items = {} for row in rows: if row["id"] in oversized: diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index 18dc11b7..20471e60 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -292,3 +292,77 @@ def traced_connect(*args, **kwargs): row_reads = [statement for statement in statements if "FROM claims WHERE" in statement] assert len(row_reads) == 1 assert "id IN (1)" in row_reads[0] and "id > 1" in row_reads[0] + + +@pytest.mark.parametrize("source", ["claims", "citations"]) +def test_source_cursor_stops_before_materializing_over_byte_budget(project_config, monkeypatch, source): + from memorymaster.profile import project_repository as repository + + claim(project_config, text="safe") + with sqlite3.connect(project_config.db_path) as conn: + if source == "claims": + for id in range(2, 102): + conn.execute( + "INSERT INTO claims(id,text,scope,status) VALUES (?,?,'project:demo','candidate')", + (id, "bounded " * 150), + ) + else: + conn.executemany( + "INSERT INTO citations(claim_id,source,locator,excerpt) VALUES (1,'session','authored',?)", + [("evidence " * 200,)] * 100, + ) + consumed = [] + + class TrackingCursor(sqlite3.Cursor): + def execute(self, query, parameters=()): + self.query = query + return super().execute(query, parameters) + + def fetchall(self): + rows = super().fetchall() + if f"FROM {source} WHERE" in self.query: + consumed.extend(rows) + return rows + + def __next__(self): + row = super().__next__() + if f"FROM {source} WHERE" in self.query: + consumed.append(row) + return row + + class TrackingConnection(sqlite3.Connection): + def execute(self, query, parameters=()): + return self.cursor(factory=TrackingCursor).execute(query, parameters) + + original_connect = sqlite3.connect + + def tracking_connect(*args, **kwargs): + return original_connect(*args, **kwargs, factory=TrackingConnection) + + monkeypatch.setattr(repository.sqlite3, "connect", tracking_connect) + monkeypatch.setattr(repository, "MAX_SOURCE_BYTES", 5000) + snapshot = load_project_snapshot(project_config) + assert not snapshot["complete"] + assert len(consumed) < 10 + + +def test_oversized_multibyte_rows_are_not_materialized_before_field_rejection(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + claim(project_config) + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("UPDATE claims SET text=?,subject=?,object_value=? WHERE id=1", ("😀" * 18000,) * 3) + observed = [] + original_connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + conn = original_connect(*args, **kwargs) + conn.set_trace_callback(observed.append) + return conn + + monkeypatch.setattr(repository.sqlite3, "connect", traced_connect) + snapshot = load_project_snapshot(project_config) + assert not snapshot["complete"] + assert snapshot["coverage"]["__scan_fence__"]["watched_ids"] == [1] + query = next(query for query in observed if "FROM claims WHERE" in query) + assert "CASE WHEN" in query and "ELSE NULL END AS text" in query From 03a27df7a95483d846030375a10d54fdc392d7b0 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:14:17 -0300 Subject: [PATCH 11/51] test(T-1703): verify bounded continuity and document gated rollback --- DOCS-MAP.md | 5 +- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- docs/project-projections.md | 171 ++++++++++++++++++++++++++++++ tests/test_project_integration.py | 6 +- tests/test_project_projection.py | 4 +- tests/test_project_sessions.py | 40 ++++++- 7 files changed, 219 insertions(+), 11 deletions(-) create mode 100644 docs/project-projections.md diff --git a/DOCS-MAP.md b/DOCS-MAP.md index 7f87d825..8716f6d7 100644 --- a/DOCS-MAP.md +++ b/DOCS-MAP.md @@ -1,9 +1,9 @@ - + # DOCS-MAP - memorymaster # Covers: trust verdicts and replacements for every canonical documentation surface. # Key terms: CURRENT, SUPERSEDED, workflow intelligence, compiled user profile, roadmap, ADR. # Read when: locating authoritative project documentation before reading doc bodies. -# Updated: 2026-09-28 for shared CLI navigation; earlier dated entries retain their recorded scope. +# Updated: 2026-10-22 for AO-3 source configuration; earlier entries retain their dated scope. # Rule: generated profiles are disposable projections; MemoryMaster remains authority. @@ -21,6 +21,7 @@ | .planning/UPSTREAM-COMPARISON-2026-09-05.md | CURRENT | 2026-09-05 | Seventeen pinned upstream snapshots, source comparisons and bounded proposals; no head-to-head runtime benchmark. | | .planning/UPSTREAM-SNAPSHOTS-2026-09-05.json | CURRENT | 2026-09-05 | Origins, baselines, exact SHAs, branches, license labels and explicit inspection coverage. | | docs/development.md | CURRENT | 2026-09-05 | Scoped verification, public lifecycle demonstration, and deeper agent/GitNexus guidance. | +| docs/project-projections.md | CURRENT | 2026-10-22 | Default-off AO-3 source configuration, continuity, budgets and rollback; installation and real M2 remain pending. | | .planning/DREAMING-SOURCE-STEWARD-V1.md | CURRENT | 2026-09-05 | Dated deployed v1 checkpoint; its receipt contract is superseded for the candidate by USEFUL-SELECTION.md, without retroactive runtime activation. | | .planning/ENGINEERING-CONSOLIDATION-2026-09-05.md | CURRENT | 2026-09-05 | Bounded implementation evidence and remaining measured gaps, subordinate to ROADMAP.md. | | CHANGELOG.md | CURRENT | 2026-09-05 | Historical releases plus local 4.8.7 deployment; distinguishes fleet reconnection and public publication. | diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 68f3f1b0..fc97c49b 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -144,7 +144,7 @@ "console_entrypoints": 9, "mcp_tools": 51, "ops_cli_commands": 5, - "pytest_test_functions": 5782 + "pytest_test_functions": 5853 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 42a7de62..5d2a5e00 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -13,7 +13,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **127** - Operations CLI commands: **5** - Console entrypoints: **9** -- Pytest source test functions: **5782** +- Pytest source test functions: **5853** ## MCP tools diff --git a/docs/project-projections.md b/docs/project-projections.md new file mode 100644 index 00000000..ad212897 --- /dev/null +++ b/docs/project-projections.md @@ -0,0 +1,171 @@ + + + + +# Project projections + +The deterministic project reader serves confirmed cited decisions, constraints, +purpose and gotchas, plus an authorized filtered ledger export. SQLite remains +authoritative. Cards and generated Markdown never become claim evidence. + +The implementation follows the complete AO-3 design in +`artifacts/inputs/2026-10-22-AO-3-PROJECTION-DESIGN-v3.md` and its round-two +review. It changes package source and hook templates only. Installation, +external Cockpit integration, activation and the independent real three-repository +M2 probe remain separate owner actions. Fixture success is not installed M2 +acceptance. Original operator notes and GitNexus symbols remain v1.1 work. + +## Configuration + +All five switches default off. A child switch cannot bypass the master gate: + +| Environment variable | Purpose | +|---|---| +| `MEMORYMASTER_PROJECT_PROJECTION_ENABLED` | Master build, serve and state gate | +| `MEMORYMASTER_PROJECT_PROJECTION_REFRESH` | Separate deterministic beat phase | +| `MEMORYMASTER_PROJECT_PROJECTION_SESSION_START` | Shared project hook delivery | +| `MEMORYMASTER_PROJECT_SESSION_DELTA` | Start baselines and explicit end receipts | +| `MEMORYMASTER_PROJECT_PROJECTION_COCKPIT` | Reserved external consumer activation | + +`MEMORYMASTER_PROJECT_PROJECTION_PROJECTS` defaults to an empty JSON list. +Each of at most three mappings supplies `scope`, `context_key`, `project_root`, +`db_path`, explicit `tenant_id` (JSON null means only the unset legacy tenant), +optional `aliases`, `ledger_repo` and `ledger_filtered: true`. The latter asserts +that the lane supplied an authorized visibility-filtered export. Blind/private +cards are still excluded. Unknown mapping or unavailable/malformed ledger data +cannot mean zero open cards. + +Approved absolute roots are supplied through +`MEMORYMASTER_PROJECT_PROJECTION_DIR`, `MEMORYMASTER_PROJECT_SESSION_STATE_DIR` +and `MEMORYMASTER_PROJECT_LEDGER_TASKS_DIR`. There are no home-directory output +defaults. `MEMORYMASTER_PROJECT_PROJECTION_EPOCH` is a required fresh activation +token; retired-epoch cache and continuity state are ignored. Context keys and +epochs are bounded opaque names, not paths. The repository owner, strict +transcript resolver and generic scope resolver must agree before hook delivery. + +Manual commands branch before service construction or migrations: + +```text +python -m memorymaster.profile project-build --help +python -m memorymaster.profile project-read --help +``` + +Both require explicit database, scope, tenant, context, epoch, repository, +projection and state roots. Use `--tenant legacy-null` only for NULL tenancy. +Ledger arguments identify the filtered input. The master flag is still required. +These commands do not enable automatic refresh or consumers. + +## Integrity and budgets + +`project-.md` and its JSON manifest live below the explicit context root. +One kernel lock serializes publishers, unique temporary files are flushed and +replaced, and readers verify the pair, generation, byte counts, item spans and +hashes. A torn pair is unavailable. Rehydration uses a read-only snapshot, exact +tenant/scope checks and bounded primary-key support reads. A changed claim +withholds only its dependent bullets. A changed still-open card is rendered +from its current authorized record, without a false trimmed-card warning. + +The complete projection file, including its marker pair, is at most 2,200 UTF-8 +bytes. The delta is at most 500 bytes including labels, references and counts. +SessionStart wraps the marker-free shared payload, delta and fitting complete +profile lines in one marker pair and checks the final 3,000-character ceiling. +It never slices a validated project payload. Cards and the latest three +decisions reserve space before optional prose. Mandatory omissions visibly +block M2 readiness. The public manifest excludes protected source coverage. + +Builder snapshots cap all watched project rows at 512, citations at 2,048, +cards at 256, configured ledger records at 1,024 and source bytes at 8 MiB. +The all-status watch cap is deliberately stricter than counting eligible +claims: candidates and unsupported rows may later become eligible. An exceeded +cap is partial. Reader support limits are 64 claims, 256 citations and 2 MiB +shared source bytes, with a 25 ms SQLite busy timeout and cooperative 100 ms +SQL deadline. OS filesystem stalls are not hard real-time bounded. + +## Continuity and cheap refresh + +Session baselines contain fingerprints and an internal bounded scan fence, +never source prose or raw session tokens. The fence watches existing source +IDs, including ineligible rows, and the greatest observed ID. Incremental starts +rehydrate those IDs plus inclusively changed rows and IDs above the fence. +This catches same-second writes, a previously unsupported claim gaining a +citation, and old-timestamp writes committed after the prior read. Store +AUTOINCREMENT and SQLite's single writer underpin the new-ID fence. Incomplete +or older fence-less metadata is partial, not a false no-change result. + +Delta selection unions differences from the selected prior start baseline and +the active projection's coverage. End events keep the start snapshot; they do +not acknowledge changes at end time. Missing projection coverage still permits +the prior-baseline comparison, labeled partial. After twelve hours an unclosed +active start can supply the newest stale baseline, explicitly labeled +`prior_session_unclosed`; inactivity never fabricates an end receipt. + +One atomic `sessions.json` per context/project retains at most sixteen active +starts, four completed starts and thirty-six deduplicated baseline blobs, each +at most 128 KiB, in a file capped at 8 MiB. Starts collect sources outside the +state lock and validate the sequence when reacquiring it. Duplicate starts +preserve their baseline. Replayed compacted ends, future ends, malformed state, +capacity exhaustion and lock/write failures cannot advance the latest-end +pointer. End recording precedes optional capture or distillation. Library +`run` never invents lifecycle receipts; the CLI requires both `--session-id` +and `--end-at` to record an explicit observed end. + +Refresh is separately gated and runs on enabled beat outcomes including SKIP, +capture-only and worker-busy. It checks SQLite data version, database/WAL and +ledger metadata before scanning; a bounded manifest hint survives scheduler +process restarts. Temporal boundaries defeat the cheap cache. An unchanged +SKIP does not rewrite Markdown or `checked_at`; successful non-SKIP validation +may update manifest freshness without rewriting Markdown. Changes rebuild. +Source failure or budget/lock refusal leaves the successful watermark intact. +Refresh rotates at most three pilot contexts under a cooperative 750 ms tick +budget. A validation older than sixty minutes is withheld pending refresh. + +## Activation and rollback + +Activation requires the lane's Argon source review and specific installation +and consumer-change authority. Install package/templates with every switch off, +configure approved contexts/roots and a new epoch, inspect manual builds first, +then separately authorize consumers, real M2 and automatic refresh. Existing +capture, Dreaming application and user-profile approvals remain independent. +The operational check reports cache/readiness and bounded receipt health. It +cannot substitute a healthy cache for a real independent installed M2 receipt. + +For rollback: + +1. Remove the affected allowlist mapping (or all pilot mappings). Set refresh, + SessionStart, delta and Cockpit switches to `0`, then set the master to `0` + in each affected environment. Record the retired epoch and sanitized reason + on the existing build card. +2. Reload or restart only affected long-lived consumers/jobs as separately + authorized. Fresh hooks read the changed environment on their next call. + Start a fresh conversation to verify rollback; previously injected context + cannot be retracted from an active conversation. +3. Verify the fresh hook uses its legacy branch, external Cockpit hides its + generated section, beat does no projection source/cache work and SessionEnd + writes no project continuity state. Disabled gates return before cache or + source reads. Do not disable unrelated capture/profile/beat features. +4. Retain derived files for inspection. Do not delete claims, restore a database + or ingest generated output. Keep the shipped span guard enforced. A later + activation uses a new epoch and fresh baselines. + +## Local verification record + +Red-first checks demonstrated missing modules and the new behavioral failures +before implementation: raw generated-span intake/dedup and HTML capture; +full-turn transcript filtering; hook lifecycle ordering; per-bullet card edits; +reserved UTF-8 budgets; malformed/out-of-order session state; missing projection +continuity; unresolved scope refusal; scheduler restart hints; same-second and +late committed writes; first citations; and equal-stat ledger rewrites. + +The explicit ten-thousand-session stress test passed with bounded deduplicated +state. A fixture M2 exercise uses three disposable repositories and template +hooks with the shared reader, with no independent SQL/model fallback. External +Cockpit and the real installed probe are not represented as completed. + +Windows sandbox directory creation required a test-launcher-only wrapper that +uses mode `0777` for disposable pytest directories. Production code and shared +pytest fixtures were not changed for that issue. The first broad service +coverage attempt was interrupted during provider initialization; affected +ingest cases passed with explicit hash embeddings. The first non-ML attempt +was interrupted before failures to give the ten-thousand-session stress test +its explicit 180-second timeout; it had no test failures at interruption. +Final suite counts and commands are recorded in the task close report. diff --git a/tests/test_project_integration.py b/tests/test_project_integration.py index ba1cdc55..b123c620 100644 --- a/tests/test_project_integration.py +++ b/tests/test_project_integration.py @@ -10,9 +10,7 @@ import sqlite3 import sys -from test_project_repository import claim - -pytest_plugins = ["test_project_repository"] +from test_project_repository import claim, project_config as _project_config from memorymaster.operations.operational_review import ReviewConfig, Verdict, check_project_projection from memorymaster.profile import project_repository, project_runtime @@ -21,6 +19,8 @@ ) from memorymaster.profile.project_sessions import record_project_session_start +project_config = _project_config + def _hook(name): path = Path(__file__).resolve().parents[1] / "memorymaster/config_templates/hooks" / name diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 25be900a..34ec06ec 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -2,7 +2,7 @@ import json import sqlite3 -from test_project_repository import claim +from test_project_repository import claim, project_config as _project_config from memorymaster.profile.project_projection import ( @@ -13,7 +13,7 @@ project_lock, ) -pytest_plugins = ["test_project_repository"] +project_config = _project_config def test_exact_manifest_and_per_bullet_invalidation(project_config): diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py index f7f5c3f3..ef8d30df 100644 --- a/tests/test_project_sessions.py +++ b/tests/test_project_sessions.py @@ -4,6 +4,8 @@ from datetime import datetime, timedelta, timezone from types import SimpleNamespace +import pytest + from memorymaster.profile import project_sessions as sessions @@ -46,11 +48,12 @@ def test_session_gap_no_observable_change_lost(tmp_path): assert "Changed decision" in delta.text +@pytest.mark.timeout(180) def test_session_end_state_is_atomic_bounded_idempotent(tmp_path): cfg = config(tmp_path) current = snapshot(**{"claim:1": claim("decision")}) projection = {"source_coverage": current["coverage"], "read_at": NOW.isoformat()} - for index in range(100): + for index in range(10_000): now = NOW + timedelta(minutes=index) sessions.record_project_session_start(cfg, str(index), projection=projection, snapshot_loader=lambda: current, now=now) @@ -63,7 +66,7 @@ def test_session_end_state_is_atomic_bounded_idempotent(tmp_path): previous = path.read_bytes() assert not sessions.record_project_session_end(cfg, "0", now=NOW + timedelta(days=2)) assert path.read_bytes() == previous - assert "99" not in state["latest_end"]["session"] + assert state["latest_end"]["session"] != "9999" def test_duplicate_start_does_not_reset_baseline(tmp_path): @@ -170,3 +173,36 @@ def test_internal_scan_fence_never_becomes_delta_support_or_omission(): assert delta.omitted == 0 assert not delta.supports assert "No changes" in delta.text + + +def test_out_of_order_ends_capacity_epoch_and_interrupted_replace(tmp_path, monkeypatch): + from memorymaster.profile import project_projection + + cfg = config(tmp_path) + current = snapshot() + projection = {"source_coverage": {}, "read_at": NOW.isoformat()} + def start(sid, minute=0): + return sessions.record_project_session_start(cfg, sid, projection=projection, + snapshot_loader=lambda: current, now=NOW + timedelta(minutes=minute)) + start("old") + start("latest", 1) + assert sessions.record_project_session_end(cfg, "latest", now=NOW + timedelta(minutes=5)) + assert sessions.record_project_session_end(cfg, "old", now=NOW + timedelta(minutes=3)) + latest = sessions.last_project_session_end(cfg) + assert latest["end_at"] == (NOW + timedelta(minutes=5)).isoformat() + for index in range(16): + assert start(f"active-{index}", 6)["status"] == "ok" + assert start("overflow", 6)["status"] == "active_capacity" + assert sessions.last_project_session_end(cfg) == latest + path = sessions.session_state_path(cfg) + before = path.read_bytes() + with project_projection.project_lock(path.with_suffix(".lock")) as acquired: + assert acquired + assert start("locked", 6)["status"] == "continuity_pending" + def fail_replace(*args): + raise OSError("interrupted replace") + monkeypatch.setattr(project_projection, "atomic_json", fail_replace) + assert not sessions.record_project_session_end(cfg, "active-0", now=NOW + timedelta(minutes=7)) + assert path.read_bytes() == before + cfg.epoch = "new-activation" + assert sessions.last_project_session_end(cfg) is None From 5e3f5fc2a22f1531e9d47ef259ccce0d64c1479e Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:21:34 -0300 Subject: [PATCH 12/51] fix(T-1703): reject generated marker variants in projection evidence --- memorymaster/profile/project_repository.py | 11 +++++++++++ tests/test_project_repository.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index 5c3348a4..04f0890e 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -13,6 +13,7 @@ from types import SimpleNamespace from typing import Any +from memorymaster.core.generated_artifacts import contains_generated_marker from memorymaster.core.security import scan_persisted_value from memorymaster.core.temporal_policy import claim_is_temporally_current from memorymaster.profile.egress import prepare_claim_egress @@ -38,10 +39,20 @@ def utc_now() -> str: def _safe(value: Any) -> bool: + def marked(raw: Any) -> bool: + if isinstance(raw, str): + return contains_generated_marker(raw) + if isinstance(raw, dict): + return any(marked(key) or marked(item) for key, item in raw.items()) + if isinstance(raw, (list, tuple)): + return any(marked(item) for item in raw) + return False + text = json.dumps(value, ensure_ascii=False) lowered = text.lower() return ( len(text.encode()) <= MAX_FIELD_BYTES + and not marked(value) and not any(marker in lowered for marker in ("generated by memorymaster", "[redacted", "[erased_claim_text]")) and not scan_persisted_value(value) and not prepare_claim_egress(text).blocked diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index 20471e60..5993df93 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -9,6 +9,17 @@ from memorymaster.profile.project_repository import load_project_snapshot, snapshot_since +@pytest.mark.parametrize("field", ["text", "subject", "citation"]) +@pytest.mark.parametrize("gap", [" ", "\n"]) +def test_generated_marker_variants_never_qualify(project_config, field, gap): + value = f"generated" + claim(project_config, **({field: value} if field != "citation" else {})) + if field == "citation": + with sqlite3.connect(project_config.db_path) as connection: + connection.execute("UPDATE citations SET excerpt=?", (value,)) + assert "claim:1" not in load_project_snapshot(project_config)["items"] + + @pytest.fixture def project_config(tmp_path): root = tmp_path / "demo" From 99d11bfd2e3a34543d39b4e61fbe126c90e7a85a Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:21:39 -0300 Subject: [PATCH 13/51] fix(T-1703): preserve reserved cards through multibyte title edits --- memorymaster/profile/project_projection.py | 28 +++++++++++++++++++++- tests/test_project_projection.py | 19 +++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index d6ec5bec..678e4b02 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -174,6 +174,24 @@ def _line(item: dict) -> str: return f"- {item['text']} [{reference}]" +def _fit_card_text(item: dict, byte_budget: int) -> str: + """Keep the current ID/state slot; titles consume only its remaining UTF-8 bytes.""" + text = item["text"] + if len(text.encode("utf-8")) <= byte_budget: + return text + marker = f"[{item['support']['state']}]" + end = text.find(marker) + if end < 0: + raise ValueError("invalid_card_rendering") + prefix = text[:end + len(marker)] + available = byte_budget - len(prefix.encode("utf-8")) + if available < 5: + return prefix + title = text[end + len(marker):].strip() + retained = title.encode("utf-8")[:available - 4].decode("utf-8", "ignore") + return prefix + " " + retained + "…" + + def render_project_projection( config: ProjectProjectionConfig, items: list[dict], @@ -261,7 +279,9 @@ def size(payload: str) -> int: for item in selected: text = item["text"] minimum = 48 if item["section"] == "Open cards" else 80 - if len(text) > minimum: + if item["section"] == "Open cards": + item["text"] = _fit_card_text(item, minimum) + elif len(text) > minimum: item["text"] = text[: minimum - 1] + "…" compacted = True continue @@ -482,6 +502,7 @@ def read_project_projection(config: ProjectProjectionConfig) -> ProjectProjectio selected = [] diagnostics = list(manifest["diagnostics"]) omissions = dict(manifest["omissions"]) + card_spare_bytes = max(0, config.budget_bytes - manifest["output_bytes"]) for record in manifest["items"]: item = current["items"].get(record["key"]) if record["supports"][0]["kind"] == "card": @@ -490,6 +511,11 @@ def read_project_projection(config: ProjectProjectionConfig) -> ProjectProjectio if item and item.get("open"): if item["support"] == record["supports"][0]: item = {**item, "text": record["text"]} + else: + # A title edit cannot take space reserved for an unrelated card. + reserved = len(record["text"].encode("utf-8")) + item = {**item, "text": _fit_card_text(item, reserved + card_spare_bytes)} + card_spare_bytes += reserved - len(item["text"].encode("utf-8")) selected.append(item) elif "m2_not_ready:cards_changed" not in diagnostics: diagnostics.append("m2_not_ready:cards_changed") diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 34ec06ec..b2444106 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -159,3 +159,22 @@ def test_manifest_generation_and_span_tampering_are_rejected(project_config): manifest["items"][0]["span"] = [0, 1] path.write_text(json.dumps(manifest)) assert read_project_projection(project_config) is None + + +def test_modified_multibyte_card_title_preserves_all_reserved_slots(project_config): + path = project_config.ledger_root / "tasks.json" + tasks = [{"id": f"T-{index}-aaaaa", "repo": "demo", "state": "ready", "title": "X"} + for index in range(51)] + path.write_text(json.dumps({"complete": True, "tasks": tasks})) + built = build_project_projection(project_config) + assert len(built.items) == 51 + assert publish_project_projection(project_config, built) + original = {record["key"]: record["text"] for record in built.manifest["items"]} + tasks[0]["title"] = "😀" * 80 + path.write_text(json.dumps({"complete": True, "tasks": tasks})) + result = read_project_projection(project_config) + assert len(result.items) == 51 + assert "open_cards_trimmed" not in result.payload + assert len(result.markdown.encode("utf-8")) <= 2200 + assert all(record["text"] == original[record["key"]] for record in result.manifest["items"] + if record["key"] != "card:T-0-aaaaa") From d7de358b908dba61ba1bf4e1187299d5aea05176 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:26:06 -0300 Subject: [PATCH 14/51] fix(T-1703): retain recent completed baselines during stale overlaps --- memorymaster/profile/project_sessions.py | 4 +++- tests/test_project_sessions.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py index 6f1c54ee..431f2cfb 100644 --- a/memorymaster/profile/project_sessions.py +++ b/memorymaster/profile/project_sessions.py @@ -180,7 +180,9 @@ def _select_prior(state: dict, now: datetime) -> tuple[dict | None, str]: stale = [record for record in state["active"].values() if now - _time(record["start_at"]) >= ACTIVE_TTL] fallback = max(stale, key=lambda record: record["start_at"], default=None) - if fallback and (latest is None or fallback["start_at"] > latest["start_at"]): + if fallback and (latest is None or ( + now - _time(latest["end_at"]) >= ACTIVE_TTL + and _time(fallback["start_at"]) > _time(latest["start_at"]))): return fallback, "prior_session_unclosed" return latest, "" diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py index ef8d30df..424546ec 100644 --- a/tests/test_project_sessions.py +++ b/tests/test_project_sessions.py @@ -206,3 +206,20 @@ def fail_replace(*args): assert path.read_bytes() == before cfg.epoch = "new-activation" assert sessions.last_project_session_end(cfg) is None + + +def test_stale_active_start_cannot_override_a_recent_explicit_end(tmp_path): + cfg = config(tmp_path) + old = snapshot(**{"claim:1": claim("before overlapping work")}) + changed = snapshot(**{"claim:1": claim("retained overlapping work", 2)}) + projection = {"source_coverage": old["coverage"], "read_at": NOW.isoformat()} + sessions.record_project_session_start(cfg, "completed-later", projection=projection, + snapshot_loader=lambda: old, now=NOW) + sessions.record_project_session_start(cfg, "crashed-overlap", projection=projection, + snapshot_loader=lambda: changed, now=NOW + timedelta(hours=6)) + sessions.record_project_session_end(cfg, "completed-later", now=NOW + timedelta(hours=20)) + result = sessions.record_project_session_start(cfg, "next", projection={ + "source_coverage": changed["coverage"], "read_at": changed["read_at"]}, + snapshot_loader=lambda: changed, now=NOW + timedelta(hours=20, minutes=1)) + assert "retained overlapping work" in result["delta"].text + assert "prior_session_unclosed" not in result["delta"].text From 12d6fdbdcb106573a7257ee981f90bc44f8bed31 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 15:49:51 -0300 Subject: [PATCH 15/51] chore(T-1703): refresh test inventory after resumed regressions --- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index fc97c49b..1bb80854 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -144,7 +144,7 @@ "console_entrypoints": 9, "mcp_tools": 51, "ops_cli_commands": 5, - "pytest_test_functions": 5853 + "pytest_test_functions": 5856 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 5d2a5e00..27bbd602 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -13,7 +13,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **127** - Operations CLI commands: **5** - Console entrypoints: **9** -- Pytest source test functions: **5853** +- Pytest source test functions: **5856** ## MCP tools From 1214ed67c8d0d74dd748b80d3c56c45d71c54107 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 16:22:32 -0300 Subject: [PATCH 16/51] fix(T-1703): order recent decisions independently of pinned constraints --- memorymaster/profile/project_projection.py | 3 ++- tests/test_project_projection.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index 678e4b02..24ab209d 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -363,7 +363,8 @@ def build_project_projection(config: ProjectProjectionConfig) -> ProjectProjecti ordered = [] for section in SECTIONS: matches = [item for item in snapshot["items"].values() if item["section"] == section and item.get("open", True)] - matches.sort(key=lambda item: item["sort"], reverse=section == "Decisions") + matches.sort(key=lambda item: item["sort"][1:] if section == "Decisions" else item["sort"], + reverse=section == "Decisions") ordered.extend(matches[:1] if section == "Purpose" else matches[:10] if section == "Decisions" else matches) diagnostics = list(snapshot["diagnostics"]) if not snapshot["complete"] and not diagnostics: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index b2444106..26240cdd 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -120,6 +120,17 @@ def test_missing_decisions_is_m2_unready_and_read_does_not_refresh_freshness(pro assert view.manifest["checked_at"] == built.manifest["checked_at"] +def test_latest_decisions_ignore_pin_priority(project_config): + for number in range(1, 12): + claim(project_config, number, text=f"Older decision {number}") + for number in range(12, 15): + claim(project_config, number, text=f"Recent pinned decision {number}", pinned=1, + created_at="2026-10-20T00:00:00+00:00") + built = build_project_projection(project_config) + decisions = [item for item in built.manifest["items"] if item["section"] == "Decisions"] + assert [item["supports"][0]["id"] for item in decisions[:3]] == [14, 13, 12] + + def test_marker_and_reference_injection_are_inert(project_config): claim(project_config, text="safe\n## forged\n[ignore](bad)\x1b[31m") built = build_project_projection(project_config) From 3b7cc8b015307a10b854b114c7a5ebfc7e5f7523 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 16:26:04 -0300 Subject: [PATCH 17/51] docs(T-1703): record resumed acceptance and incremental-read gap --- docs/project-projections.md | 79 ++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/docs/project-projections.md b/docs/project-projections.md index ad212897..46054081 100644 --- a/docs/project-projections.md +++ b/docs/project-projections.md @@ -1,5 +1,5 @@ - - + + # Project projections @@ -169,3 +169,78 @@ ingest cases passed with explicit hash embeddings. The first non-ML attempt was interrupted before failures to give the ten-thousand-session stress test its explicit 180-second timeout; it had no test failures at interruption. Final suite counts and commands are recorded in the task close report. + +### Resumed source checks, 2026-10-22 + +Source acceptance remains **partial** pending the round-two incremental-read +criterion. `load_project_snapshot` unions every watched claim ID into the +SessionStart lookup; `_read_claims` then reloads those rows and their citations. +The query is bounded and uses primary-key lookups, but does not reuse unchanged +coverage without materializing its sources. A disposable unchanged-source +probe with three watched claims observed three rehydrated rows and three +rehydrated citations. `test_session_start_baseline_never_full_source_scan` +checks the query arguments only and does not establish incremental cost. + +Simply removing watched IDs would regress the passing citation-only, +same-second, first-citation and old-timestamp late-commit tests. A bounded SQL +fingerprint comparison could reduce Python materialization while still reading +all watched source fingerprints. Strict change-only reads need a trustworthy +mutation sequence or another approved design. The lane's Argon review must +resolve that difference before claiming the incremental criterion or activating +the pilot; no journal, migration or activation was added in this resume. + +The resume also found and fixed pinned decisions incorrectly sorting behind +older unpinned decisions. The new +`test_latest_decisions_ignore_pin_priority` first failed with observed IDs +`[11, 10, 9]` instead of `[14, 13, 12]`; the complete projection file then passed +all twelve tests. Pins still order constraints, while decisions use date and ID. + +Freshly executed checks, one test file per Python process: + +| Test file (`tests/`) | Passed | Skipped | +|---|---:|---:| +| `test_project_projection.py` | 12 | 0 | +| `test_project_repository.py` | 22 | 0 | +| `test_project_sessions.py` | 13 | 0 | +| `test_project_runtime.py` | 8 | 0 | +| `test_project_integration.py` | 5 | 0 | +| `test_generated_artifacts.py` | 3 | 0 | +| `test_compiled_profile_surfaces.py` | 6 | 0 | +| `test_profile_config_defaults.py` | 2 | 0 | +| `test_compiled_profile.py` | 9 | 0 | +| `test_compiled_profile_claim_supports.py` | 50 | 0 | +| `test_intake_policy.py` | 22 | 0 | +| `test_service.py` | 6 | 0 | +| `test_capture_adapters.py` | 24 | 1 | +| `test_public_v1.py` | 14 | 0 | +| `test_dreaming_self_observation.py` | 35 | 0 | +| `test_session_end_capture_hook.py` | 2 | 0 | +| `test_session_end_ingest.py` | 4 | 0 | +| `test_scheduled_task_runtime.py` | 7 | 0 | +| `test_operational_review.py` | 16 | 0 | +| `test_public_demo.py` | 2 | 0 | +| `test_scope_utils.py` | 17 | 0 | +| **Total** | **279** | **1** | + +Runs used `WEZBRIDGE_OPERATOR_ALLOW_HEAVY=1`, disposable workspace basetemps +and the local directory-mode wrapper described above. Service-backed suites +used `MEMORYMASTER_EMBEDDING_PROVIDER=hash`; default temporary-directory roots +were redirected into the workspace. The sole skip requires Windows symlink +privilege. The ten-thousand-session case passed in the complete session file +(125 seconds). No full-suite or installed/live acceptance is claimed. + +The first projection attempt had eleven fixture setup errors because its +basetemp parent did not exist; the next exposed the sandbox's `0700` directory +access failure. Creating the parent and using the disposable-only wrapper +resolved those harness errors. The first standalone incremental probe imported +the installed package because its script path omitted the checkout; explicitly +prepending the checkout resolved that harness error before the observation +above. These were retained as failed attempts, not product regressions. + +Ruff passed on all thirty-three changed Python files. Release truth was +regenerated for the new regression and `--check` passed. Contract witness +passed against `origin/main` (seventeen code files, sixteen test files, no +orphan witnesses). GitNexus reported `INDEX_MISSING`; deferred tool discovery +exposed neither GitNexus nor Serena. Scoped source references and the reviewed +diff covered the builder's CLI/refresh callers before committing. Branch +freshness passed with zero commits behind `origin/main`. From 7403eb8b2d0d36fe7d0b2ba594ab209c65a082e4 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 16:49:14 -0300 Subject: [PATCH 18/51] fix(profile): make SessionStart collection incremental (T-1703) --- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- docs/project-projections.md | 78 +++- memorymaster/profile/project_repository.py | 118 ++++-- memorymaster/profile/project_sessions.py | 54 ++- .../migrations/0029_project_source_cursor.py | 115 ++++++ memorymaster/stores/storage.py | 8 + tests/test_project_projection.py | 102 +++++ tests/test_project_repository.py | 84 +++- tests/test_project_sessions.py | 17 + tests/test_project_source_cursor.py | 375 ++++++++++++++++++ 11 files changed, 892 insertions(+), 63 deletions(-) create mode 100644 memorymaster/stores/migrations/0029_project_source_cursor.py create mode 100644 tests/test_project_source_cursor.py diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 1bb80854..dad20cf5 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -144,7 +144,7 @@ "console_entrypoints": 9, "mcp_tools": 51, "ops_cli_commands": 5, - "pytest_test_functions": 5856 + "pytest_test_functions": 5874 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 27bbd602..15cd0a78 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -13,7 +13,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **127** - Operations CLI commands: **5** - Console entrypoints: **9** -- Pytest source test functions: **5856** +- Pytest source test functions: **5874** ## MCP tools diff --git a/docs/project-projections.md b/docs/project-projections.md index 46054081..ee81467f 100644 --- a/docs/project-projections.md +++ b/docs/project-projections.md @@ -1,5 +1,5 @@ - - + + # Project projections @@ -172,7 +172,7 @@ Final suite counts and commands are recorded in the task close report. ### Resumed source checks, 2026-10-22 -Source acceptance remains **partial** pending the round-two incremental-read +At the end of the previous resume, source acceptance was **partial** pending the round-two incremental-read criterion. `load_project_snapshot` unions every watched claim ID into the SessionStart lookup; `_read_claims` then reloads those rows and their citations. The query is bounded and uses primary-key lookups, but does not reuse unchanged @@ -244,3 +244,75 @@ orphan witnesses). GitNexus reported `INDEX_MISSING`; deferred tool discovery exposed neither GitNexus nor Serena. Scoped source references and the reviewed diff covered the builder's CLI/refresh callers before committing. Branch freshness passed with zero commits behind `origin/main`. + +## Round-three incremental collection (T-1703) + +The incremental source criterion now passes locally. The first red run of +`test_session_start_incremental_counts_only_changed_rows_and_citations` +failed all three cases: forty known claims caused forty claim and citation +loads even with zero, one or three changes. The corrected cases load exactly +zero, one or three claim rows and their citations, including overlapping claim +and citation edits, while retaining unchanged support hashes. Unchanged rows +from the same second are also excluded. SQL query plans assert primary-key +payload lookups and indexed citation lookups without scanning the source tables. + +The previous timestamp-only gap demonstrated the need for versioned migration +29: a singleton epoch/sequence clock and one latest-change row per claim, +including deletion tombstones. Transactional triggers track claim and citation +inserts, updates, deletes and identity moves without storing source prose. +This is bounded per claim identity rather than an append-only event history. +Indexed sequence changes are unioned with indexed inclusive `updated_at` and +temporal-boundary ranges and new IDs, then deduplicated before loading payloads. +The same read-only SQLite transaction captures the clock, selected IDs, claims +and citations. Coverage carries unchanged manifest support hashes and removes +changed identities before adding currently eligible supports. + +The source-load cost is O(N + R): N unique changed claims and their citations, +plus R supports needed for prior-session delta prose when a refreshed +projection already includes those changes. Lazy delta loading stops at the +500-byte output limit or the first unavailable support, with at most one +additional candidate. A forty-claim refreshed-projection regression counts +those loads against the rendered delta supports. The shared projection keeps +its 2,200-byte cap and delta entries retain exact support hashes and spans. +Coverage and bounded session-state serialization still operate on compact +metadata, and configured ledger reads retain their existing bounds. + +Migration 29 is SQLite-only and PostgreSQL fails closed. Writable store +initialization installs it, including convergence after a standalone migration +runner stamped an empty database. SessionStart never initializes or migrates +the store. Missing, foreign or rewound cursors withhold the baseline without +falling back to a source scan; old projections require a fresh generation after +schema initialization. The migration and initialization checks used disposable +databases only, including backup/restore and late WAL commits. No authoritative +database or installed service was changed; activation and independent Argon/M2 +acceptance remain separate lane actions. + +Retained failed evidence: the first repository integration run had two failures +from tests that assumed an empty coverage could bootstrap an incremental read +and that payload SQL contained the old watched-ID union. Those witnesses now +check temporal activation/expiration with known coverage and the indexed +selection followed by a deduplicated payload lookup. Cursor tests first exposed +an untracked old primary key on identity moves and a bare-runner initialization +gap; both received failing regressions before their fixes. + +Fresh round-three verification used one test file per Python process and the +same workspace-only temporary-directory launcher as the previous resume: + +| Test file (`tests/`) | Passed | +|---|---:| +| `test_project_projection.py` | 16 | +| `test_project_repository.py` | 31 | +| `test_project_sessions.py` | 14 | +| `test_project_runtime.py` | 8 | +| `test_project_integration.py` | 5 | +| `test_project_source_cursor.py` | 13 | +| `test_public_demo.py` | 2 | +| `test_migrations.py` | 18 | +| `test_initdb_fastpath.py` | 6 | +| **Total** | **113** | + +Ruff passed on all eight changed Python files. Release truth was regenerated +and `--check` passed. Contract witness passed against `origin/main`; source +navigation and final change-scope review used live references and Git diffs +because this checkout still has no GitNexus index or exposed navigation tools. +No push, merge, installed-hook edit or live database check belongs to this round. diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index 04f0890e..4a958e62 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -275,6 +275,7 @@ def _read_claims( changed_since: str | None = None, validation: bool = False, max_seen_id: int | None = None, + prior_fence: dict | None = None, ) -> tuple[dict, bool, dict]: limit = 64 if validation else MAX_CLAIMS citation_limit = 256 if validation else MAX_CITATIONS @@ -289,35 +290,67 @@ def _read_claims( conn.execute("PRAGMA query_only=ON") conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000) conn.execute("BEGIN") + try: + clock = conn.execute("SELECT epoch,seq FROM project_source_clock WHERE singleton=1").fetchone() + source_cursor = {"epoch": clock[0], "seq": clock[1]} if clock else None + except sqlite3.OperationalError: + source_cursor = None scopes = (config.scope, *config.aliases) params: list[Any] = [config.tenant_id, *scopes] where = "tenant_id IS ? AND scope IN (" + ",".join("?" for _ in scopes) + ")" - if ids is not None: - clauses = [] - if ids: - clauses.append("id IN (" + ",".join("?" for _ in ids) + ")") - params.extend(ids) - if changed_since: - # Store writes use whole UTC seconds; read_at retains microseconds. - # A bare second prefix includes every same-second timestamp suffix. - instant = datetime.fromisoformat(changed_since.replace("Z", "+00:00")) - if instant.tzinfo is None: - raise ValueError("unqualified_baseline_time") - lower_bound = instant.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") - clauses.extend( - [ - "updated_at >= ?", - "valid_from >= ?", - "EXISTS (SELECT 1 FROM citations c WHERE c.claim_id=claims.id AND c.created_at >= ?)", - ] + changed_ids = [] + if changed_since: + cursor = (prior_fence or {}).get("source_cursor") + if (not source_cursor or not isinstance(cursor, dict) + or cursor.get("epoch") != source_cursor["epoch"] + or type(cursor.get("seq")) is not int + or not 0 <= cursor["seq"] <= source_cursor["seq"]): + raise ValueError("incremental_cursor_unavailable") + # Timestamp ranges cover ordinary writes and wall-clock eligibility. + # The transactional cursor also covers late commits, deletes, scope + # moves and citation edits without a claim timestamp change. + instant = datetime.fromisoformat(changed_since.replace("Z", "+00:00")) + if instant.tzinfo is None: + raise ValueError("unqualified_baseline_time") + # Whole-second and old-timestamp writes are covered by the cursor; + # flooring here would reload unchanged rows from the same second. + lower_bound = instant.astimezone(timezone.utc).isoformat() + now = utc_now() + if instant > datetime.fromisoformat(now.replace("Z", "+00:00")): + raise ValueError("future_baseline_time") + watched = (prior_fence or {}).get("watched_ids", []) + if len(watched) > MAX_CLAIMS or any(type(id) is not int or id <= 0 for id in watched): + raise ValueError("invalid_scan_fence") + watched_sql = ",".join("?" for _ in watched) or "NULL" + queries = [ + "SELECT changes.claim_id AS id FROM project_source_changes changes " + "INDEXED BY idx_project_source_changes_seq " + "LEFT JOIN claims ON claims.id=changes.claim_id " + f"WHERE changes.seq > ? AND changes.seq <= ? AND (({where}) " + f"OR changes.claim_id IN ({watched_sql}))", + ] + changes_params: list[Any] = [cursor["seq"], source_cursor["seq"], *params, *watched] + for column, index in (("updated_at", "updated"), ("valid_from", "valid_from"), + ("valid_until", "valid_until")): + temporal = column != "updated_at" + queries.append( + f"SELECT id FROM claims INDEXED BY idx_project_claims_{index} " + f"WHERE {where} AND {column} >= ?" + (f" AND {column} <= ?" if temporal else "") ) - params.extend([lower_bound] * 3) - if max_seen_id is not None: - clauses.append("id > ?") - params.append(max_seen_id) - if not clauses: - return {}, True, {"data_version": conn.execute("PRAGMA data_version").fetchone()[0]} - where += " AND (" + " OR ".join(clauses) + ")" + changes_params.extend([*params, lower_bound, *([now] if temporal else [])]) + queries.append(f"SELECT id FROM claims NOT INDEXED WHERE {where} AND id > ?") + changes_params.extend([*params, max_seen_id or 0]) + changed_ids = [row[0] for row in conn.execute( + "SELECT id FROM (" + " UNION ".join(queries) + ") ORDER BY id LIMIT ?", + [*changes_params, limit + 1], + )] + if len(changed_ids) > limit: + return {}, False, {} + ids = sorted(set(ids or ()) | set(changed_ids)) + if ids is not None: + # An empty IN set reads neither claim payloads nor citations. + where += " AND id IN (" + (",".join("?" for _ in ids) or "NULL") + ")" + params.extend(ids) # Builder scans watch bounded all-status IDs. Ineligible rows can become # eligible without a reliable post-commit timestamp; their IDs are only # protected continuity metadata, never factual support or public output. @@ -354,8 +387,9 @@ def _read_claims( f"CASE WHEN ({size}) <= {MAX_FIELD_BYTES} THEN {name} ELSE NULL END AS {name}" if name != "id" else name for name in names ) + claim_table = "claims NOT INDEXED" if ids is not None else "claims" cursor = conn.execute( - f"SELECT {columns}, ({size}) AS _source_bytes FROM claims WHERE {where} ORDER BY id LIMIT ?", + f"SELECT {columns}, ({size}) AS _source_bytes FROM {claim_table} WHERE {where} ORDER BY id LIMIT ?", [*params, limit + 1], ) rows = [] @@ -364,7 +398,8 @@ def _read_claims( if len(rows) >= limit: fingerprint = {"source_bytes": total} if not validation: - watched = sorted(set(ids or ()) | {int(value["id"]) for value in rows}) + watched = sorted(set((prior_fence or {}).get("watched_ids", ())) + | set(ids or ()) | {int(value["id"]) for value in rows}) fingerprint["scan_fence"] = { "kind": "scan_fence", "watched_ids": watched[:MAX_CLAIMS], @@ -386,6 +421,7 @@ def _read_claims( citations: dict[int, list] = {} if ids_read: placeholders = ",".join("?" for _ in ids_read) + citation_index = " INDEXED BY idx_project_citations_claim" if source_cursor else "" citation_size = "length(CAST(COALESCE(source,'')||COALESCE(locator,'')||COALESCE(excerpt,'') AS BLOB))" citation_cursor = conn.execute( "SELECT id,claim_id," @@ -394,7 +430,7 @@ def _read_claims( for name in ("source", "locator", "excerpt") ) + f",{citation_size} AS _source_bytes " - f"FROM citations WHERE claim_id IN ({placeholders}) ORDER BY id LIMIT ?", + f"FROM citations{citation_index} WHERE claim_id IN ({placeholders}) ORDER BY id LIMIT ?", [*ids_read, citation_limit + 1], ) for index, citation in enumerate(citation_cursor): @@ -428,14 +464,17 @@ def _read_claims( "data_version": conn.execute("PRAGMA data_version").fetchone()[0], "next_temporal_boundary": min(boundaries) if boundaries else None, "source_bytes": total, + "changed_ids": changed_ids, } if not validation: - watched_ids = sorted(set(ids or ()) | {int(row["id"]) for row in rows}) + watched_ids = sorted(set((prior_fence or {}).get("watched_ids", ())) + | set(ids or ()) | {int(row["id"]) for row in rows}) fingerprint["scan_fence"] = { "kind": "scan_fence", "watched_ids": watched_ids[:MAX_CLAIMS], "max_seen_id": max([max_seen_id or 0, *watched_ids]), - "complete": complete and len(watched_ids) <= MAX_CLAIMS, + "complete": complete and len(watched_ids) <= MAX_CLAIMS and source_cursor is not None, + "source_cursor": source_cursor, } complete = complete and len(watched_ids) <= MAX_CLAIMS return items, complete, fingerprint @@ -469,19 +508,19 @@ def load_project_snapshot( ids = claim_ids prior_fence = None if coverage is not None: + if not changed_since: + raise ValueError("incremental_read_time_unavailable") prior_fence = coverage.get("__scan_fence__") - ids = sorted( - set(int(key.split(":", 1)[1]) for key in coverage if key.startswith("claim:")) - | set(prior_fence.get("watched_ids", ()) if isinstance(prior_fence, dict) else ()) - ) - if any(type(id) is not int or id <= 0 for id in ids): - raise ValueError("invalid_scan_fence") + # Coverage already holds unchanged support hashes. Only explicit + # display supports and indexed changes need source materialization. + ids = claim_ids or [] items, complete, fingerprints = _read_claims( config, ids=ids, changed_since=changed_since, validation=validation, max_seen_id=prior_fence.get("max_seen_id") if isinstance(prior_fence, dict) else None, + prior_fence=prior_fence, ) if coverage is not None and (not prior_fence or not prior_fence.get("complete")): complete = False @@ -496,7 +535,12 @@ def load_project_snapshot( items.update((card["key"], card) for card in cards.values()) result.update(items=items, complete=complete and ledger_complete, fingerprints=fingerprints) result["next_temporal_boundary"] = fingerprints.get("next_temporal_boundary") - result["coverage"] = {key: item["support"] for key, item in items.items() if item.get("open", True)} + carried = {key: value for key, value in (coverage or {}).items() if key.startswith("claim:")} + for id in fingerprints.pop("changed_ids", []): + carried.pop(f"claim:{id}", None) + result["coverage"] = {**carried, **{ + key: item["support"] for key, item in items.items() if item.get("open", True) + }} fence = fingerprints.pop("scan_fence", None) if fence is not None: fence["complete"] = fence["complete"] and complete diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py index 431f2cfb..c597282a 100644 --- a/memorymaster/profile/project_sessions.py +++ b/memorymaster/profile/project_sessions.py @@ -6,6 +6,7 @@ import hashlib import json import re +import sqlite3 from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path @@ -111,7 +112,8 @@ def _short(text: str, size: int) -> str: def build_session_delta(baseline: dict | None, projection_coverage: dict | None, - current: dict, *, diagnostic: str = "") -> SessionDelta: + current: dict, *, diagnostic: str = "", + item_loader: Callable[[str], dict | None] | None = None) -> SessionDelta: """Union start and projection differences, with current authorized supports only.""" def factual(value): return {key: support for key, support in value.items() if key.startswith(("claim:", "card:"))} @@ -133,25 +135,34 @@ def factual(value): label += "; " + diagnostic ordered = sorted(changed) chosen: list[tuple[str, dict]] = [] - omitted = 0 + omitted = len(ordered) + def render(): + lines = [heading, label, *(line for line, _ in chosen)] + if not changed and not withdrawn and complete and baseline is not None: + lines.append("No changes.") + lines.append(f"withdrawn={withdrawn}; omitted={omitted}") + return "\n".join(lines) + for key in ordered: + # Once full, later candidates cannot be rendered. Rehydrate at most + # the displayed delta supports plus one candidate that does not fit. + if len(render().encode("utf-8")) + 12 > 500: + break item = items.get(key) + if item is None and item_loader is not None: + item = item_loader(key) + if item is None: + break if not item or not item.get("support"): - omitted += 1 continue support = item["support"] reference = f"#{support['id']}" if support.get("kind") == "claim" else str(support["id"]) chosen.append((f"- {_short(item.get('text', ''), 110)} [{reference}]", support)) - # Always reserve explicit omission/withdrawal counts before selecting lines. - def render(): - lines = [heading, label, *(line for line, _ in chosen)] - if not changed and not withdrawn and complete and baseline is not None: - lines.append("No changes.") - lines.append(f"withdrawn={withdrawn}; omitted={omitted}") - return "\n".join(lines) - while len(render().encode("utf-8")) > 500 and chosen: - chosen.pop() - omitted += 1 + omitted -= 1 + if len(render().encode("utf-8")) > 500: + chosen.pop() + omitted += 1 + break text = render() if len(text.encode("utf-8")) > 500: raise ValueError("delta_budget") @@ -232,8 +243,23 @@ def record_project_session_start(config, session_id: str, *, projection: dict | coverage = current["coverage"] if len(_encoded(coverage)) > MAX_BASELINE_BYTES: return {"status": "baseline_capacity"} + + def load_delta_item(key): + if not key.startswith("claim:"): + return None + from memorymaster.profile.project_repository import _read_claims + + try: + items, complete, _ = _read_claims(config, ids=[int(key.split(":", 1)[1])], validation=True) + except (sqlite3.Error, OSError, ValueError): + return None + item = items.get(key) if complete else None + # A racing write cannot substitute support for the snapshot + # being recorded or leave stale prose in the delta manifest. + return item if item and item["support"] == coverage.get(key) else None + delta = build_session_delta(baseline, projection.get("source_coverage") if projection else None, - current, diagnostic=diagnostic) + current, diagnostic=diagnostic, item_loader=load_delta_item) if duplicate: return {"status": "duplicate", "delta": delta} with project_lock(lock, timeout=0.05) as acquired: diff --git a/memorymaster/stores/migrations/0029_project_source_cursor.py b/memorymaster/stores/migrations/0029_project_source_cursor.py new file mode 100644 index 00000000..434a0e23 --- /dev/null +++ b/memorymaster/stores/migrations/0029_project_source_cursor.py @@ -0,0 +1,115 @@ +"""Track per-claim project-source changes without reading source payloads. + +The project projection uses this cursor to read only the claims whose source +rows changed since its previous SessionStart. Timestamps are not a safe +watermark: a transaction may commit after a read while retaining an older +``updated_at`` value. The clock therefore advances inside the source-row +transaction and leaves a latest-sequence tombstone for deleted claims. +""" + +from __future__ import annotations + +from typing import Any + + +VERSION = 29 +DESCRIPTION = "SQLite project-source per-claim change cursor" +# This adds only derived metadata and triggers. Claim and citation data remain +# valid across a v28/v29 bridge exchange. +MERGE_COMPATIBLE = True + + +_SQLITE = """ +CREATE TABLE IF NOT EXISTS project_source_clock ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + epoch TEXT NOT NULL, + seq INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS project_source_changes ( + claim_id INTEGER PRIMARY KEY, + seq INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_project_source_changes_seq + ON project_source_changes(seq, claim_id); + +INSERT OR IGNORE INTO project_source_clock(singleton, epoch, seq) +VALUES (1, lower(hex(randomblob(16))), 0); + +CREATE INDEX IF NOT EXISTS idx_project_claims_updated + ON claims(tenant_id, scope, updated_at, id); +CREATE INDEX IF NOT EXISTS idx_project_claims_valid_from + ON claims(tenant_id, scope, valid_from, id); +CREATE INDEX IF NOT EXISTS idx_project_claims_valid_until + ON claims(tenant_id, scope, valid_until, id); +CREATE INDEX IF NOT EXISTS idx_project_citations_created + ON citations(created_at, claim_id); +CREATE INDEX IF NOT EXISTS idx_project_citations_claim + ON citations(claim_id, id); + +CREATE TRIGGER IF NOT EXISTS project_source_claims_ai AFTER INSERT ON claims BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT NEW.id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; +CREATE TRIGGER IF NOT EXISTS project_source_claims_au AFTER UPDATE ON claims BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT OLD.id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; + INSERT INTO project_source_changes(claim_id, seq) + SELECT NEW.id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; +CREATE TRIGGER IF NOT EXISTS project_source_claims_ad AFTER DELETE ON claims BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT OLD.id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; + +CREATE TRIGGER IF NOT EXISTS project_source_citations_ai AFTER INSERT ON citations BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT NEW.claim_id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; +CREATE TRIGGER IF NOT EXISTS project_source_citations_au AFTER UPDATE ON citations BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT OLD.claim_id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; + INSERT INTO project_source_changes(claim_id, seq) + SELECT NEW.claim_id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; +CREATE TRIGGER IF NOT EXISTS project_source_citations_ad AFTER DELETE ON citations BEGIN + UPDATE project_source_clock SET seq = seq + 1 WHERE singleton = 1; + INSERT INTO project_source_changes(claim_id, seq) + SELECT OLD.claim_id, seq FROM project_source_clock WHERE singleton = 1 + ON CONFLICT(claim_id) DO UPDATE SET seq = excluded.seq; +END; +""" + + +def _has_table(conn: Any, name: str) -> bool: + return conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,) + ).fetchone() is not None + + +def apply_sqlite(conn: Any) -> None: + """Install the cursor only when both authoritative source tables exist.""" + # The migration framework can stamp an empty database before schema.sql + # creates its tables. Creating half of this cursor would be misleading and + # cannot safely track citation writes, so leave such databases untouched. + if not _has_table(conn, "claims") or not _has_table(conn, "citations"): + return + conn.executescript(_SQLITE) + conn.commit() + + +def apply_postgres(conn: Any) -> None: + """Fail closed until a PostgreSQL source-cursor contract exists.""" + del conn + raise RuntimeError("migration 29 is SQLite-only; project source cursor is SQLite-only") diff --git a/memorymaster/stores/storage.py b/memorymaster/stores/storage.py index 28e44e20..8c6b92fd 100644 --- a/memorymaster/stores/storage.py +++ b/memorymaster/stores/storage.py @@ -215,6 +215,14 @@ def init_db(self) -> None: # A standalone `migrate` may have stamped v17 on an empty database. # Converge again after all prerequisite tables now exist. self._ensure_governed_capture_schema(mig_conn) + from importlib import import_module + + # Migration 29 intentionally no-ops on a bare database. A later + # init must converge its SQLite-only source cursor after schema.sql + # has created claims and citations, even if the runner stamped it. + import_module( + "memorymaster.stores.migrations.0029_project_source_cursor" + ).apply_sqlite(mig_conn) _refresh_planner_statistics(mig_conn) if fastpath: # Stamp ONLY after the full path succeeded end-to-end — an diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 26240cdd..57af0875 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -2,6 +2,8 @@ import json import sqlite3 +import pytest + from test_project_repository import claim, project_config as _project_config @@ -16,6 +18,106 @@ project_config = _project_config +@pytest.mark.parametrize("changed_count", [0, 1, 3]) +def test_session_start_incremental_counts_only_changed_rows_and_citations(project_config, monkeypatch, changed_count): + from memorymaster.profile import project_repository as repository + from memorymaster.profile import project_sessions as sessions + + monkeypatch.setattr(repository, "utc_now", lambda: "2026-10-22T09:00:00.750000+00:00") + for number in range(1, 41): + claim(project_config, number, text=f"Decision {number}", updated_at="2026-10-22T09:00:00.100000+00:00") + projection = build_project_projection(project_config) + with sqlite3.connect(project_config.db_path) as connection: + for number in range(1, changed_count + 1): + connection.execute( + "UPDATE claims SET text=?,version=2,updated_at=? WHERE id=?", + (f"Changed decision {number}", projection.manifest["read_at"], number), + ) + connection.execute("UPDATE citations SET excerpt='Updated evidence' WHERE claim_id=?", (number,)) + loaded = {"claims": [], "citations": []} + statements = [] + original_row = sqlite3.Row + original_connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + connection = original_connect(*args, **kwargs) + connection.set_trace_callback(statements.append) + return connection + + def counting_row(cursor, values): + row = original_row(cursor, values) + if "text" in row.keys() and "id" in row.keys(): + loaded["claims"].append(row["id"]) + elif "excerpt" in row.keys() and "claim_id" in row.keys(): + loaded["citations"].append(row["claim_id"]) + return row + + monkeypatch.setattr(repository.sqlite3, "Row", counting_row) + monkeypatch.setattr(repository.sqlite3, "connect", traced_connect) + result = sessions.record_project_session_start(project_config, "incremental", projection=projection.manifest) + assert result["status"] == "ok" + expected = list(range(1, changed_count + 1)) + assert loaded == {"claims": expected, "citations": expected} + with original_connect(project_config.db_path) as connection: + for statement in statements: + if not statement.startswith("SELECT"): + continue + plan = " ".join(row[3] for row in connection.execute("EXPLAIN QUERY PLAN " + statement)) + assert not any(f"SCAN {table}" in plan for table in ("claims", "citations", "changes")), plan + if "_source_bytes FROM claims" in statement: + assert "INTEGER PRIMARY KEY (rowid=?)" in plan + if "FROM citations" in statement: + assert "idx_project_citations_claim (claim_id=?)" in plan + state = sessions.read_session_state(project_config) + baseline = next(iter(state["baselines"].values())) + assert len([key for key in baseline if key.startswith("claim:")]) == 40 + for number in range(changed_count + 1, 41): + assert baseline[f"claim:{number}"] == projection.manifest["source_coverage"][f"claim:{number}"] + assert len(result["delta"].text.encode()) <= 500 + assert len(projection.markdown.encode()) <= 2200 + for entry in result["delta"].supports: + support = entry["support"] + assert support == baseline[f"claim:{support['id']}"] + + +def test_refreshed_projection_rehydrates_only_displayed_prior_session_delta(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + from memorymaster.profile import project_sessions as sessions + + for number in range(1, 41): + claim(project_config, number, text=f"Decision {number}") + before = build_project_projection(project_config) + assert sessions.record_project_session_start(project_config, "before", projection=before.manifest)["status"] == "ok" + assert sessions.record_project_session_end(project_config, "before") + with sqlite3.connect(project_config.db_path) as connection: + connection.execute("UPDATE claims SET text='Changed decision with details ' || id || ' ' || text,version=2") + refreshed = build_project_projection(project_config) + loaded = {"claims": [], "citations": []} + original_row = sqlite3.Row + + def counting_row(cursor, values): + row = original_row(cursor, values) + if "text" in row.keys() and "id" in row.keys(): + loaded["claims"].append(row["id"]) + elif "excerpt" in row.keys() and "claim_id" in row.keys(): + loaded["citations"].append(row["claim_id"]) + return row + + monkeypatch.setattr(repository.sqlite3, "Row", counting_row) + result = sessions.record_project_session_start(project_config, "after", projection=refreshed.manifest) + assert result["status"] == "ok" + delta = result["delta"] + assert delta.supports and "Changed decision" in delta.text + assert len(loaded["claims"]) <= len(delta.supports) + 1 < 40 + assert loaded["claims"] == loaded["citations"] + for entry in delta.supports: + assert entry["support"] == refreshed.manifest["source_coverage"][f"claim:{entry['support']['id']}"] + state = sessions.read_session_state(project_config) + assert state["baselines"][next(record["baseline"] for record in state["active"].values())] == refreshed.manifest[ + "source_coverage" + ] + + def test_exact_manifest_and_per_bullet_invalidation(project_config): config = project_config for i in range(1, 4): diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index 5993df93..d50ce376 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -1,4 +1,6 @@ from dataclasses import replace +from datetime import datetime, timezone +from importlib import import_module import json import os import sqlite3 @@ -35,6 +37,7 @@ def project_config(tmp_path): created_at TEXT,updated_at TEXT,event_time TEXT,valid_from TEXT,valid_until TEXT); CREATE TABLE citations(id INTEGER PRIMARY KEY,claim_id INTEGER,source TEXT, locator TEXT,excerpt TEXT,created_at TEXT);""") + import_module("memorymaster.stores.migrations.0029_project_source_cursor").apply_sqlite(conn) ledger = tmp_path / "ledger" ledger.mkdir() (ledger / "tasks.json").write_text(json.dumps({"complete": True, "tasks": []})) @@ -163,10 +166,23 @@ def test_oversized_citation_cannot_hide_sensitive_or_generated_payload(project_c assert not snapshot["complete"] -def test_incremental_includes_temporal_boundary_without_row_edit(project_config): - claim(project_config, valid_from="2026-10-01T00:00:00+00:00") - snapshot = snapshot_since(project_config, {}, "2026-09-30T00:00:00+00:00") +def test_incremental_includes_temporal_boundary_without_row_edit(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + now = "2026-10-22T09:00:00+00:00" + original_current = repository.claim_is_temporally_current + monkeypatch.setattr(repository, "utc_now", lambda: now) + monkeypatch.setattr(repository, "claim_is_temporally_current", lambda row: original_current( + row, now=datetime.fromisoformat(now).astimezone(timezone.utc))) + claim(project_config, valid_from="2026-10-22T09:01:00+00:00") + claim(project_config, 2, valid_until="2026-10-22T09:01:00+00:00") + baseline = load_project_snapshot(project_config) + assert "claim:1" not in baseline["coverage"] and "claim:2" in baseline["coverage"] + now = "2026-10-22T09:02:00+00:00" + snapshot = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) assert "claim:1" in snapshot["coverage"] + assert "claim:2" not in snapshot["coverage"] + assert snapshot["complete"] assert "next_temporal_boundary" in snapshot @@ -253,6 +269,58 @@ def test_incremental_fence_finds_first_citation_without_claim_timestamp_edit(pro assert current["complete"] +@pytest.mark.parametrize("mutation", [ + "DELETE FROM claims WHERE id=1", + "DELETE FROM citations WHERE claim_id=1", + "UPDATE claims SET scope='project:other' WHERE id=1", + "UPDATE claims SET tenant_id='other' WHERE id=1", + "UPDATE claims SET id=3 WHERE id=1", +]) +def test_incremental_withdraws_deleted_or_unauthorized_supports(project_config, mutation): + claim(project_config) + claim(project_config, 2) + baseline = load_project_snapshot(project_config) + with sqlite3.connect(project_config.db_path) as connection: + connection.execute(mutation) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert current["complete"] + assert "claim:1" not in current["coverage"] + assert current["coverage"]["claim:2"] == baseline["coverage"]["claim:2"] + + +@pytest.mark.parametrize("invalid_cursor", [None, {"epoch": "another-database", "seq": 0}, + {"epoch": "same", "seq": 1000000}]) +def test_incremental_missing_or_rewound_cursor_fails_closed_without_source_scan(project_config, monkeypatch, invalid_cursor): + from memorymaster.profile import project_repository as repository + + claim(project_config) + baseline = load_project_snapshot(project_config) + if invalid_cursor and invalid_cursor["epoch"] == "same": + invalid_cursor = {**invalid_cursor, "epoch": baseline["coverage"]["__scan_fence__"]["source_cursor"]["epoch"]} + baseline["coverage"]["__scan_fence__"]["source_cursor"] = invalid_cursor + statements = [] + original_connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + connection = original_connect(*args, **kwargs) + connection.set_trace_callback(statements.append) + return connection + + monkeypatch.setattr(repository.sqlite3, "connect", traced_connect) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert not current["complete"] + assert not any("FROM claims" in statement or "FROM citations" in statement for statement in statements) + + +def test_incremental_coverage_requires_read_watermark(project_config): + claim(project_config) + baseline = load_project_snapshot(project_config) + current = load_project_snapshot(project_config, coverage=baseline["coverage"]) + assert not current["complete"] + assert "__scan_fence__" not in current["coverage"] + assert not current["items"] + + def test_incomplete_scan_fence_never_claims_complete_incremental_coverage(project_config, monkeypatch): from memorymaster.profile import project_repository as repository @@ -300,9 +368,11 @@ def traced_connect(*args, **kwargs): current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) assert {"claim:1", "claim:2"} <= set(current["coverage"]) assert current["complete"] - row_reads = [statement for statement in statements if "FROM claims WHERE" in statement] + row_reads = [statement for statement in statements if "_source_bytes FROM claims" in statement] assert len(row_reads) == 1 - assert "id IN (1)" in row_reads[0] and "id > 1" in row_reads[0] + assert "id IN (1,2)" in row_reads[0] + selection = next(statement for statement in statements if "FROM project_source_changes" in statement) + assert "changes.seq >" in selection and "id > 1" in selection @pytest.mark.parametrize("source", ["claims", "citations"]) @@ -331,13 +401,13 @@ def execute(self, query, parameters=()): def fetchall(self): rows = super().fetchall() - if f"FROM {source} WHERE" in self.query: + if f"FROM {source}" in self.query: consumed.extend(rows) return rows def __next__(self): row = super().__next__() - if f"FROM {source} WHERE" in self.query: + if f"FROM {source}" in self.query: consumed.append(row) return row diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py index 424546ec..2d4fe017 100644 --- a/tests/test_project_sessions.py +++ b/tests/test_project_sessions.py @@ -115,6 +115,23 @@ def test_delta_cap_withdrawal_and_no_false_empty(): assert "no changes" not in partial.text.lower() +def test_delta_stops_loading_after_unavailable_support(): + current = snapshot(**{f"claim:{number}": {**claim("changed"), "key": f"claim:{number}"} + for number in range(1, 41)}) + current["items"] = {} + loaded = [] + + def unavailable(key): + loaded.append(key) + return None + + delta = sessions.build_session_delta({}, {}, current, item_loader=unavailable) + assert len(loaded) == 1 + assert not delta.supports + assert delta.omitted == 40 + assert "No changes" not in delta.text + + def test_disabled_state_never_reads_or_writes(tmp_path): cfg = config(tmp_path) cfg.enabled = False diff --git a/tests/test_project_source_cursor.py b/tests/test_project_source_cursor.py new file mode 100644 index 00000000..8b2096fd --- /dev/null +++ b/tests/test_project_source_cursor.py @@ -0,0 +1,375 @@ +"""Contract tests for the SQLite project-source change cursor.""" + +from __future__ import annotations + +import importlib +import sqlite3 +from pathlib import Path + +import pytest + + +cursor_migration = importlib.import_module("memorymaster.stores.migrations.0029_project_source_cursor") + + +_SOURCE_SCHEMA = """ +CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT, + tenant_id TEXT, + scope TEXT, + updated_at TEXT, + valid_from TEXT, + valid_until TEXT +); +CREATE TABLE citations ( + id INTEGER PRIMARY KEY, + claim_id INTEGER NOT NULL, + source TEXT, + locator TEXT, + excerpt TEXT, + created_at TEXT +); +""" + + +def _open(path: Path | str = ":memory:") -> sqlite3.Connection: + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + conn.executescript(_SOURCE_SCHEMA) + cursor_migration.apply_sqlite(conn) + return conn + + +def _clock(conn: sqlite3.Connection) -> sqlite3.Row: + return conn.execute("SELECT epoch, seq FROM project_source_clock WHERE singleton = 1").fetchone() + + +def _changes(conn: sqlite3.Connection) -> dict[int, int]: + return { + int(row["claim_id"]): int(row["seq"]) + for row in conn.execute("SELECT claim_id, seq FROM project_source_changes ORDER BY claim_id") + } + + +def _claim(conn: sqlite3.Connection, claim_id: int, **overrides: object) -> None: + fields: dict[str, object] = { + "id": claim_id, + "text": f"claim {claim_id}", + "tenant_id": "tenant-a", + "scope": "project:demo", + "updated_at": "2026-10-22T00:00:00+00:00", + "valid_from": "2026-10-01T00:00:00+00:00", + "valid_until": None, + } + fields.update(overrides) + conn.execute( + f"INSERT INTO claims ({','.join(fields)}) VALUES ({','.join('?' for _ in fields)})", + tuple(fields.values()), + ) + + +def test_schema_clock_and_indexes_are_metadata_only_and_idempotent() -> None: + conn = _open() + try: + before = _clock(conn) + assert len(before["epoch"]) == 32 + int(before["epoch"], 16) + assert before["seq"] == 0 + assert _changes(conn) == {} + assert cursor_migration.MERGE_COMPATIBLE is True + + cursor_migration.apply_sqlite(conn) + assert dict(_clock(conn)) == dict(before) + names = { + row["name"] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type IN ('index', 'trigger')") + } + assert { + "idx_project_claims_updated", + "idx_project_claims_valid_from", + "idx_project_claims_valid_until", + "idx_project_citations_created", + "idx_project_citations_claim", + "project_source_claims_ai", + "project_source_claims_au", + "project_source_claims_ad", + "project_source_citations_ai", + "project_source_citations_au", + "project_source_citations_ad", + } <= names + assert [row["name"] for row in conn.execute("PRAGMA table_info(project_source_changes)")] == ["claim_id", "seq"] + finally: + conn.close() + + +def test_bare_database_is_a_noop_and_postgres_fails_closed() -> None: + bare = sqlite3.connect(":memory:") + try: + cursor_migration.apply_sqlite(bare) + assert bare.execute( + "SELECT 1 FROM sqlite_master WHERE name = 'project_source_clock'" + ).fetchone() is None + finally: + bare.close() + + with pytest.raises(RuntimeError, match="migration 29 is SQLite-only"): + cursor_migration.apply_postgres(object()) + + +@pytest.mark.parametrize("present_table", ["claims", "citations"]) +def test_partial_source_schema_is_a_noop(present_table: str) -> None: + conn = sqlite3.connect(":memory:") + try: + conn.execute(f"CREATE TABLE {present_table}(id INTEGER PRIMARY KEY)") + cursor_migration.apply_sqlite(conn) + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE name = 'project_source_clock'" + ).fetchone() is None + finally: + conn.close() + + +def test_populated_pre29_store_upgrade_preserves_payload_without_lenient_schema_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + import logging + + from memorymaster.core.models import CitationInput + from memorymaster.stores.storage import SQLiteStore + + database = tmp_path / "pre29.sqlite" + store = SQLiteStore(database) + original_apply = cursor_migration.apply_sqlite + monkeypatch.setattr(cursor_migration, "apply_sqlite", lambda _conn: None) + store.init_db() + monkeypatch.setattr(cursor_migration, "apply_sqlite", original_apply) + + created = store.create_claim( + "Legacy claim payload survives upgrade.", + [CitationInput(source="fixture://pre29", locator="line:1", excerpt="Legacy evidence.")], + scope="project:legacy", + source_agent="fixture", + ) + with sqlite3.connect(database) as conn: + conn.execute("DELETE FROM schema_versions WHERE version = 29") + + caplog.clear() + with caplog.at_level(logging.WARNING, logger="memorymaster.stores.storage"): + store.init_db() + + assert not any("lenient schema initialization" in record.getMessage() for record in caplog.records) + with sqlite3.connect(database) as conn: + assert conn.execute("SELECT text FROM claims WHERE id = ?", (created.id,)).fetchone() == ( + "Legacy claim payload survives upgrade.", + ) + assert conn.execute( + "SELECT source, locator, excerpt FROM citations WHERE claim_id = ?", (created.id,) + ).fetchone() == ("fixture://pre29", "line:1", "Legacy evidence.") + assert conn.execute("SELECT seq FROM project_source_clock WHERE singleton = 1").fetchone() == (0,) + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = 'project_source_claims_ai'" + ).fetchone() == (1,) + + +def test_real_sqlite_init_applies_cursor_after_creating_source_tables(tmp_path: Path) -> None: + from memorymaster.stores.storage import SQLiteStore + + database = tmp_path / "fresh.sqlite" + SQLiteStore(database).init_db() + conn = sqlite3.connect(database) + try: + assert conn.execute("SELECT version FROM schema_versions WHERE version = 29").fetchone() == (29,) + assert conn.execute("SELECT seq FROM project_source_clock WHERE singleton = 1").fetchone() == (0,) + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = 'project_source_claims_ai'" + ).fetchone() == (1,) + finally: + conn.close() + + +def test_standalone_bare_runner_then_init_converges_cursor(tmp_path: Path) -> None: + from memorymaster.stores.migrations import MigrationRunner + from memorymaster.stores.storage import SQLiteStore + + database = tmp_path / "standalone-before-init.sqlite" + bare = sqlite3.connect(database) + try: + assert 29 in MigrationRunner(bare, backend="sqlite").apply_pending() + finally: + bare.close() + + SQLiteStore(database).init_db() + conn = sqlite3.connect(database) + try: + assert conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name = 'project_source_claims_ai'" + ).fetchone() == (1,) + finally: + conn.close() + + +def test_claim_changes_advance_sequence_retain_tombstone_and_rollback_atomically() -> None: + conn = _open() + try: + _claim(conn, 1) + assert _clock(conn)["seq"] == 1 + assert _changes(conn) == {1: 1} + + conn.execute("UPDATE claims SET scope = 'project:moved', updated_at = '2001-01-01' WHERE id = 1") + assert _clock(conn)["seq"] == 2 + assert _changes(conn) == {1: 2} + + conn.execute("DELETE FROM claims WHERE id = 1") + assert _clock(conn)["seq"] == 3 + assert _changes(conn) == {1: 3} + + conn.commit() + committed = dict(_clock(conn)) + conn.execute("BEGIN") + _claim(conn, 2) + conn.execute("UPDATE claims SET text = 'transient' WHERE id = 2") + assert _clock(conn)["seq"] == 5 + conn.rollback() + assert dict(_clock(conn)) == committed + assert _changes(conn) == {1: 3} + finally: + conn.close() + + +def test_claim_primary_key_move_marks_old_and_new_ids() -> None: + conn = _open() + try: + _claim(conn, 1) + conn.execute("UPDATE claims SET id = 2 WHERE id = 1") + assert _clock(conn)["seq"] == 2 + assert _changes(conn) == {1: 2, 2: 2} + finally: + conn.close() + + +def test_citation_move_and_delete_mark_old_and_new_claims() -> None: + conn = _open() + try: + _claim(conn, 1) + _claim(conn, 2) + conn.execute( + "INSERT INTO citations(id, claim_id, source, created_at) VALUES (1, 1, 'session', '2026-10-22')" + ) + assert _changes(conn) == {1: 3, 2: 2} + + conn.execute("UPDATE citations SET claim_id = 2, excerpt = 'moved evidence' WHERE id = 1") + assert _clock(conn)["seq"] == 4 + assert _changes(conn) == {1: 4, 2: 4} + + conn.execute("DELETE FROM citations WHERE id = 1") + assert _clock(conn)["seq"] == 5 + assert _changes(conn) == {1: 4, 2: 5} + finally: + conn.close() + + +def test_wal_late_commit_with_old_timestamps_uses_monotonic_sequence(tmp_path: Path) -> None: + database = tmp_path / "cursor.sqlite" + reader = _open(database) + writer = sqlite3.connect(database) + try: + reader.execute("PRAGMA journal_mode=WAL") + _claim(reader, 1) + reader.commit() + baseline = int(_clock(reader)["seq"]) + + writer.execute("BEGIN") + writer.execute("UPDATE claims SET updated_at = '2001-01-01T00:00:00+00:00' WHERE id = 1") + writer.execute( + "INSERT INTO claims(id, text, tenant_id, scope, updated_at) VALUES " + "(2, 'late', 'tenant-a', 'project:demo', '2001-01-01T00:00:00+00:00')" + ) + assert int(_clock(reader)["seq"]) == baseline + writer.commit() + + assert int(_clock(reader)["seq"]) == baseline + 2 + assert _changes(reader) == {1: baseline + 1, 2: baseline + 2} + finally: + writer.close() + reader.close() + + +def test_query_plans_use_project_source_indexes() -> None: + conn = _open() + try: + for claim_id in range(1, 41): + _claim( + conn, + claim_id, + updated_at=f"2026-10-{claim_id:02d}", + valid_from=f"2026-09-{claim_id:02d}", + valid_until=f"2026-11-{claim_id:02d}", + ) + conn.execute( + "INSERT INTO citations(id, claim_id, created_at) VALUES (?, ?, ?)", + (claim_id, claim_id, f"2026-10-{claim_id:02d}"), + ) + plans = [ + ( + "SELECT id FROM claims WHERE tenant_id = ? AND scope = ? AND updated_at >= ? ORDER BY updated_at, id", + ("tenant-a", "project:demo", "2026-10-01"), + "idx_project_claims_updated", + ), + ( + "SELECT id FROM claims WHERE tenant_id = ? AND scope = ? AND valid_from <= ? ORDER BY valid_from, id", + ("tenant-a", "project:demo", "2026-10-31"), + "idx_project_claims_valid_from", + ), + ( + "SELECT id FROM claims WHERE tenant_id = ? AND scope = ? AND valid_until >= ? ORDER BY valid_until, id", + ("tenant-a", "project:demo", "2026-10-01"), + "idx_project_claims_valid_until", + ), + ( + "SELECT claim_id FROM citations WHERE created_at >= ? ORDER BY created_at, claim_id", + ("2026-10-01",), + "idx_project_citations_created", + ), + ( + "SELECT id FROM citations WHERE claim_id = ? ORDER BY id", + (20,), + "idx_project_citations_claim", + ), + ] + for query, values, index in plans: + detail = " ".join(row[3] for row in conn.execute(f"EXPLAIN QUERY PLAN {query}", values)) + assert index in detail + finally: + conn.close() + + +def test_backup_restore_preserves_cursor_and_trigger_behavior(tmp_path: Path) -> None: + source_path = tmp_path / "source.sqlite" + restored_path = tmp_path / "restored.sqlite" + source = _open(source_path) + try: + _claim(source, 1) + source.execute("INSERT INTO citations(id, claim_id, created_at) VALUES (1, 1, '2026-10-22')") + source.commit() + expected_clock = dict(_clock(source)) + expected_changes = _changes(source) + restored = sqlite3.connect(restored_path) + try: + source.backup(restored) + finally: + restored.close() + finally: + source.close() + + restored = sqlite3.connect(restored_path) + restored.row_factory = sqlite3.Row + try: + assert dict(_clock(restored)) == expected_clock + assert _changes(restored) == expected_changes + cursor_migration.apply_sqlite(restored) + restored.execute("UPDATE claims SET text = 'restored edit' WHERE id = 1") + assert int(_clock(restored)["seq"]) == int(expected_clock["seq"]) + 1 + assert _changes(restored) == {1: int(expected_clock["seq"]) + 1} + finally: + restored.close() From e1b827e9b9430162138d39a163b739a30d9e7a61 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:19:25 -0300 Subject: [PATCH 19/51] fix(T-1703): reject generated lineage in stored dedup matches --- memorymaster/core/service.py | 15 +++++++++++++++ tests/test_service.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 45ed5b38..4c1c3380 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -486,6 +486,17 @@ def init_db(self) -> None: ) self.store.init_db() + @staticmethod + def _validate_dedup_lineage(claim: Claim) -> None: + """Old rows and race winners must satisfy the same intake boundary.""" + from memorymaster.core.generated_artifacts import contains_generated_marker + + fields = [claim.text, claim.subject, claim.predicate, claim.object_value] + fields.extend(value for citation in claim.citations + for value in (citation.source, citation.locator, citation.excerpt)) + if any(contains_generated_marker(value) for value in fields if value): + raise ValueError("Stored dedup claim has generated content or citation evidence.") + def _revive_archived_dedup_match(self, claim: Claim, source_agent: str | None) -> Claim: """Resurface an archived claim matched by ingest dedup. @@ -498,6 +509,7 @@ def _revive_archived_dedup_match(self, claim: Claim, source_agent: str | None) - review. Explicit forget/rollback intent blocks automatic revival. Non-archived matches pass through untouched (dedup unchanged). """ + self._validate_dedup_lineage(claim) if claim.status != "archived": return claim if self.store.latest_archive_reason(claim.id) in {"friendly_forget_claim", "ao7_rollback"}: @@ -517,6 +529,8 @@ def _revive_archived_dedup_match(self, claim: Claim, source_agent: str | None) - # Another writer touched the row between dedup and revival — # the freshest state wins; never fail the whole ingest for it. reloaded = self.store.get_claim(claim.id) + if reloaded is not None: + self._validate_dedup_lineage(reloaded) return reloaded if reloaded is not None else claim # getattr-guard: some paths build MemoryService via __new__ and skip # __init__, so the qdrant attribute may not exist. @@ -670,6 +684,7 @@ def ingest( visibility=visibility, source_agent=source_agent, ) if retired is not None: + self._validate_dedup_lineage(retired) fill_outcome_receipt(outcome_receipt, retired, created=False, persisted=self.store.ingest_creation_receipt(retired.id) if outcome_receipt is not None else None) observability.bump_claim_ingested(source_agent) diff --git a/tests/test_service.py b/tests/test_service.py index c1643078..52027642 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -20,6 +20,34 @@ def test_generated_only_cannot_revive_archived_producer_key(service): assert service.store.get_claim(claim.id).status == "archived" +@pytest.mark.parametrize("match_path", ["producer", "content_hash", "concurrent"]) +@pytest.mark.parametrize("field", ["text", "subject", "predicate", "object_value", "citation"]) +def test_legacy_generated_dedup_key_must_not_revive(service, monkeypatch, match_path, field): + import hashlib + + clean = "A newly authored clean claim." + key = "producer-key" + if match_path == "content_hash": + key = "hash-" + hashlib.sha256(f"{clean.lower()}|project|".encode()).hexdigest()[:16] + generated = GENERATED_OPENER + "Recalled falsehood." + GENERATED_CLOSER + fields = {field: generated} if field not in {"text", "citation"} else {} + old = service.store.create_claim( + generated if field == "text" else "Previously authored claim.", + [CitationInput(source="fixture", excerpt=generated if field == "citation" else None)], + idempotency_key=key, **fields, + ) + service.store.apply_status_transition(old, to_status="archived", reason="fixture", event_type="transition") + if match_path == "concurrent": + # Simulate a winner becoming visible only inside atomic create_claim. + monkeypatch.setattr(service.store, "get_claim_by_idempotency_key", lambda *args, **kwargs: None) + with pytest.raises(ValueError, match="generated"): + service.ingest(clean, [CitationInput(source="fixture")], + idempotency_key=None if match_path == "content_hash" else key) + persisted = service.store.get_claim(old.id) + assert persisted.status == "archived" + assert persisted.text == old.text + + def test_mixed_atomic_claim_retains_one_authored_span_and_recomputes_dedup(service): text = f"A durable authored claim.\n{GENERATED_OPENER}Generated falsehood.{GENERATED_CLOSER}" claim = service.ingest(text, [CitationInput(source="fixture", excerpt="A durable authored claim.")]) From 3867922ee6f55135aca9653d758c5a290651a1d2 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:20:34 -0300 Subject: [PATCH 20/51] fix(T-1703): preserve disjoint authored evidence spans --- memorymaster/capture/adapters.py | 6 ++++ memorymaster/dreaming/capture.py | 32 ++++++++++++--------- memorymaster/surfaces/session_end_ingest.py | 2 +- tests/test_capture_adapters.py | 13 +++++++++ tests/test_dreaming_self_observation.py | 20 +++++++++++++ 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/memorymaster/capture/adapters.py b/memorymaster/capture/adapters.py index a69d747d..abda9488 100644 --- a/memorymaster/capture/adapters.py +++ b/memorymaster/capture/adapters.py @@ -128,6 +128,12 @@ def _retained_text(text: str, *, html: bool = False) -> tuple[str, tuple[str, .. if span.start == 0 and span.text.startswith("\ufeff"): span = SourceSpan(1, span.end, span.text[1:]) value = _html_text(span.text.encode("utf-8")) if html else span.text + if html and strip_generated_spans(value).excluded_spans: + # HTML entity decoding changes offsets. Without a reversible source + # map, splitting decoded text would fabricate original attribution. + raise CaptureRejected( + "ambiguous_generated_provenance", "Decoded HTML generated markers cannot be attributed safely." + ) bounded, warnings = _bounded_text(value) warning_codes.extend(warnings) if bounded.strip(): diff --git a/memorymaster/dreaming/capture.py b/memorymaster/dreaming/capture.py index e65af3d9..5edd9b81 100644 --- a/memorymaster/dreaming/capture.py +++ b/memorymaster/dreaming/capture.py @@ -10,6 +10,7 @@ from typing import Any, Mapping from memorymaster.core.capture_control import CaptureLedger +from memorymaster.core.generated_artifacts import SourceSpan from memorymaster.core.security import redact_text from memorymaster.core.scope_utils import resolve_transcript_workspace_scope, scope_from_cwd from memorymaster.dreaming.ledger import DreamLedger @@ -35,7 +36,7 @@ def _text_parts(content: Any, allowed: set[str]) -> str: def _conversation_record( entry: dict[str, Any], provider: str, *, state: dict[str, Any] | None = None, -) -> tuple[str, str] | None: +) -> tuple[str, tuple[SourceSpan, ...]] | None: state = state if state is not None else {} payload = entry.get("payload") if provider == "codex" else entry.get("message", entry) payload = payload if isinstance(payload, dict) else entry @@ -73,7 +74,9 @@ def _conversation_record( "injected": bool(entry.get("hook_additional_context") or entry.get("isMeta") or payload.get("hook_additional_context")), "generated_session": bool(entry.get("backfill_generated") or entry.get("memorymaster_generated")), }) - return (normalized_role, clean.text) if normalized_role and clean.text else None + # The convenience joined text is never attributable evidence: a removed + # range can separate an authored prefix from an unrelated suffix. + return (normalized_role, clean.spans) if normalized_role and clean.spans else None def _sanitize(text: str) -> str: @@ -98,18 +101,21 @@ def parse_transcript_lines( record = _conversation_record(entry, provider.lower(), state=state) if record is None: continue - role, content = record - sanitized = _sanitize(content) - if not sanitized: - continue + role, spans = record timestamp = str(entry.get("timestamp") or _now()) - identity = (role, sanitized, timestamp) - if identity in seen: - continue - seen.add(identity) - raw_id = str(entry.get("uuid") or entry.get("id") or "|".join(identity)) - message_id = "dm-" + hashlib.sha256(raw_id.encode("utf-8", errors="replace")).hexdigest()[:20] - messages.append(DreamMessage(message_id, role, sanitized, timestamp)) + for span in spans: + sanitized = _sanitize(span.text) + if not sanitized: + continue + identity = (role, sanitized, timestamp) + if identity in seen: + continue + seen.add(identity) + raw_id = str(entry.get("uuid") or entry.get("id") or "|".join(identity)) + if len(spans) > 1: + raw_id += f":span:{span.start}:{span.end}" + message_id = "dm-" + hashlib.sha256(raw_id.encode("utf-8", errors="replace")).hexdigest()[:20] + messages.append(DreamMessage(message_id, role, sanitized, timestamp)) return messages diff --git a/memorymaster/surfaces/session_end_ingest.py b/memorymaster/surfaces/session_end_ingest.py index 780f70a0..7c1b124c 100644 --- a/memorymaster/surfaces/session_end_ingest.py +++ b/memorymaster/surfaces/session_end_ingest.py @@ -58,7 +58,7 @@ def _assistant_text(transcript_path: str) -> str: entry = {**entry, "message": payload, "type": role or entry.get("type", "")} record = _conversation_record(entry, provider, state=state) if record is not None and record[0] == "assistant": - eligible.append((index, record[1])) + eligible.extend((index, span.text) for span in record[1]) for index, content in reversed(eligible): if index < len(lines) - 200: break diff --git a/tests/test_capture_adapters.py b/tests/test_capture_adapters.py index 7add16ff..3f45794b 100644 --- a/tests/test_capture_adapters.py +++ b/tests/test_capture_adapters.py @@ -51,6 +51,19 @@ def test_generated_only_inline_does_not_create_capture(): assert exc.value.code == "generated_artifact" +def test_html_escaped_generated_markers_reject_ambiguous_evidence(capture_root): + from html import escape + + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = "

Authored prefix.

" + escape(GENERATED_OPENER + "excluded" + GENERATED_CLOSER) + "

Authored suffix.

" + path = capture_root / "encoded.html" + path.write_text(raw, encoding="utf-8") + with pytest.raises(CaptureRejected) as exc: + LocalFileAdapter(path).capture() + assert exc.value.code == "ambiguous_generated_provenance" + + def test_generated_only_html_wrapper_does_not_create_capture(capture_root): from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER path = capture_root / "generated.html" diff --git a/tests/test_dreaming_self_observation.py b/tests/test_dreaming_self_observation.py index fed9b754..94867e04 100644 --- a/tests/test_dreaming_self_observation.py +++ b/tests/test_dreaming_self_observation.py @@ -84,3 +84,23 @@ def test_live_capture_keeps_contamination_across_incremental_reads(tmp_path): with transcript.open("a", encoding="utf-8") as handle: handle.write(json.dumps({"type": "assistant", "message": {"role": "assistant", "content": "Unmarked echo."}}) + "\n") assert capture_transcript(transcript, **args)["reason"] == "no_conversation" + + +def test_separated_authored_spans_are_distinct_persisted_extractor_evidence(tmp_path): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + from memorymaster.dreaming.worker import DreamConfig, DreamWorker + + raw = "Use " + GENERATED_OPENER + "injected gap" + GENERATED_CLOSER + "SQLite for production." + row = {"role": "assistant", "uuid": "original-turn", "content": raw} + messages = parse_transcript_lines(json.dumps(row), "claude") + assert [message.text for message in messages] == ["Use", "SQLite for production."] + assert len({message.message_id for message in messages}) == 2 + transcript = tmp_path / "span-session.jsonl" + transcript.write_text(json.dumps(row) + "\n", encoding="utf-8") + ledger = DreamLedger(tmp_path / "spans.db") + result = capture_transcript(transcript, provider="claude", session_id="spans", cwd=str(tmp_path), + ledger=ledger, live_workspace=True) + persisted = ledger.get_capture(result["capture_id"])["messages"] + worker = object.__new__(DreamWorker) + worker.config = DreamConfig() + assert [message["text"] for message in worker._bounded_messages(persisted)] == ["Use", "SQLite for production."] From 6fa1e268acb7778956c3f5d93387834ff9e00e26 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:20:38 -0300 Subject: [PATCH 21/51] fix(T-1703): fail closed on normalized project consumer flags --- .../hooks/memorymaster-session-start.py | 9 ++-- tests/test_compiled_profile_surfaces.py | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/memorymaster/config_templates/hooks/memorymaster-session-start.py b/memorymaster/config_templates/hooks/memorymaster-session-start.py index 3cba2733..496a9c6e 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-start.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-start.py @@ -365,18 +365,21 @@ def main(): except Exception: pass + project_enabled = True # Import failure must also withhold legacy delivery. try: - from memorymaster.profile.project_runtime import session_start_context + from memorymaster.profile.project_runtime import active, session_start_context + project_enabled = active("MEMORYMASTER_PROJECT_PROJECTION_ENABLED") context = session_start_context(data, profile_loader=_load_compiled_profile) if context is not None: sys.stdout.write(json.dumps({"hookSpecificOutput": { "hookEventName": "SessionStart", "additionalContext": context}})) sys.stdout.flush() sys.exit(0) - except Exception: + except Exception as exc: # An enabled project consumer must never fall through to less restrictive legacy SQL. - if os.environ.get("MEMORYMASTER_PROJECT_PROJECTION_ENABLED", "").lower() in ("1", "true", "yes", "on"): + if project_enabled: + _log("project-consumer-error", cause=type(exc).__name__) sys.exit(0) if not DB_PATH.exists(): diff --git a/tests/test_compiled_profile_surfaces.py b/tests/test_compiled_profile_surfaces.py index ad9d0af1..4af9d524 100644 --- a/tests/test_compiled_profile_surfaces.py +++ b/tests/test_compiled_profile_surfaces.py @@ -127,6 +127,47 @@ def test_project_session_start_uses_payload_before_legacy_database(tmp_path, mon assert json.loads(capsys.readouterr().out)["hookSpecificOutput"]["additionalContext"] == "validated:" + str(tmp_path) +@pytest.mark.parametrize("flag", ["1", "true", "yes", "on", " 1 ", " TRUE ", " Yes ", " ON "]) +def test_project_consumer_error_never_falls_back_to_private_legacy_claim(tmp_path, monkeypatch, capsys, flag): + import sqlite3 + from memorymaster.profile import project_runtime + from memorymaster.recall import delivery + + module = _hook_module() + db = tmp_path / "legacy.db" + with sqlite3.connect(db) as connection: + connection.execute("CREATE TABLE claims(id INTEGER PRIMARY KEY, text TEXT, claim_type TEXT, " + "status TEXT, created_at TEXT, scope TEXT, tenant_id TEXT, visibility TEXT)") + connection.execute("INSERT INTO claims VALUES(1, 'Private unrelated tenant context', 'fact', " + "'confirmed', '2026-10-22', 'project:fixture', 'other', 'private')") + monkeypatch.setenv("MEMORYMASTER_PROJECT_PROJECTION_ENABLED", flag) + monkeypatch.setenv("MEMORYMASTER_PROJECT_PROJECTION_SESSION_START", "1") + monkeypatch.setattr(module, "DB_PATH", db) + monkeypatch.setattr(module, "_derive_scope", lambda: "project:fixture") + monkeypatch.setattr(module, "_load_recent_wiki_articles", lambda *_: []) + monkeypatch.setattr(module, "_load_compiled_profile", lambda: "") + monkeypatch.setattr(module, "_jev_session_active", lambda: False) + monkeypatch.setattr(delivery, "reset_session", lambda *_: None) + logged = [] + monkeypatch.setattr(module, "_log", lambda event, **values: logged.append((event, values))) + monkeypatch.setattr(sys, "stdin", io.StringIO("{}")) + + def broken_consumer(*args, **kwargs): + raise ValueError("consumer failure with private source text") + + monkeypatch.setattr(project_runtime, "session_start_context", broken_consumer) + queries = [] + original_query = module._query_recent_claims + monkeypatch.setattr(module, "_query_recent_claims", + lambda *args: queries.append(True) or original_query(*args)) + with pytest.raises(SystemExit) as stopped: + module.main() + assert stopped.value.code == 0 + assert not queries + assert capsys.readouterr().out == "" + assert logged == [("project-consumer-error", {"cause": "ValueError"})] + + def test_project_cli_branches_before_service_initialization(tmp_path, monkeypatch, capsys): from memorymaster.profile import __main__ as cli From b41bd67063bba90fe17f2b121099a011f0a44848 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:21:16 -0300 Subject: [PATCH 22/51] fix(T-1703): compare project validity boundaries as UTC instants --- memorymaster/profile/project_repository.py | 71 ++++++++++++++-------- tests/test_project_repository.py | 52 ++++++++++++++++ 2 files changed, 99 insertions(+), 24 deletions(-) diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index 4a958e62..89a131dc 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -14,6 +14,7 @@ from typing import Any from memorymaster.core.generated_artifacts import contains_generated_marker +from memorymaster.core.models import _parse_iso_strict from memorymaster.core.security import scan_persisted_value from memorymaster.core.temporal_policy import claim_is_temporally_current from memorymaster.profile.egress import prepare_claim_egress @@ -38,6 +39,20 @@ def utc_now() -> str: return datetime.now(timezone.utc).isoformat() +def _validity_instants(rows) -> list[tuple[int, datetime]]: + """Compare accepted ISO forms as instants without rewriting support fields.""" + result = [] + for row in rows: + for name in ("valid_from", "valid_until"): + try: + instant = _parse_iso_strict(name, row[name]) + except ValueError: + continue # Malformed persisted bounds are ineligible at rehydration. + if instant is not None: + result.append((row["id"], instant.astimezone(timezone.utc))) + return result + + def _safe(value: Any) -> bool: def marked(raw: Any) -> bool: if isinstance(raw, str): @@ -299,6 +314,8 @@ def _read_claims( params: list[Any] = [config.tenant_id, *scopes] where = "tenant_id IS ? AND scope IN (" + ",".join("?" for _ in scopes) + ")" changed_ids = [] + temporal_instants = [] + temporal_bytes = 0 if changed_since: cursor = (prior_fence or {}).get("source_cursor") if (not source_cursor or not isinstance(cursor, dict) @@ -306,22 +323,36 @@ def _read_claims( or type(cursor.get("seq")) is not int or not 0 <= cursor["seq"] <= source_cursor["seq"]): raise ValueError("incremental_cursor_unavailable") - # Timestamp ranges cover ordinary writes and wall-clock eligibility. - # The transactional cursor also covers late commits, deletes, scope - # moves and citation edits without a claim timestamp change. + # The transactional cursor covers writes regardless of wall-clock + # spelling, late commits, deletes, scope moves and citation edits. instant = datetime.fromisoformat(changed_since.replace("Z", "+00:00")) if instant.tzinfo is None: raise ValueError("unqualified_baseline_time") - # Whole-second and old-timestamp writes are covered by the cursor; - # flooring here would reload unchanged rows from the same second. - lower_bound = instant.astimezone(timezone.utc).isoformat() now = utc_now() - if instant > datetime.fromisoformat(now.replace("Z", "+00:00")): + now_instant = datetime.fromisoformat(now.replace("Z", "+00:00")) + if instant > now_instant: raise ValueError("future_baseline_time") watched = (prior_fence or {}).get("watched_ids", []) if len(watched) > MAX_CLAIMS or any(type(id) is not int or id <= 0 for id in watched): raise ValueError("invalid_scan_fence") watched_sql = ",".join("?" for _ in watched) or "NULL" + # A complete fence contains every previously scanned ID. Read only + # its bounded validity metadata by primary key, then normalize in + # Python: indexed TEXT ranges cannot compare offsets or fractions. + temporal_rows = list(conn.execute( + "SELECT id," + "CASE WHEN length(valid_from)<=128 THEN valid_from END AS valid_from," + "CASE WHEN length(valid_until)<=128 THEN valid_until END AS valid_until," + "(length(COALESCE(valid_from,''))>128 OR length(COALESCE(valid_until,''))>128) AS oversized " + f"FROM claims NOT INDEXED WHERE {where} AND id IN ({watched_sql}) LIMIT ?", + [*params, *watched, MAX_CLAIMS + 1], + )) + temporal_bytes = sum(len(json.dumps(dict(row)).encode()) for row in temporal_rows) + if (len(temporal_rows) > MAX_CLAIMS or temporal_bytes > byte_limit + or any(row["oversized"] for row in temporal_rows)): + return {}, False, {"source_bytes": byte_limit} + temporal_instants = _validity_instants(temporal_rows) + temporal_ids = sorted({id for id, boundary in temporal_instants if instant <= boundary <= now_instant}) queries = [ "SELECT changes.claim_id AS id FROM project_source_changes changes " "INDEXED BY idx_project_source_changes_seq " @@ -330,14 +361,10 @@ def _read_claims( f"OR changes.claim_id IN ({watched_sql}))", ] changes_params: list[Any] = [cursor["seq"], source_cursor["seq"], *params, *watched] - for column, index in (("updated_at", "updated"), ("valid_from", "valid_from"), - ("valid_until", "valid_until")): - temporal = column != "updated_at" - queries.append( - f"SELECT id FROM claims INDEXED BY idx_project_claims_{index} " - f"WHERE {where} AND {column} >= ?" + (f" AND {column} <= ?" if temporal else "") - ) - changes_params.extend([*params, lower_bound, *([now] if temporal else [])]) + if temporal_ids: + queries.append("SELECT id FROM claims NOT INDEXED WHERE id IN (" + + ",".join("?" for _ in temporal_ids) + ")") + changes_params.extend(temporal_ids) queries.append(f"SELECT id FROM claims NOT INDEXED WHERE {where} AND id > ?") changes_params.extend([*params, max_seen_id or 0]) changed_ids = [row[0] for row in conn.execute( @@ -393,7 +420,7 @@ def _read_claims( [*params, limit + 1], ) rows = [] - total = 0 + total = temporal_bytes for row in cursor: if len(rows) >= limit: fingerprint = {"source_bytes": total} @@ -453,16 +480,12 @@ def _read_claims( item = _claim_item(dict(row), citations.get(row["id"], [])) if item: items[item["key"]] = item - now = utc_now() - boundaries = [ - str(row[name]) - for row in rows - for name in ("valid_from", "valid_until") - if row[name] and str(row[name]) > now - ] + now = datetime.fromisoformat(utc_now().replace("Z", "+00:00")) + boundaries = [boundary for _, boundary in [*temporal_instants, *_validity_instants(rows)] + if boundary > now] fingerprint = { "data_version": conn.execute("PRAGMA data_version").fetchone()[0], - "next_temporal_boundary": min(boundaries) if boundaries else None, + "next_temporal_boundary": min(boundaries).isoformat() if boundaries else None, "source_bytes": total, "changed_ids": changed_ids, } diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index d50ce376..c02e6fbb 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -447,3 +447,55 @@ def traced_connect(*args, **kwargs): assert snapshot["coverage"]["__scan_fence__"]["watched_ids"] == [1] query = next(query for query in observed if "FROM claims WHERE" in query) assert "CASE WHEN" in query and "ELSE NULL END AS text" in query + +@pytest.mark.parametrize("field", ["valid_from", "valid_until"]) +@pytest.mark.parametrize("boundary", [ + "2026-10-22T06:01:00-03:00", + "2026-10-22T11:01:00+02:00", + "2026-10-22T09:01:00Z", + "2026-10-22T09:01:00.000001+00:00", + "2026-10-22T09:01:00.000001Z", +]) +def test_incremental_iso_temporal_instants_match_full_snapshot(project_config, monkeypatch, field, boundary): + from memorymaster.profile import project_repository as repository + + instant = datetime(2026, 10, 22, 9, tzinfo=timezone.utc) + original_current = repository.claim_is_temporally_current + monkeypatch.setattr(repository, "utc_now", lambda: instant.isoformat()) + monkeypatch.setattr(repository, "claim_is_temporally_current", lambda row: original_current(row, now=instant)) + claim(project_config, **{field: boundary}) + baseline = load_project_snapshot(project_config) + # Keep exact original ISO strings in support identity, independent of comparison normalization. + with sqlite3.connect(project_config.db_path) as conn: + conn.row_factory = sqlite3.Row + row = dict(conn.execute("SELECT * FROM claims WHERE id=1").fetchone()) + citations = [dict(value) for value in conn.execute( + "SELECT id,claim_id,source,locator,excerpt FROM citations WHERE claim_id=1")] + instant = datetime(2026, 10, 22, 9, 2, tzinfo=timezone.utc) + incremental = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + full = load_project_snapshot(project_config) + assert incremental["complete"] and full["complete"] + assert incremental["coverage"] == full["coverage"] + assert baseline["next_temporal_boundary"] == datetime.fromisoformat(boundary.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() + assert incremental["next_temporal_boundary"] is None + if field == "valid_from": + expected = repository._claim_item(row, citations)["support"] + assert incremental["coverage"]["claim:1"] == expected + row[field] = datetime.fromisoformat(boundary.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() + if row[field] != boundary: + assert repository._claim_item(row, citations)["support"]["hash"] != expected["hash"] + + +def test_incremental_keeps_unchanged_future_temporal_boundary(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + instant = datetime(2026, 10, 22, 9, tzinfo=timezone.utc) + original_current = repository.claim_is_temporally_current + monkeypatch.setattr(repository, "utc_now", lambda: instant.isoformat()) + monkeypatch.setattr(repository, "claim_is_temporally_current", lambda row: original_current(row, now=instant)) + claim(project_config, valid_until="2026-10-22T06:10:00-03:00") + baseline = load_project_snapshot(project_config) + instant = datetime(2026, 10, 22, 9, 2, tzinfo=timezone.utc) + incremental = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert incremental["complete"] + assert incremental["next_temporal_boundary"] == "2026-10-22T09:10:00+00:00" From 10c1b3fd0fcde7a16c0eecf6a2b6537aa58e4bca Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:21:36 -0300 Subject: [PATCH 23/51] fix(T-1703): reclaim unused shared safety bytes and label purpose omissions --- memorymaster/profile/project_projection.py | 13 +++++---- tests/test_project_projection.py | 31 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index 24ab209d..3b237331 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -216,12 +216,15 @@ def render_project_projection( # literal excerpts keeps references and support identities intact. safety = [item for item in selected if item["section"] == "Purpose" or item.get("pinned")] remaining = 400 - for item in safety: - cap = min(remaining, 400 // max(1, len(safety))) - if cap < 4: + # Fund short excerpts first, returning their unused share to longer ones. + # Stable sorting leaves equal-sized items deterministic and display order intact. + for index, item in enumerate(sorted(safety, key=lambda item: len(item["text"].encode("utf-8")))): + cap = remaining // (len(safety) - index) + if cap < 4 and len(item["text"].encode("utf-8")) > cap: selected.remove(item) - omissions["pinned_constraints"] = omissions.get("pinned_constraints", 0) + 1 - diagnostics.append("m2_not_ready:pinned_constraints_trimmed") + key = "purpose" if item["section"] == "Purpose" else "pinned_constraints" + omissions[key] = omissions.get(key, 0) + 1 + diagnostics.append(f"m2_not_ready:{key}_trimmed") continue text = item["text"] while len(text.encode()) > cap: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 57af0875..b3c40e08 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -18,6 +18,37 @@ project_config = _project_config +@pytest.mark.parametrize("purpose_first", [True, False]) +def test_safety_budget_reclaims_unused_bytes_for_longer_constraint(project_config, purpose_first): + from memorymaster.profile.project_projection import render_project_projection + + constraint = "Keep backups. " * 17 + purpose = {"key": "claim:1", "section": "Purpose", "text": "Purpose", + "support": {"kind": "claim", "id": 1}} + pinned = {"key": "claim:2", "section": "Constraints", "text": constraint, + "pinned": True, "support": {"kind": "claim", "id": 2}} + safety = [purpose, pinned] if purpose_first else [pinned, purpose] + decisions = [{"key": f"claim:{i}", "section": "Decisions", "text": "Decision", + "support": {"kind": "claim", "id": i}} for i in range(3, 6)] + projection = render_project_projection(project_config, safety + decisions, + generation="fixture", read_at="2026-10-22T09:00:00Z") + assert [item["text"] for item in projection.items[:2]] == [item["text"] for item in safety] + assert projection.status == "ready" + assert not projection.manifest["omissions"] + + +def test_safety_budget_counts_dropped_purpose_separately(project_config): + from memorymaster.profile.project_projection import render_project_projection + + items = [{"key": f"claim:{i}", "section": "Purpose" if i == 0 else "Constraints", + "pinned": i != 0, "text": "Safety", + "support": {"kind": "claim", "id": i}} for i in range(101)] + projection = render_project_projection(project_config, items, + generation="fixture", read_at="2026-10-22T09:00:00Z") + assert projection.manifest["omissions"]["purpose"] == 1 + assert "m2_not_ready:purpose_trimmed" in projection.manifest["diagnostics"] + + @pytest.mark.parametrize("changed_count", [0, 1, 3]) def test_session_start_incremental_counts_only_changed_rows_and_citations(project_config, monkeypatch, changed_count): from memorymaster.profile import project_repository as repository From 205a186978f056b627f319f4433868143e28f8b4 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:21:54 -0300 Subject: [PATCH 24/51] fix(T-1703): replay session provenance across cursor chunks --- .../hooks/memorymaster-session-end.py | 14 ++---- memorymaster/surfaces/session_end_ingest.py | 18 +++++-- tests/test_session_end_capture_hook.py | 47 ++++++++++++++++++- 3 files changed, 63 insertions(+), 16 deletions(-) diff --git a/memorymaster/config_templates/hooks/memorymaster-session-end.py b/memorymaster/config_templates/hooks/memorymaster-session-end.py index 451f4838..12359a73 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-end.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-end.py @@ -3,8 +3,6 @@ import json import os import sys -import tempfile -from pathlib import Path PROJECT_ROOT = "__MEMORYMASTER_PROJECT_ROOT__" DB_PATH = os.path.join(PROJECT_ROOT, "memorymaster.db") @@ -48,12 +46,11 @@ def main(): reservation = ledger.reserve_llm(provider, session_id, "session-end-distill") if reservation is None: return - temp_path = "" try: - with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".jsonl", delete=False) as handle: - handle.write(chunk.text) - temp_path = handle.name - run(DB_PATH, temp_path, source_agent="session-end-hook", cwd=cwd) + # Reconstruct structural provenance from the chronological source, + # but select evidence only within this cursor's complete records. + run(DB_PATH, transcript_path, source_agent="session-end-hook", cwd=cwd, + start_offset=chunk.start_offset, end_offset=chunk.end_offset) ledger.finish_llm(reservation, input_bytes=len(chunk.text.encode("utf-8")), output_bytes=0, outcome="ok") ledger.commit_cursor(chunk) except Exception: # swallow-ok: hook quiet-by-contract; `capture_usage.outcome` @@ -61,9 +58,6 @@ def main(): # Guardar la causa pide una columna nueva en capture_usage, que es otro # subsistema. Deuda anotada, no escondida. ledger.finish_llm(reservation, input_bytes=len(chunk.text.encode("utf-8")), output_bytes=0, outcome="error") - finally: - if temp_path: - Path(temp_path).unlink(missing_ok=True) except Exception: return diff --git a/memorymaster/surfaces/session_end_ingest.py b/memorymaster/surfaces/session_end_ingest.py index 7c1b124c..53e978bc 100644 --- a/memorymaster/surfaces/session_end_ingest.py +++ b/memorymaster/surfaces/session_end_ingest.py @@ -29,19 +29,23 @@ def _sensitive(claim: dict[str, Any]) -> bool: return bool(redact_text(joined)[1]) -def _assistant_text(transcript_path: str) -> str: +def _assistant_text(transcript_path: str, *, start_offset: int = 0, end_offset: int | None = None) -> str: from memorymaster.dreaming.capture import _conversation_record path = Path(transcript_path) if not path.is_file(): return "" messages: list[str] = [] - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + with path.open("rb") as handle: + lines = handle.read(end_offset if end_offset is not None else -1).splitlines(keepends=True) state: dict[str, Any] = {} eligible: list[tuple[int, str]] = [] + offset = 0 for index, line in enumerate(lines): + in_increment = offset >= start_offset + offset += len(line) try: - entry = json.loads(line) + entry = json.loads(line.decode("utf-8", errors="replace")) except json.JSONDecodeError: continue if not isinstance(entry, dict): @@ -57,7 +61,9 @@ def _assistant_text(transcript_path: str) -> str: role = "assistant" entry = {**entry, "message": payload, "type": role or entry.get("type", "")} record = _conversation_record(entry, provider, state=state) - if record is not None and record[0] == "assistant": + # Replay old records for provenance only. Extract solely from the new + # committed byte interval; do not re-distill earlier authored content. + if in_increment and record is not None and record[0] == "assistant": eligible.extend((index, span.text) for span in record[1]) for index, content in reversed(eligible): if index < len(lines) - 200: @@ -126,8 +132,10 @@ def run( *, source_agent: str, cwd: str | None, + start_offset: int = 0, + end_offset: int | None = None, ) -> int: - text = _assistant_text(transcript_path) + text = _assistant_text(transcript_path, start_offset=start_offset, end_offset=end_offset) return ingest_learnings( db_path, _distill(text), source_agent=source_agent, cwd=cwd ) if text else 0 diff --git a/tests/test_session_end_capture_hook.py b/tests/test_session_end_capture_hook.py index 83b101d2..1f80c760 100644 --- a/tests/test_session_end_capture_hook.py +++ b/tests/test_session_end_capture_hook.py @@ -18,7 +18,7 @@ def test_session_end_distills_once_and_cursor_survives_restart(tmp_path: Path) - (script_dir / "__init__.py").write_text("", encoding="utf-8") (script_dir / "agent_session_end_ingest.py").write_text( """from pathlib import Path -def run(db_path, transcript_path, *, source_agent, cwd): +def run(db_path, transcript_path, *, source_agent, cwd, **kwargs): Path(cwd, 'distilled.txt').write_text(Path(transcript_path).read_text(encoding='utf-8'), encoding='utf-8') return 1 """, @@ -72,3 +72,48 @@ def test_lifecycle_receipt_independent_of_distillation(monkeypatch): monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps({"session_id": "fixture", "cwd": "explicit"}))) module.main() assert recorded == [{"session_id": "fixture", "cwd": "explicit"}] + + +def test_incremental_hook_restores_prior_tool_provenance_until_genuine_user(tmp_path, monkeypatch): + import importlib.util + import io + + from memorymaster.core.capture_control import CaptureLedger + from memorymaster.profile import project_runtime + from memorymaster.surfaces import session_end_ingest + + spec = importlib.util.spec_from_file_location("provenance_end_hook", TEMPLATE) + hook = importlib.util.module_from_spec(spec) + spec.loader.exec_module(hook) + state_path = tmp_path / "capture.db" + monkeypatch.setenv("MEMORYMASTER_CAPTURE_STATE_DB", str(state_path)) + monkeypatch.setenv("MEMORYMASTER_SESSION_END_CAPTURE", "1") + monkeypatch.setenv("MEMORYMASTER_LLM_PROVIDER", "fake") + monkeypatch.setattr(project_runtime, "record_session_end", lambda data: None) + distilled = [] + monkeypatch.setattr(session_end_ingest, "_distill", lambda text: distilled.append(text) or []) + monkeypatch.setattr(session_end_ingest, "ingest_learnings", lambda *args, **kwargs: 0) + transcript = tmp_path / "session.jsonl" + first = {"role": "assistant", "content": [{"type": "tool_use", "id": "recall", "name": "mcp__memorymaster__query_memory"}]} + transcript.write_text(json.dumps(first) + "\n", encoding="utf-8") + payload = {"session_id": "fixture", "cwd": str(tmp_path), "transcript_path": str(transcript)} + + def invoke(): + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(payload))) + hook.main() + + invoke() + assert not distilled + assert CaptureLedger(state_path).read_increment(transcript, "fixture:session-end").text == "" + with transcript.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"role": "assistant", "content": "Unmarked recalled echo should never be distilled into a new claim."}) + "\n") + invoke() + assert not distilled, "The second invocation forgot the committed memory-tool provenance" + with transcript.open("a", encoding="utf-8") as handle: + for role, text in [("user", "Make a fresh authored decision for this database."), + ("assistant", "Use SQLite transactions for the newly authored database design.")]: + handle.write(json.dumps({"role": role, "content": text}) + "\n") + invoke() + assert len(distilled) == 1 + assert "SQLite transactions" in distilled[0] + assert "Unmarked recalled echo" not in distilled[0] From 6cf7b99a0a0182ef32cc73ac7ee32498dd760f74 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:22:57 -0300 Subject: [PATCH 25/51] fix(T-1703): report sanitized session receipt failures --- .../hooks/memorymaster-session-end.py | 11 +++- tests/test_session_end_capture_hook.py | 55 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/memorymaster/config_templates/hooks/memorymaster-session-end.py b/memorymaster/config_templates/hooks/memorymaster-session-end.py index 12359a73..2fa8f57e 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-end.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-end.py @@ -2,6 +2,7 @@ import json import os +import sqlite3 import sys PROJECT_ROOT = "__MEMORYMASTER_PROJECT_ROOT__" @@ -27,8 +28,14 @@ def main(): from memorymaster.profile.project_runtime import record_session_end record_session_end(data) - except Exception: - pass # Lifecycle failure must not suppress independently authorized capture. + except (ImportError, OSError, RuntimeError, ValueError, sqlite3.Error) as exc: + # Report only types: exception messages can carry session text, paths, + # identities or credentials. Capture has its own activation decision. + cause = exc.__cause__ or exc.__context__ or exc + print( + f"[MemoryMaster] session-end receipt error: type={type(exc).__name__} cause={type(cause).__name__}", + file=sys.stderr, + ) if not _enabled("MEMORYMASTER_SESSION_END_CAPTURE", True): return transcript_path = data.get("transcript_path", "") diff --git a/tests/test_session_end_capture_hook.py b/tests/test_session_end_capture_hook.py index 1f80c760..c9bdde94 100644 --- a/tests/test_session_end_capture_hook.py +++ b/tests/test_session_end_capture_hook.py @@ -6,6 +6,8 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] TEMPLATE = ROOT / "memorymaster" / "config_templates" / "hooks" / "memorymaster-session-end.py" @@ -117,3 +119,56 @@ def invoke(): assert len(distilled) == 1 assert "SQLite transactions" in distilled[0] assert "Unmarked recalled echo" not in distilled[0] + + +@pytest.mark.parametrize("capture_enabled", [False, True]) +def test_lifecycle_receipt_failure_reports_sanitized_cause_and_continues_capture(tmp_path, monkeypatch, capsys, capture_enabled): + import importlib.util + import io + + from memorymaster.profile import project_runtime + from memorymaster.surfaces import session_end_ingest + + spec = importlib.util.spec_from_file_location("observable_end_hook", TEMPLATE) + hook = importlib.util.module_from_spec(spec) + spec.loader.exec_module(hook) + monkeypatch.setenv("MEMORYMASTER_SESSION_END_CAPTURE", "1" if capture_enabled else "0") + monkeypatch.setenv("MEMORYMASTER_CAPTURE_STATE_DB", str(tmp_path / "capture.db")) + monkeypatch.setenv("MEMORYMASTER_LLM_PROVIDER", "fake") + + def fail_receipt(data): + if not capture_enabled: + raise RuntimeError("fixture failure") + try: + raise OSError("private underlying cause") + except OSError as exc: + raise RuntimeError("fixture failure") from exc + + monkeypatch.setattr(project_runtime, "record_session_end", fail_receipt) + captures = [] + monkeypatch.setattr(session_end_ingest, "run", lambda *args, **kwargs: captures.append(kwargs) or 0) + transcript = tmp_path / "session.jsonl" + transcript.write_text(json.dumps({"role": "assistant", "content": "A separately authorized authored learning about SQLite transactions."}) + "\n", encoding="utf-8") + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps({"session_id": "private-session-identifier", "cwd": str(tmp_path), "transcript_path": str(transcript)}))) + hook.main() + diagnostic = capsys.readouterr().err + assert "type=RuntimeError" in diagnostic + assert f"cause={'OSError' if capture_enabled else 'RuntimeError'}" in diagnostic + assert "fixture failure" not in diagnostic and "private" not in diagnostic + assert str(tmp_path) not in diagnostic + assert len(captures) == int(capture_enabled) + + +def test_lifecycle_receipt_unexpected_programming_error_is_not_swallowed(monkeypatch): + import importlib.util + import io + + from memorymaster.profile import project_runtime + + spec = importlib.util.spec_from_file_location("unexpected_end_hook", TEMPLATE) + hook = importlib.util.module_from_spec(spec) + spec.loader.exec_module(hook) + monkeypatch.setattr(project_runtime, "record_session_end", lambda data: (_ for _ in ()).throw(TypeError("programming error"))) + monkeypatch.setattr(sys, "stdin", io.StringIO("{}")) + with pytest.raises(TypeError): + hook.main() From 1944a5b47b064b5874a3fdf5df45c207dde20be5 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:23:13 -0300 Subject: [PATCH 26/51] fix(T-1703): enforce exact UTF-8 card reservations from structured slots --- memorymaster/profile/project_projection.py | 39 ++++++++++++++++------ memorymaster/profile/project_repository.py | 1 + tests/test_project_projection.py | 38 +++++++++++++++++++++ 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index 3b237331..7d240844 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -19,6 +19,7 @@ GENERATED_CLOSER as CLOSER, ) from memorymaster.profile.project_repository import ( + _display, digest, load_project_snapshot, utc_now, @@ -176,18 +177,21 @@ def _line(item: dict) -> str: def _fit_card_text(item: dict, byte_budget: int) -> str: """Keep the current ID/state slot; titles consume only its remaining UTF-8 bytes.""" - text = item["text"] - if len(text.encode("utf-8")) <= byte_budget: - return text - marker = f"[{item['support']['state']}]" - end = text.find(marker) - if end < 0: + support = item["support"] + if not all(isinstance(support.get(key), str) and support[key].strip() for key in ("id", "state")): raise ValueError("invalid_card_rendering") - prefix = text[:end + len(marker)] + prefix = f"{_display(support['id'], 48)} [{_display(support['state'], 24)}]" available = byte_budget - len(prefix.encode("utf-8")) + if available < 0: + raise ValueError("card_prefix_exceeds_byte_budget") + title = item.get("title") + if not isinstance(title, str): + raise ValueError("invalid_card_rendering") + text = prefix + " " + title + if len(text.encode("utf-8")) <= byte_budget: + return text if available < 5: return prefix - title = text[end + len(marker):].strip() retained = title.encode("utf-8")[:available - 4].decode("utf-8", "ignore") return prefix + " " + retained + "…" @@ -283,7 +287,13 @@ def size(payload: str) -> int: text = item["text"] minimum = 48 if item["section"] == "Open cards" else 80 if item["section"] == "Open cards": - item["text"] = _fit_card_text(item, minimum) + try: + item["text"] = _fit_card_text(item, minimum) + except ValueError as exc: + if str(exc) != "card_prefix_exceeds_byte_budget": + raise + # Keep its mandatory slots until normal budget eviction + # can remove the card with an explicit omission receipt. elif len(text) > minimum: item["text"] = text[: minimum - 1] + "…" compacted = True @@ -518,7 +528,16 @@ def read_project_projection(config: ProjectProjectionConfig) -> ProjectProjectio else: # A title edit cannot take space reserved for an unrelated card. reserved = len(record["text"].encode("utf-8")) - item = {**item, "text": _fit_card_text(item, reserved + card_spare_bytes)} + try: + fitted = _fit_card_text(item, reserved + card_spare_bytes) + except ValueError as exc: + if str(exc) != "card_prefix_exceeds_byte_budget": + raise + omissions["open_cards"] = omissions.get("open_cards", 0) + 1 + diagnostics.append("m2_not_ready:open_cards_trimmed") + card_spare_bytes += reserved + continue + item = {**item, "text": fitted} card_spare_bytes += reserved - len(item["text"].encode("utf-8")) selected.append(item) elif "m2_not_ready:cards_changed" not in diagnostics: diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index 89a131dc..ff487c6f 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -185,6 +185,7 @@ def read_bytes(path: Path) -> bytes: "key": "card:" + card_id, "section": "Open cards", "text": f"{_display(card_id, 48)} [{_display(selected['state'], 24)}] {_display(selected['title'], 80)}", + "title": _display(selected["title"], 80), "support": support, "open": selected["state"] in OPEN_STATES, "sort": [selected["state"], card_id], diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index b3c40e08..74402530 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -49,6 +49,44 @@ def test_safety_budget_counts_dropped_purpose_separately(project_config): assert "m2_not_ready:purpose_trimmed" in projection.manifest["diagnostics"] +def test_fit_card_text_rejects_review_prefix_that_exceeds_reservation(): + from memorymaster.profile.project_projection import _fit_card_text + + with pytest.raises(ValueError): + _fit_card_text({"text": "T-123456789 [running] title", "support": {"state": "running"}}, 8) + + +@pytest.mark.parametrize("card_id,state", [("T-123456789", "running"), ("卡😀", "ready"), + ("T-[ready]", "ready")]) +@pytest.mark.parametrize("extra", [-1, 0, 1, 4, 5, 20]) +def test_fit_card_text_enforces_utf8_reservations_from_structured_slots(card_id, state, extra): + from memorymaster.profile.project_projection import _fit_card_text + from memorymaster.profile.project_repository import _display + + prefix = f"{_display(card_id, 48)} [{_display(state, 24)}]" + budget = len(prefix.encode("utf-8")) + extra + title = "Título 😀 " * 20 + item = {"text": f"{prefix} {title}", "title": title, + "support": {"kind": "card", "id": card_id, "state": state}} + if extra < 0: + with pytest.raises(ValueError, match="card_prefix_exceeds_byte_budget"): + _fit_card_text(item, budget) + else: + result = _fit_card_text(item, budget) + assert result.startswith(prefix) + assert len(result.encode("utf-8")) <= budget + + +def test_oversized_card_prefix_is_omitted_with_diagnostic(project_config): + tasks = [{"id": "😀" * 48, "repo": "demo", "state": "running", "title": "Title"} + for _ in range(1)] + path = project_config.ledger_root / "tasks.json" + path.write_text(json.dumps({"complete": True, "tasks": tasks}), encoding="utf-8") + built = build_project_projection(replace(project_config, budget_bytes=400)) + assert len(built.markdown.encode("utf-8")) <= 400 + assert "m2_not_ready:open_cards_trimmed" in built.payload + + @pytest.mark.parametrize("changed_count", [0, 1, 3]) def test_session_start_incremental_counts_only_changed_rows_and_citations(project_config, monkeypatch, changed_count): from memorymaster.profile import project_repository as repository From ef2c08ae6731a50b13e83fa294ef034e84803949 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 17:24:23 -0300 Subject: [PATCH 27/51] fix(T-1703): preserve below-TTL unclosed session card changes --- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- memorymaster/profile/project_sessions.py | 23 +++++++++++++- tests/test_project_sessions.py | 40 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index dad20cf5..70066c78 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -144,7 +144,7 @@ "console_entrypoints": 9, "mcp_tools": 51, "ops_cli_commands": 5, - "pytest_test_functions": 5874 + "pytest_test_functions": 5889 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 15cd0a78..f3fe3c97 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -13,7 +13,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **127** - Operations CLI commands: **5** - Console entrypoints: **9** -- Pytest source test functions: **5874** +- Pytest source test functions: **5889** ## MCP tools diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py index c597282a..2932c5d1 100644 --- a/memorymaster/profile/project_sessions.py +++ b/memorymaster/profile/project_sessions.py @@ -125,7 +125,8 @@ def factual(value): # A currently authorized closed card may be carried in items outside open-card coverage. changed.update(key for key in removed if key in items) withdrawn = len(removed - changed) - complete = bool(current.get("complete")) and projection_coverage is not None + complete = (bool(current.get("complete")) and projection_coverage is not None + and "overlap_unresolved" not in diagnostic) status = "partial" if not complete else "first_recorded_session" if baseline is None else "ok" heading = "Prior-session changes and pending projection changes; current-state comparison." if not comparisons: @@ -195,6 +196,13 @@ def _select_prior(state: dict, now: datetime) -> tuple[dict | None, str]: now - _time(latest["end_at"]) >= ACTIVE_TTL and _time(fallback["start_at"]) > _time(latest["start_at"]))): return fallback, "prior_session_unclosed" + # A start alone cannot distinguish a crashed predecessor from overlap. + # Preserve its comparison immediately, with an explicit partial result; + # waiting for TTL can lose closed cards after the projection refreshes. + unclosed = [record for record in state["active"].values() + if latest is None or _time(record["start_at"]) > _time(latest["end_at"])] + if unclosed: + return max(unclosed, key=lambda record: record["seq"]), "prior_session_unclosed; overlap_unresolved" return latest, "" @@ -228,6 +236,17 @@ def record_project_session_start(config, session_id: str, *, projection: dict | else: prior_id = prior["baseline"] if prior else None baseline = copy.deepcopy(state["baselines"].get(prior_id)) + if not duplicate and "overlap_unresolved" in diagnostic: + # Compare all unresolved starts: the newest may already + # omit a card retained by an earlier overlapping start. + baseline = {} + latest = state["latest_end"] + for record in sorted(state["active"].values(), key=lambda value: value["seq"]): + if latest is None or _time(record["start_at"]) > _time(latest["end_at"]): + baseline.update(state["baselines"][record["baseline"]]) + if len(_encoded(baseline)) > MAX_BASELINE_BYTES: + return {"status": "baseline_capacity"} + prior_id = _digest(baseline) if snapshot_loader is None: if projection is None and baseline is None: return {"status": "baseline_unavailable"} @@ -274,6 +293,8 @@ def load_delta_item(key): if len(state["active"]) >= 16: return {"status": "active_capacity"} baseline_id = _digest(coverage) + if prior_id is not None: + state["baselines"][prior_id] = baseline state["baselines"][baseline_id] = coverage state["seq"] += 1 state["active"][sid] = {"session": sid, "start_at": now.isoformat(), diff --git a/tests/test_project_sessions.py b/tests/test_project_sessions.py index 2d4fe017..7f04d3ec 100644 --- a/tests/test_project_sessions.py +++ b/tests/test_project_sessions.py @@ -240,3 +240,43 @@ def test_stale_active_start_cannot_override_a_recent_explicit_end(tmp_path): snapshot_loader=lambda: changed, now=NOW + timedelta(hours=20, minutes=1)) assert "retained overlapping work" in result["delta"].text assert "prior_session_unclosed" not in result["delta"].text + + +def test_unclosed_session_before_ttl_retains_closed_card_after_refresh(tmp_path): + cfg = config(tmp_path) + support = {"kind": "card", "id": "T-1", "hash": "open", "state": "ready"} + opened = {"key": "card:T-1", "support": support, "text": "T-1 [ready] Visible card", "open": True} + original = {"coverage": {"card:T-1": support}, "items": {"card:T-1": opened}, + "complete": True, "read_at": NOW.isoformat()} + projection = {"source_coverage": original["coverage"], "read_at": original["read_at"]} + assert sessions.record_project_session_start(cfg, "S", projection=projection, + snapshot_loader=lambda: original, now=NOW)["status"] == "ok" + closed = {**opened, "support": {**support, "hash": "closed", "state": "done"}, + "text": "T-1 [done] Visible card", "open": False} + refreshed = {"coverage": {}, "items": {"card:T-1": closed}, "complete": True, + "read_at": (NOW + timedelta(minutes=9)).isoformat()} + result = sessions.record_project_session_start(cfg, "T", projection={ + "source_coverage": {}, "read_at": refreshed["read_at"]}, + snapshot_loader=lambda: refreshed, now=NOW + timedelta(minutes=10)) + delta = result["delta"] + assert delta.supports and delta.supports[0]["support"] == closed["support"] + assert "T-1 [done] Visible card" in delta.text + assert "prior_session_unclosed" in delta.text and "overlap_unresolved" in delta.text + assert delta.status == "partial" + assert "first recorded session" not in delta.text and "No changes" not in delta.text + assert sessions.last_project_session_end(cfg) is None + # A start alone cannot establish whether its predecessor crashed or is + # overlapping. Keep both live receipts; a later genuine end still works. + state = sessions.read_session_state(cfg) + assert len(state["active"]) == 2 and not state["completed"] + again = sessions.record_project_session_start(cfg, "U", projection={ + "source_coverage": {}, "read_at": refreshed["read_at"]}, + snapshot_loader=lambda: refreshed, now=NOW + timedelta(minutes=10, seconds=30)) + assert again["delta"].status == "partial" + assert again["delta"].supports[0]["support"] == closed["support"] + assert sessions.record_project_session_end(cfg, "S", now=NOW + timedelta(minutes=11)) + latest = sessions.record_project_session_start(cfg, "V", projection={ + "source_coverage": {}, "read_at": refreshed["read_at"]}, + snapshot_loader=lambda: refreshed, now=NOW + timedelta(minutes=12)) + assert latest["delta"].status == "ok" + assert "overlap_unresolved" not in latest["delta"].text From 00437881a421f4fb60bb05bd24f97b0c1bf1d062 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:07:19 -0300 Subject: [PATCH 28/51] fix(T-1703): reconstruct cached projection text from authorized sources --- memorymaster/profile/project_projection.py | 56 +++++++++++++++------- tests/test_project_projection.py | 32 +++++++++++++ 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index 7d240844..4ad6bb18 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -453,6 +453,23 @@ def read_manifest(config: ProjectProjectionConfig) -> dict | None: or len(manifest["source_coverage"]) > 769 ): return None + # A pair hash proves internal consistency, never authority. Only fixed + # status vocabulary may be copied from the disposable cache to output. + allowed_diagnostics = { + "source_incomplete", "ledger_unavailable", "ledger_incomplete", + "claims_unavailable", "coverage_limit", "inactive", + "incremental_scan_fence_incomplete", + *(f"m2_not_ready:{reason}" for reason in ( + "decisions_incomplete", "purpose_trimmed", "pinned_constraints_trimmed", + "open_cards_trimmed", "decisions_trimmed", "ledger_incomplete", + "cards_changed", "context_withdrawn", + )), + } + if (any(value not in allowed_diagnostics for value in manifest["diagnostics"]) + or any(key not in {"purpose", "constraints", "decisions", "open_cards", "gotchas", "pinned_constraints"} + or type(value) is not int or value < 0 + for key, value in manifest["omissions"].items())): + return None citations = 0 keys = set() for record in records: @@ -523,29 +540,32 @@ def read_project_projection(config: ProjectProjectionConfig) -> ProjectProjectio # Modified still-open cards are safe to render from the authorized # ledger snapshot and retain their reserved place in the payload. if item and item.get("open"): - if item["support"] == record["supports"][0]: - item = {**item, "text": record["text"]} - else: + reserved = len(record["text"].encode("utf-8")) + unchanged = item["support"] == record["supports"][0] + try: # A title edit cannot take space reserved for an unrelated card. - reserved = len(record["text"].encode("utf-8")) - try: - fitted = _fit_card_text(item, reserved + card_spare_bytes) - except ValueError as exc: - if str(exc) != "card_prefix_exceeds_byte_budget": - raise - omissions["open_cards"] = omissions.get("open_cards", 0) + 1 - diagnostics.append("m2_not_ready:open_cards_trimmed") - card_spare_bytes += reserved - continue - item = {**item, "text": fitted} - card_spare_bytes += reserved - len(item["text"].encode("utf-8")) + fitted = _fit_card_text(item, reserved + (0 if unchanged else card_spare_bytes)) + except ValueError as exc: + if str(exc) != "card_prefix_exceeds_byte_budget": + raise + omissions["open_cards"] = omissions.get("open_cards", 0) + 1 + diagnostics.append("m2_not_ready:open_cards_trimmed") + card_spare_bytes += reserved + continue + item = {**item, "text": fitted} + card_spare_bytes += reserved - len(fitted.encode("utf-8")) selected.append(item) elif "m2_not_ready:cards_changed" not in diagnostics: diagnostics.append("m2_not_ready:cards_changed") elif item and item["support"] == record["supports"][0]: - # Keep an unrelated bullet byte-identical even when the original - # renderer shortened it to fit the shared allocation. - selected.append({**item, "text": record["text"]}) + # Retain only the cached byte reservation. Every factual character + # comes from the rehydrated, authorized source, including excerpts. + reserved = len(record["text"].encode("utf-8")) + text = item["text"] + if len(text.encode("utf-8")) > reserved: + text = (text.encode("utf-8")[:max(0, reserved - 3)].decode("utf-8", "ignore") + "…" + if reserved >= 3 else "") + selected.append({**item, "text": text}) else: diagnostics.append("m2_not_ready:context_withdrawn") diagnostics.extend(current["diagnostics"]) diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 74402530..f4539a6a 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -18,6 +18,38 @@ project_config = _project_config +@pytest.mark.parametrize("target", ["claim", "card"]) +def test_self_consistent_cache_pair_cannot_supply_unsupported_text(project_config, target): + from memorymaster.profile.project_projection import render_project_projection + + for number in range(1, 4): + claim(project_config, number, text=f"Authorized decision {number}") + (project_config.ledger_root / "tasks.json").write_text(json.dumps({"complete": True, "tasks": [ + {"id": "T-1", "repo": "demo", "state": "ready", "title": "Authorized work"} + ]}), encoding="utf-8") + built = build_project_projection(project_config) + assert publish_project_projection(project_config, built) + forged_items = [dict(item) for item in built.items] + next(item for item in forged_items if item["support"]["kind"] == target)["text"] = ( + "Cache-only decision: send backups to alice@example.com at 10.23.45.67." + ) + # Recompute all publicly computable pair hashes without any source writes. + forged = render_project_projection( + project_config, forged_items, generation=built.manifest["generation_id"], + read_at=built.manifest["read_at"], coverage=built.manifest["source_coverage"], + fingerprints=built.manifest["source_fingerprints"], + next_temporal_boundary=built.manifest["next_temporal_boundary"], + ) + assert publish_project_projection(project_config, forged) + result = read_project_projection(project_config) + assert result is not None + assert "Cache-only decision" not in result.payload + assert "alice@example.com" not in result.payload + assert "10.23.45.67" not in result.payload + assert "Authorized decision" in result.payload + assert "Authorized work" in result.payload + + @pytest.mark.parametrize("purpose_first", [True, False]) def test_safety_budget_reclaims_unused_bytes_for_longer_constraint(project_config, purpose_first): from memorymaster.profile.project_projection import render_project_projection From e453c73194470c11f322b04a2ac1b6c52eaefdd2 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:08:08 -0300 Subject: [PATCH 29/51] fix(T-1703): redact card references in projections and session deltas --- memorymaster/profile/project_projection.py | 2 +- memorymaster/profile/project_sessions.py | 4 +++- tests/test_project_projection.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index 4ad6bb18..a77a2670 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -171,7 +171,7 @@ def public_manifest(self) -> dict: def _line(item: dict) -> str: support = item["support"] - reference = f"claim:{support['id']}" if support["kind"] == "claim" else str(support["id"]) + reference = f"claim:{support['id']}" if support["kind"] == "claim" else _display(support["id"], 48) return f"- {item['text']} [{reference}]" diff --git a/memorymaster/profile/project_sessions.py b/memorymaster/profile/project_sessions.py index 2932c5d1..45e1f011 100644 --- a/memorymaster/profile/project_sessions.py +++ b/memorymaster/profile/project_sessions.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Callable +from memorymaster.profile.project_repository import _display + SCHEMA = "memorymaster.project-sessions.v1" MAX_STATE_BYTES = 8 * 1024 * 1024 MAX_BASELINE_BYTES = 128 * 1024 @@ -157,7 +159,7 @@ def render(): if not item or not item.get("support"): continue support = item["support"] - reference = f"#{support['id']}" if support.get("kind") == "claim" else str(support["id"]) + reference = f"#{support['id']}" if support.get("kind") == "claim" else _display(support["id"], 48) chosen.append((f"- {_short(item.get('text', ''), 110)} [{reference}]", support)) omitted -= 1 if len(render().encode("utf-8")) > 500: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index f4539a6a..9bff7e8a 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -18,6 +18,24 @@ project_config = _project_config +@pytest.mark.parametrize("card_id", ["alice@example.com", "10.2.3.4", "T-1\n## injected", "T-[ready]"]) +def test_card_reference_receives_egress_filter(project_config, card_id): + from memorymaster.profile.project_repository import _display, load_project_snapshot + from memorymaster.profile.project_sessions import build_session_delta + + (project_config.ledger_root / "tasks.json").write_text(json.dumps({"complete": True, "tasks": [ + {"id": card_id, "repo": "demo", "state": "ready", "title": "Visible work"} + ]}), encoding="utf-8") + result = build_project_projection(project_config) + snapshot = load_project_snapshot(project_config) + delta = build_session_delta({}, {}, snapshot) + for text in (result.payload, delta.text): + assert f"[{_display(card_id, 48)}]" in text + assert card_id not in text + # Exact identity remains private support metadata for authoritative rehydration. + assert snapshot["items"][f"card:{card_id}"]["support"]["id"] == card_id + + @pytest.mark.parametrize("target", ["claim", "card"]) def test_self_consistent_cache_pair_cannot_supply_unsupported_text(project_config, target): from memorymaster.profile.project_projection import render_project_projection From aad3555aa4038a13b139c568519a97905b6e7afe Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:09:30 -0300 Subject: [PATCH 30/51] fix(T-1703): preserve HTML hidden context across generated exclusions --- memorymaster/capture/adapters.py | 81 ++++++++++++++++++++++++++++++-- tests/test_capture_adapters.py | 29 ++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/memorymaster/capture/adapters.py b/memorymaster/capture/adapters.py index abda9488..ea58c722 100644 --- a/memorymaster/capture/adapters.py +++ b/memorymaster/capture/adapters.py @@ -8,7 +8,9 @@ import mimetypes import os import re +from bisect import bisect_right from dataclasses import dataclass +from html import unescape from html.parser import HTMLParser from pathlib import Path from typing import Protocol @@ -86,21 +88,84 @@ class _VisibleTextParser(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.hidden_depth = 0 + self.hidden_tags: list[str] = [] self.parts: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag.lower() in {"script", "style", "noscript", "template"}: - self.hidden_depth += 1 + self.hidden_tags.append(tag.lower()) + self.hidden_depth = len(self.hidden_tags) def handle_endtag(self, tag: str) -> None: - if tag.lower() in {"script", "style", "noscript", "template"} and self.hidden_depth: - self.hidden_depth -= 1 + if self.hidden_tags and tag.lower() == self.hidden_tags[-1]: + self.hidden_tags.pop() + self.hidden_depth = len(self.hidden_tags) def handle_data(self, data: str) -> None: if not self.hidden_depth and data.strip(): self.parts.append(data.strip()) +class _RetainedHTMLParser(_VisibleTextParser): + """Parse the original document once; map visible data to retained ranges.""" + + def __init__(self, source: str, spans: tuple[SourceSpan, ...]) -> None: + super().__init__() + # Decode entities after mapping their original source ranges. + self.convert_charrefs = False + self.source = source + self.spans = spans + self.ends = [span.end for span in spans] + self.line_starts = [0, *(match.end() for match in re.finditer("\n", source))] + self.mapped: list[list[tuple[int, int, str]]] = [[] for _ in spans] + + def _record(self, data: str, raw_length: int, *, entity: bool = False) -> None: + if self.hidden_depth: + return + line, column = self.getpos() + start = self.line_starts[line - 1] + column + end = start + raw_length + index = bisect_right(self.ends, start) + while index < len(self.spans) and self.spans[index].start < end: + span = self.spans[index] + left, right = max(start, span.start), min(end, span.end) + if entity and (left != start or right != end): + raise CaptureRejected("ambiguous_generated_provenance", "Split HTML entity cannot be attributed safely.") + value = data if entity else data[left - start:right - start] + self.mapped[index].append((left, right, value)) + index += 1 + + def handle_data(self, data: str) -> None: + self._record(data, len(data)) + + def _entity(self, name: str, prefix: str) -> None: + line, column = self.getpos() + start = self.line_starts[line - 1] + column + raw = prefix + name + if self.source[start + len(raw):start + len(raw) + 1] == ";": + raw += ";" + self._record(unescape(raw), len(raw), entity=True) + + def handle_entityref(self, name: str) -> None: + self._entity(name, "&") + + def handle_charref(self, name: str) -> None: + self._entity(name, "&#") + + def values(self) -> list[str]: + result = [] + for parts in self.mapped: + chunks = [] + previous_end = None + for start, end, value in parts: + if chunks and previous_end != start: + chunks.append("\n") + chunks.append(value) + previous_end = end + result.append("".join(chunks).strip()) + return result + + def _digest(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() @@ -124,10 +189,16 @@ def _retained_text(text: str, *, html: bool = False) -> tuple[str, tuple[str, .. raise CaptureRejected("generated_artifact", "Generated-only content cannot become capture evidence.") spans: list[SourceSpan] = [] warning_codes: list[str] = ["generated_spans_excluded"] if clean.excluded_spans else [] - for span in clean.spans: + html_values = [] + if html: + parser = _RetainedHTMLParser(text, clean.spans) + parser.feed(text) + parser.close() + html_values = parser.values() + for index, span in enumerate(clean.spans): if span.start == 0 and span.text.startswith("\ufeff"): span = SourceSpan(1, span.end, span.text[1:]) - value = _html_text(span.text.encode("utf-8")) if html else span.text + value = html_values[index].lstrip("\ufeff") if html else span.text if html and strip_generated_spans(value).excluded_spans: # HTML entity decoding changes offsets. Without a reversible source # map, splitting decoded text would fabricate original attribution. diff --git a/tests/test_capture_adapters.py b/tests/test_capture_adapters.py index 3f45794b..ecf4ed14 100644 --- a/tests/test_capture_adapters.py +++ b/tests/test_capture_adapters.py @@ -51,6 +51,35 @@ def test_generated_only_inline_does_not_create_capture(): assert exc.value.code == "generated_artifact" +@pytest.mark.parametrize("tag", ["script", "style", "noscript", "template"]) +def test_hidden_html_context_survives_generated_span_removal(capture_root, tag): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = (f"<{tag}>hidden before " + GENERATED_OPENER + "generated" + GENERATED_CLOSER + + f"This script-only decision must never become evidence.

Authored note.

") + path = capture_root / "hidden.html" + path.write_text(raw, encoding="utf-8") + envelope = LocalFileAdapter(path).capture() + assert envelope.text == "Authored note." + assert [span.text for span in envelope.retained_spans] == ["Authored note."] + span = envelope.retained_spans[0] + assert raw[span.start:span.end].endswith("

Authored note.

") + + +def test_html_crossing_exclusions_preserves_entities_and_source_spans(capture_root): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = ("

Before & é.

After <note>.

") + path = capture_root / "mapped.html" + path.write_text(raw, encoding="utf-8") + envelope = LocalFileAdapter(path).capture() + assert [span.text for span in envelope.retained_spans] == ["Before & é.", "After ."] + first, second = envelope.retained_spans + assert first.end <= raw.index(GENERATED_OPENER) + assert second.start >= raw.index(GENERATED_CLOSER) + len(GENERATED_CLOSER) + + def test_html_escaped_generated_markers_reject_ambiguous_evidence(capture_root): from html import escape From 2948fb4429ba4b4771e82a4f1e8547e30edba3b9 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:12:34 -0300 Subject: [PATCH 31/51] fix(T-1703): retain normalized temporal boundaries for bounded incremental reads --- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- memorymaster/profile/project_repository.py | 59 +++++++++------ tests/test_project_repository.py | 88 ++++++++++++++++++++++ 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 70066c78..537c4dc2 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -144,7 +144,7 @@ "console_entrypoints": 9, "mcp_tools": 51, "ops_cli_commands": 5, - "pytest_test_functions": 5889 + "pytest_test_functions": 5896 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index f3fe3c97..51592d53 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -13,7 +13,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **127** - Operations CLI commands: **5** - Console entrypoints: **9** -- Pytest source test functions: **5889** +- Pytest source test functions: **5896** ## MCP tools diff --git a/memorymaster/profile/project_repository.py b/memorymaster/profile/project_repository.py index ff487c6f..b01d5bef 100644 --- a/memorymaster/profile/project_repository.py +++ b/memorymaster/profile/project_repository.py @@ -3,6 +3,7 @@ from __future__ import annotations from contextlib import closing +from bisect import bisect_left, bisect_right from datetime import datetime, timezone import hashlib import json @@ -315,8 +316,7 @@ def _read_claims( params: list[Any] = [config.tenant_id, *scopes] where = "tenant_id IS ? AND scope IN (" + ",".join("?" for _ in scopes) + ")" changed_ids = [] - temporal_instants = [] - temporal_bytes = 0 + temporal_boundaries = [] if changed_since: cursor = (prior_fence or {}).get("source_cursor") if (not source_cursor or not isinstance(cursor, dict) @@ -336,24 +336,28 @@ def _read_claims( watched = (prior_fence or {}).get("watched_ids", []) if len(watched) > MAX_CLAIMS or any(type(id) is not int or id <= 0 for id in watched): raise ValueError("invalid_scan_fence") + watched_set = set(watched) watched_sql = ",".join("?" for _ in watched) or "NULL" - # A complete fence contains every previously scanned ID. Read only - # its bounded validity metadata by primary key, then normalize in - # Python: indexed TEXT ranges cannot compare offsets or fractions. - temporal_rows = list(conn.execute( - "SELECT id," - "CASE WHEN length(valid_from)<=128 THEN valid_from END AS valid_from," - "CASE WHEN length(valid_until)<=128 THEN valid_until END AS valid_until," - "(length(COALESCE(valid_from,''))>128 OR length(COALESCE(valid_until,''))>128) AS oversized " - f"FROM claims NOT INDEXED WHERE {where} AND id IN ({watched_sql}) LIMIT ?", - [*params, *watched, MAX_CLAIMS + 1], - )) - temporal_bytes = sum(len(json.dumps(dict(row)).encode()) for row in temporal_rows) - if (len(temporal_rows) > MAX_CLAIMS or temporal_bytes > byte_limit - or any(row["oversized"] for row in temporal_rows)): - return {}, False, {"source_bytes": byte_limit} - temporal_instants = _validity_instants(temporal_rows) - temporal_ids = sorted({id for id, boundary in temporal_instants if instant <= boundary <= now_instant}) + # Protected continuity metadata holds already-normalized future + # boundaries. Never reload every watched row to compare ISO forms. + # Older fences lack this contract and must fail closed for rebuild. + temporal_boundaries = (prior_fence or {}).get("temporal_boundaries") + if ((prior_fence or {}).get("temporal_version") != 1 + or not isinstance(temporal_boundaries, list) + or len(temporal_boundaries) > 2 * MAX_CLAIMS + or any(not isinstance(entry, list) or len(entry) != 2 + or not isinstance(entry[0], str) + or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+00:00", entry[0]) + or type(entry[1]) is not int or entry[1] not in watched_set + for entry in temporal_boundaries) + or temporal_boundaries != sorted(temporal_boundaries)): + raise ValueError("incremental_temporal_fence_unavailable") + lower = bisect_left(temporal_boundaries, instant.astimezone(timezone.utc).isoformat(timespec="microseconds"), + key=lambda entry: entry[0]) + upper = bisect_right(temporal_boundaries, now_instant.astimezone(timezone.utc).isoformat(timespec="microseconds"), + key=lambda entry: entry[0]) + temporal_ids = sorted({id for _, id in temporal_boundaries[lower:upper]}) + temporal_boundaries = temporal_boundaries[upper:] queries = [ "SELECT changes.claim_id AS id FROM project_source_changes changes " "INDEXED BY idx_project_source_changes_seq " @@ -421,7 +425,7 @@ def _read_claims( [*params, limit + 1], ) rows = [] - total = temporal_bytes + total = 0 for row in cursor: if len(rows) >= limit: fingerprint = {"source_bytes": total} @@ -482,11 +486,18 @@ def _read_claims( if item: items[item["key"]] = item now = datetime.fromisoformat(utc_now().replace("Z", "+00:00")) - boundaries = [boundary for _, boundary in [*temporal_instants, *_validity_instants(rows)] - if boundary > now] + # Refresh normalized metadata only for source rows materialized because + # they changed, crossed a boundary or were explicitly requested. + refreshed_ids = set(changed_ids) | set(ids_read) + temporal_boundaries = sorted([ + *[entry for entry in temporal_boundaries if entry[1] not in refreshed_ids], + *[[boundary.isoformat(timespec="microseconds"), id] for id, boundary in _validity_instants(rows) + if boundary > now], + ]) fingerprint = { "data_version": conn.execute("PRAGMA data_version").fetchone()[0], - "next_temporal_boundary": min(boundaries).isoformat() if boundaries else None, + "next_temporal_boundary": (datetime.fromisoformat(temporal_boundaries[0][0]).isoformat() + if temporal_boundaries else None), "source_bytes": total, "changed_ids": changed_ids, } @@ -499,6 +510,8 @@ def _read_claims( "max_seen_id": max([max_seen_id or 0, *watched_ids]), "complete": complete and len(watched_ids) <= MAX_CLAIMS and source_cursor is not None, "source_cursor": source_cursor, + "temporal_version": 1, + "temporal_boundaries": temporal_boundaries, } complete = complete and len(watched_ids) <= MAX_CLAIMS return items, complete, fingerprint diff --git a/tests/test_project_repository.py b/tests/test_project_repository.py index c02e6fbb..4773bcbd 100644 --- a/tests/test_project_repository.py +++ b/tests/test_project_repository.py @@ -82,6 +82,31 @@ def claim(config, id=1, text="Use bounded reads", **changes): ) +@pytest.mark.parametrize("count", [40, 400]) +@pytest.mark.parametrize("changed_count", [0, 1, 3]) +def test_unchanged_baseline_does_not_reload_all_temporal_metadata(project_config, monkeypatch, count, changed_count): + from memorymaster.profile import project_repository as repository + + for number in range(1, count + 1): + claim(project_config, number, text=f"Decision {number}") + baseline = load_project_snapshot(project_config) + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("UPDATE claims SET text='Changed decision' WHERE id<=?", (changed_count,)) + original_row = sqlite3.Row + loaded = [] + + def counting_row(cursor, values): + row = original_row(cursor, values) + if "valid_from" in row.keys(): + loaded.append(row["id"]) + return row + + monkeypatch.setattr(repository.sqlite3, "Row", counting_row) + result = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert result["complete"] + assert loaded == list(range(1, changed_count + 1)), {"metadata_rows_reloaded": len(loaded)} + + def test_confirmed_authorized_current_selection(project_config): config = project_config claim(config) @@ -499,3 +524,66 @@ def test_incremental_keeps_unchanged_future_temporal_boundary(project_config, mo incremental = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) assert incremental["complete"] assert incremental["next_temporal_boundary"] == "2026-10-22T09:10:00+00:00" + + +def test_incremental_temporal_fence_updates_only_changed_and_crossed_rows(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + instant = datetime(2026, 10, 22, 9, tzinfo=timezone.utc) + original_current = repository.claim_is_temporally_current + monkeypatch.setattr(repository, "utc_now", lambda: instant.isoformat()) + monkeypatch.setattr(repository, "claim_is_temporally_current", lambda row: original_current(row, now=instant)) + for number in range(1, 41): + claim(project_config, number, valid_until="2026-10-22T06:10:00-03:00", + **({"valid_from": "2026-10-22T09:04:00Z"} if number == 2 else {})) + baseline = load_project_snapshot(project_config) + normalized_ids = [] + original_instants = repository._validity_instants + + def tracking_instants(rows): + normalized_ids.extend(row["id"] for row in rows) + return original_instants(rows) + + monkeypatch.setattr(repository, "_validity_instants", tracking_instants) + with sqlite3.connect(project_config.db_path) as conn: + conn.execute("UPDATE claims SET valid_until='2026-10-22T09:03:00Z' WHERE id=1") + conn.execute("DELETE FROM claims WHERE id=3") + conn.execute("UPDATE claims SET scope='project:other' WHERE id=4") + instant = datetime(2026, 10, 22, 9, 2, tzinfo=timezone.utc) + first = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert first["complete"] + assert normalized_ids == [1] + assert first["next_temporal_boundary"] == "2026-10-22T09:03:00+00:00" + assert not {3, 4} & {id for _, id in first["coverage"]["__scan_fence__"]["temporal_boundaries"]} + normalized_ids.clear() + instant = datetime(2026, 10, 22, 9, 5, tzinfo=timezone.utc) + second = snapshot_since(project_config, first["coverage"], first["read_at"]) + assert second["complete"] + assert normalized_ids == [1, 2] + assert second["next_temporal_boundary"] == "2026-10-22T09:10:00+00:00" + assert "claim:1" not in second["coverage"] + assert "claim:2" in second["coverage"] + full = load_project_snapshot(project_config) + assert {k: v for k, v in second["coverage"].items() if k.startswith("claim:")} == { + k: v for k, v in full["coverage"].items() if k.startswith("claim:") + } + + +def test_legacy_temporal_fence_fails_closed_without_source_scan(project_config, monkeypatch): + from memorymaster.profile import project_repository as repository + + claim(project_config) + baseline = load_project_snapshot(project_config) + baseline["coverage"]["__scan_fence__"].pop("temporal_boundaries") + statements = [] + original_connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + conn = original_connect(*args, **kwargs) + conn.set_trace_callback(statements.append) + return conn + + monkeypatch.setattr(repository.sqlite3, "connect", traced_connect) + current = snapshot_since(project_config, baseline["coverage"], baseline["read_at"]) + assert not current["complete"] + assert not any("FROM claims" in statement or "FROM citations" in statement for statement in statements) From 52c1ecbc1ea5100ac425b20571b221571f5e47da Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:56:22 -0300 Subject: [PATCH 32/51] T-1703 keep exact support identities private in public manifests --- memorymaster/profile/project_projection.py | 29 ++++++++++++--- tests/test_project_projection.py | 43 ++++++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/memorymaster/profile/project_projection.py b/memorymaster/profile/project_projection.py index a77a2670..57cddf33 100644 --- a/memorymaster/profile/project_projection.py +++ b/memorymaster/profile/project_projection.py @@ -162,11 +162,30 @@ def markdown(self) -> str: @property def public_manifest(self) -> dict: - return { - key: value - for key, value in self.manifest.items() - if key not in ("source_coverage", "source_fingerprints", "tenant_id", "aliases", "context_key") - } + # Export an explicit schema. The private manifest retains the exact + # support identities/versions used for rehydration; ledger metadata is + # never recursively copied onto the CLI surface. The opaque key maps + # back to its private record by hashing that record's support identity. + public = {key: self.manifest[key] for key in ( + "schema", "algorithm_version", "scope", "activation_epoch", + "generation_id", "view_id", "read_at", "checked_at", "output_sha256", + "payload_sha256", "output_bytes", "payload_bytes", "budget_bytes", + "omissions", "diagnostics", "status", "complete", "source_complete", + "next_temporal_boundary", + )} + public["items"] = [] + for record in self.manifest["items"]: + supports = [{"kind": support["kind"], + "id": digest([support["kind"], support["id"]]), + "hash": support["hash"]} for support in record["supports"]] + public["items"].append({ + **{key: record[key] for key in ( + "section", "text", "line", "span", "text_hash", "pinned", + )}, + "key": supports[0]["kind"] + ":" + supports[0]["id"], + "supports": supports, + }) + return public def _line(item: dict) -> str: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 9bff7e8a..14035c7e 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -18,6 +18,49 @@ project_config = _project_config +@pytest.mark.parametrize("version", [None, "version-owner@example.com", {"owner": "version-owner@example.com"}]) +def test_public_card_manifest_does_not_export_redacted_identity(project_config, monkeypatch, capsys, version): + from memorymaster.profile.__main__ import main + from memorymaster.profile.project_repository import load_project_snapshot + from memorymaster.profile.project_sessions import build_session_delta + + card_id = "alice@example.com" + row = {"id": card_id, "repo": "demo", "state": "ready", "title": "Visible work"} + if version is not None: + row["version"] = version + (project_config.ledger_root / "tasks.json").write_text( + json.dumps({"complete": True, "tasks": [row]}), encoding="utf-8") + built = build_project_projection(project_config) + assert publish_project_projection(project_config, built) + view = read_project_projection(project_config) + snapshot = load_project_snapshot(project_config) + delta = build_session_delta({}, {}, snapshot) + assert snapshot["items"][f"card:{card_id}"]["support"]["id"] == card_id + assert built.manifest["items"][0]["supports"][0]["version"] == version + assert built.public_manifest["items"][0]["key"] == view.public_manifest["items"][0]["key"] + outputs = [built.markdown, view.markdown, delta.text, + json.dumps(built.public_manifest), json.dumps(view.public_manifest)] + monkeypatch.setenv("MEMORYMASTER_PROJECT_PROJECTION_ENABLED", "1") + args = ["--db", str(project_config.db_path), "--scope", project_config.scope, + "--tenant", "legacy-null", "--context-key", project_config.context_key, + "--epoch", project_config.epoch, "--project-root", str(project_config.project_root), + "--output-dir", str(project_config.output_root), "--state-dir", str(project_config.state_root), + "--ledger-dir", str(project_config.ledger_root), "--ledger-repo", "demo", "--ledger-filtered"] + for command in ("project-build", "project-read"): + assert main([command, *args]) == 0 + emitted = capsys.readouterr().out + assert json.loads(emitted)["manifest"]["items"] + outputs.append(emitted) + for output in outputs: + assert card_id not in output, output + assert "version-owner@example.com" not in output, output + # Private identity and exact version still control authoritative validation. + row["version"] = "changed" + (project_config.ledger_root / "tasks.json").write_text( + json.dumps({"complete": True, "tasks": [row]}), encoding="utf-8") + assert load_project_snapshot(project_config)["coverage"][f"card:{card_id}"] != snapshot["coverage"][f"card:{card_id}"] + + @pytest.mark.parametrize("card_id", ["alice@example.com", "10.2.3.4", "T-1\n## injected", "T-[ready]"]) def test_card_reference_receives_egress_filter(project_config, card_id): from memorymaster.profile.project_repository import _display, load_project_snapshot From 80c03467172f2e6f150bedc0831ee440fbd35c95 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Thu, 22 Oct 2026 18:57:09 -0300 Subject: [PATCH 33/51] T-1703 mask generated HTML before parsing retained structure --- memorymaster/capture/adapters.py | 60 ++++++++++++++++++++++++++++--- tests/test_capture_adapters.py | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/memorymaster/capture/adapters.py b/memorymaster/capture/adapters.py index ea58c722..18e4ebc0 100644 --- a/memorymaster/capture/adapters.py +++ b/memorymaster/capture/adapters.py @@ -107,18 +107,49 @@ def handle_data(self, data: str) -> None: class _RetainedHTMLParser(_VisibleTextParser): - """Parse the original document once; map visible data to retained ranges.""" + """Parse masked HTML once; map visible data to original retained ranges.""" - def __init__(self, source: str, spans: tuple[SourceSpan, ...]) -> None: + def __init__(self, source: str, spans: tuple[SourceSpan, ...], excluded: tuple[tuple[int, int], ...]) -> None: super().__init__() # Decode entities after mapping their original source ranges. self.convert_charrefs = False self.source = source self.spans = spans self.ends = [span.end for span in spans] + self.excluded = excluded + self.excluded_ends = [end for _, end in excluded] self.line_starts = [0, *(match.end() for match in re.finditer("\n", source))] self.mapped: list[list[tuple[int, int, str]]] = [[] for _ in spans] + def _check_token(self, length: int) -> None: + line, column = self.getpos() + start = self.line_starts[line - 1] + column + index = bisect_right(self.excluded_ends, start) + if index < len(self.excluded) and self.excluded[index][0] < start + length: + raise CaptureRejected("ambiguous_generated_provenance", "HTML token crosses excluded generated content.") + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self._check_token(len(self.get_starttag_text())) + super().handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + line, column = self.getpos() + start = self.line_starts[line - 1] + column + self._check_token(self.source.index(">", start) + 1 - start) + super().handle_endtag(tag) + + def handle_comment(self, data: str) -> None: + self._check_token(len(data) + 7) + + def handle_decl(self, decl: str) -> None: + self._check_token(len(decl) + 3) + + def handle_pi(self, data: str) -> None: + self._check_token(len(data) + 2) + + def unknown_decl(self, data: str) -> None: + self._check_token(len(data) + 5) + def _record(self, data: str, raw_length: int, *, entity: bool = False) -> None: if self.hidden_depth: return @@ -182,6 +213,26 @@ def _bounded_text(text: str) -> tuple[str, tuple[str, ...]]: return sanitized, warnings +def _masked_html_source(text: str, excluded: tuple[tuple[int, int], ...]) -> str: + """Make generated bytes inert without moving source offsets or line starts.""" + chunks = [] + position = 0 + for start, end in excluded: + prefix = text[position:start] + # Whitespace must not complete an authored tag/entity or leave an + # incomplete token to absorb the next retained span. Refuse ambiguity + # rather than inventing a tag or joining independently authored bytes. + if (re.search(r"<(?:/?[A-Za-z][^<>]*|/?|![^<>]*|\?[^<>]*)$", prefix) + or prefix.rfind("") + or re.search(r"&(?:\#(?:[xX][0-9a-fA-F]*|[0-9]*)|[A-Za-z][A-Za-z0-9]*)?$", prefix)): + raise CaptureRejected("ambiguous_generated_provenance", "Incomplete HTML token touches excluded generated content.") + chunks.append(prefix) + chunks.append(re.sub(r"[^\r\n]", " ", text[start:end])) + position = end + chunks.append(text[position:]) + return "".join(chunks) + + def _retained_text(text: str, *, html: bool = False) -> tuple[str, tuple[str, ...], tuple[SourceSpan, ...], tuple[tuple[int, int], ...]]: """Filter raw input before parsers erase the generated marker comments.""" clean = strip_generated_spans(text) @@ -191,8 +242,9 @@ def _retained_text(text: str, *, html: bool = False) -> tuple[str, tuple[str, .. warning_codes: list[str] = ["generated_spans_excluded"] if clean.excluded_spans else [] html_values = [] if html: - parser = _RetainedHTMLParser(text, clean.spans) - parser.feed(text) + masked = _masked_html_source(text, clean.excluded_spans) + parser = _RetainedHTMLParser(masked, clean.spans, clean.excluded_spans) + parser.feed(masked) parser.close() html_values = parser.values() for index, span in enumerate(clean.spans): diff --git a/tests/test_capture_adapters.py b/tests/test_capture_adapters.py index ecf4ed14..d2783eed 100644 --- a/tests/test_capture_adapters.py +++ b/tests/test_capture_adapters.py @@ -66,6 +66,67 @@ def test_hidden_html_context_survives_generated_span_removal(capture_root, tag): assert raw[span.start:span.end].endswith("

Authored note.

") +@pytest.mark.parametrize("tag", ["script", "style", "noscript", "template"]) +def test_excluded_generated_closing_tag_cannot_expose_hidden_authored_data(tag): + from memorymaster.capture.adapters import _retained_text + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = (f"<{tag}>hidden before " + GENERATED_OPENER + f"" + + GENERATED_CLOSER + f"script-only sentence

Authored

") + text, _, _, _ = _retained_text(raw, html=True) + assert text == "Authored", f"Excluded generated structure exposed hidden data: {text!r}" + + +@pytest.mark.parametrize("tag", ["script", "style", "noscript", "template"]) +def test_excluded_generated_opening_tag_cannot_hide_authored_suffix(tag): + from memorymaster.capture.adapters import _retained_text + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = ("

Authored before

" + GENERATED_OPENER + f"<{tag}>" + + GENERATED_CLOSER + "

Authored after

") + text, _, _, _ = _retained_text(raw, html=True) + assert text == "Authored before\nAuthored after", f"Excluded generated structure hid authored data: {text!r}" + + +@pytest.mark.parametrize(("prefix", "suffix"), [ + ("hidden

After

"), + ("

After

"), + ("

After

"), + ("

After

"), + ("After"), + ("&am", "p;After"), + ("", "0;After"), +]) +def test_html_token_crossing_generated_boundary_is_rejected(prefix, suffix): + from memorymaster.capture.adapters import _retained_text + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = "

Before

" + prefix + GENERATED_OPENER + "generated" + GENERATED_CLOSER + suffix + with pytest.raises(CaptureRejected) as exc: + _retained_text(raw, html=True) + assert exc.value.code == "ambiguous_generated_provenance" + + +def test_html_generated_mask_preserves_multiline_attribution(capture_root): + from memorymaster.core.generated_artifacts import GENERATED_CLOSER, GENERATED_OPENER + + raw = ("

Before & caf\u00e9

\r\n" + GENERATED_OPENER + + "\r\n