From bf622788ab15c664aff523231461b356ed362514 Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman Date: Mon, 16 Nov 2026 12:36:26 -0800 Subject: [PATCH 1/3] fix: complete journal pipeline and graph reliability repairs Adopt the original 2026-11-13 source-qualified software continuation only. Preserve scientific, corpus, provider, migration and publishing-owner holds. --- .git-blame-ignore-revs | 15 +- .pre-commit-config.yaml | 4 +- Makefile | 7 +- TODO.md | 40 +- docs/RUN_GUIDE.md | 12 + docs/export.md | 8 + docs/rag.md | 44 +- docs/web_interface.md | 23 + run.py | 59 +- src/journal_utilities/interface/app.py | 92 ++- .../rag/adapters/entity_adapter.py | 53 +- .../rag/graph/surreal_client.py | 684 +++++++----------- src/journal_utilities/rag/main.py | 339 +++------ src/journal_utilities/rag/models/entities.py | 4 + tests/AGENTS.md | 13 +- .../integration/test_pipeline_integration.py | 13 +- .../rag/test_source_identity.py | 154 ++++ .../rag/unit/test_entity_adapter.py | 45 +- .../rag/unit/test_pipeline.py | 180 ++++- .../rag/unit/test_surreal_client.py | 408 +++-------- tests/journal_utilities/test_app.py | 188 ++++- tests/journal_utilities/test_run.py | 172 +++++ 22 files changed, 1381 insertions(+), 1176 deletions(-) create mode 100644 tests/journal_utilities/rag/test_source_identity.py diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 8b86dca9..95a34263 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,12 +1,7 @@ # Formatting-only commits excluded from `git blame` (git config: # blame.ignoreRevsFile = .git-blame-ignore-revs). -# -# PLACEHOLDER — the ruff-format-only commit has NOT landed yet. Once the -# black -> ruff format switch is committed (~63 files reformatted, no code -# changes), append its full 40-char SHA below with a one-line note, e.g.: -# -# # style: switch formatter from black to ruff format (formatting-only, no code changes) -# -# -# Then enable it repo-wide: -# git config blame.ignoreRevsFile .git-blame-ignore-revs \ No newline at end of file +# style: apply ruff format across src, tests, scripts (52 files) [E2] +# Reviewed against its parent: identical ASTs and ordered comment tokens. +9cb2642687b5565d752762eb8e6c921b6c21608e + +# Enable locally: git config blame.ignoreRevsFile .git-blame-ignore-revs \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c676cda..54c0168d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # Pre-commit gates — mirror the `gate` job in .github/workflows/test.yml: # ruff check, ruff format, mypy on src (the same arms `run.py gate` runs). -# Replaces black: ruff format is the repo formatter (see [tool.ruff] in -# pyproject.toml); the black config/dev-dep removal is tracked in TODO.md. +# Ruff format is the repo formatter (see [tool.ruff] in pyproject.toml). +# The black configuration and development dependency were removed in E2. repos: - repo: https://github.com/astral-sh/ruff-pre-commit diff --git a/Makefile b/Makefile index abcdd76a..1d9887e3 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ JOURNAL_DIR ?= ../ActiveInferenceJournal MANIFEST ?= data/output/channel_videos.json +RAG_BASE_PATH ?= . # Load .env file if it exists ifneq (,$(wildcard .env)) @@ -69,8 +70,8 @@ lint: uv run python run.py gate lint format format: - uv run ruff format src/ tests/ - uv run ruff check --fix src/ tests/ + uv run ruff format src/ tests/ scripts/ + uv run ruff check --fix src/ tests/ scripts/ db-start: surreal start --log trace --user root --pass root --bind 0.0.0.0:8080 rocksdb://./data/database @@ -112,7 +113,7 @@ extract-entities: echo "Error: COHERE_API_KEY not found in .env file"; \ exit 1; \ fi - uv run python -m journal_utilities.rag.main + uv run python -m journal_utilities.rag.main --pending --base-path "$(RAG_BASE_PATH)" enumerate-channel: @echo "Enumerating videos on the Active Inference channel..." diff --git a/TODO.md b/TODO.md index 9a816fa2..ab6cbdea 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,44 @@ **Status:** Maintained **Owner:** Active Inference Institute / Journal-Utilities maintainers -**Last reviewed:** 2026-11-12 (verification pass — Publishing contract migration, gate -re-run on this checkout, pin and formatter state confirmed; see the pass-5 entry below) +**Last reviewed:** 2026-11-13 (source-qualified CLI/API, disposable-database and +browser acceptance; earlier dated passes below retain their historical scope). + +--- + +## Pass 6 — pipeline and graph reliability (2026-11-13) + +- [x] **J1/J3 — Serve ownership and corpus** — occupied ports are refused without + killing their owners. The configured corpus reaches the actual web app; + browser navigation displayed its title and transcript with external media blocked. +- [x] **J2 — Required exports** — empty outcomes and mixed-success failures return + nonzero; required failure stops `full` and default orchestration. +- [x] **J4/J6 — Source identity and replay** — trusted source/type/name identity, + record-ID endpoints and deterministic edges prevent cross-source merging. + Explicit legacy migration preserves unknown-source rows and refuses conflicting + claims; failed edges remain pending and retry preserves historical dates. + Real in-memory SurrealDB 4.0.1 acceptance used SDK 3.0.0, including injected + write failure and replay. No production database was accessed or migrated. +- [x] **J5/A1 — Extraction entry point** — explicit file or pending selection performs + work, reports totals and propagates failures. Empty queues do not construct + the provider. `make extract-entities` selects pending work under `RAG_BASE_PATH`. +- [x] **J7 — Request boundaries** — bounded body ingestion, typed chat fields, + pre-stream errors and bounded public pagination are covered by focused tests. +- [x] **O3/O4/O5/A2 — Maintenance** — reviewed format-only commit recorded for blame; + stale test-count prose and Black commentary removed; both Makefile formatter + operations now include `scripts/`. + +Focused Python 3.12 checks, actual database acceptance and browser navigation +passed. This is not a full-suite, coverage, hosted-CI, live-Cohere or production +migration claim. Historical M10 metadata stamping and ME2 module-path repair +were partial steps, not prior proof of complete identity or extraction behavior. +The dated measurement files remain unchanged. + +Still open: hosted billing, qualified exact-pin GNN admission (no automatic pin +bump), canonical Publishing producer adoption, admitted full-gate measurements, +GPU/live-provider/YouTube-write qualification, the sibling translation migration, +and the owner-scheduled description backlog. Existing dated entries retain the +details of those external and ownership constraints. --- diff --git a/docs/RUN_GUIDE.md b/docs/RUN_GUIDE.md index a951c15a..98358cba 100644 --- a/docs/RUN_GUIDE.md +++ b/docs/RUN_GUIDE.md @@ -98,3 +98,15 @@ it runs the live download tests. 2. Add the row to the arms table above and any rule it enforces to the doc it protects. 3. Run `uv run python run.py gate ` and, if the timing page should change, `uv run python run.py gate report`. + +## Listener ownership and corpus selection + +`serve` refuses an occupied port without discovering or signalling its owner. +Stop that owner explicitly or choose another configured port. +`general.data_dir` is passed to the web application: relative paths resolve +against the project root; absolute paths remain absolute. + +`make extract-entities` selects pending database work and resolves relative +transcript paths under `RAG_BASE_PATH` (default `.`). +`make format` runs both Ruff operations over `src/ tests/ scripts/`; it changes +files and is not an acceptance-only command. diff --git a/docs/export.md b/docs/export.md index 18f896ec..67b22f29 100644 --- a/docs/export.md +++ b/docs/export.md @@ -76,3 +76,11 @@ html = true # Output directory relative to project root output_dir = data/export ``` + +## Required export completion + +`run.py export` returns nonzero when no outcomes are produced or any requested +export fails, including mixed-success batches. Successfully written or +already-existing nonempty outputs remain available. `full` and the default +pipeline propagate required export failure; the default pipeline does not +continue to tests or serving after that failure, independently of `test_strict`. diff --git a/docs/rag.md b/docs/rag.md index fc9c0c3b..31fd26b8 100644 --- a/docs/rag.md +++ b/docs/rag.md @@ -35,15 +35,17 @@ Defines the ontology for extraction. Manages interaction with SurrealDB. -- **Nodes**: Stores entities as graph nodes (e.g., `person:karl_friston`). -- **Edges**: Stores relationships as graph edges (e.g., `authored` -> `concept:free_energy_principle`). -- **Idempotency**: Ensures entities are merged rather than duplicated. +- **Nodes**: Entity records are identified by trusted transcript source, entity type and name. +- **Edges**: Relationships reference the returned entity record IDs, not generic display names. +- **Idempotency**: Scoped records and deterministic relationship identities are reused on replay. - **SDK**: `surrealdb>=3.0.0` (the async-client generation). `SurrealDBClient` opens the connection with `AsyncSurreal`, types it as `AsyncSurrealConnection`, and awaits `connect()` / `signin()` / `use()` / `query()` / `close()`. `query()` answers one entry per statement, so single-statement queries are unwrapped to their rows - (`_first_statement`) before use; `create()` on a table answers with the created records. - Server-side, the SDK accepts SurrealDB 2.x and newer and refuses older servers. + (`_first_statement`) before use. The qualified 3.0.0 SDK returns a single record + mapping from `create()` and singular-endpoint `relate()`, not a list. + The source-identity migration was exercised against SurrealDB 4.0.1; this is + not a qualification of every server version accepted by the SDK. ## Pipeline (`main.py`) @@ -51,15 +53,15 @@ The `JournalRAGPipeline` class orchestrates the process: 1. **`process_transcript(transcript)`**: - Creates a `Transcript` record in DB. - -Calls `extractor.extract_core_entities()`. + - Calls `extractor.extract_core_entities()`. - Converts results via `EntityAdapter`. - Upserts entities and relationships to SurrealDB. ## Usage ```bash -# Run the extraction pipeline on all new transcripts -make extract-entities +# Select pending database transcripts; resolve their relative paths under the corpus +make extract-entities RAG_BASE_PATH=/path/to/corpus ``` ## Configuration @@ -69,3 +71,29 @@ make extract-entities | `COHERE_API_KEY` | API Key for Cohere AI | | `COHERE_MODEL` | Model ID (default `command-a-03-2025`, from `rag/settings.py`) | | `DB_URL` | SurrealDB connection string | + +## Explicit extraction and source identity + +Select files with `python -m journal_utilities.rag.main --file transcript.txt [more.txt ...]` +or pending database work with `python -m journal_utilities.rag.main --pending --base-path /path/to/corpus`. +No selection is a usage error (exit 2). Requested failures return exit 1; +successful runs, including an explicitly selected empty pending queue, return 0 +and print processed/failed/entity/relationship totals. Extraction needs Cohere +credentials and SurrealDB; an empty queue does not construct the provider client. + +Relationship endpoints are checked against the trusted source-qualified entity +identity. Missing required endpoints or failed edge writes leave the transcript +pending. Retry and replay reuse persisted records rather than duplicating edges. +Generic names or metadata claims never establish provenance for unknown-source +legacy records. File and database loaders use a valid filename date, otherwise +`datetime.min`; retries preserve stored historical dates. + +## Legacy schema migration + +Take a reviewed backup and stop concurrent writers before explicitly running +`python -m journal_utilities.rag.main --migrate-source-identity`. +Ordinary connection refuses a legacy schema rather than silently migrating it. +The migration changes schema, preserves records and performs no Cohere request. +It refuses ambiguous identities or contradictory attributed endpoints; it does +not backfill provenance. Qualify it against a disposable database before any +production migration. diff --git a/docs/web_interface.md b/docs/web_interface.md index 0022e0b9..5ed5d6fa 100644 --- a/docs/web_interface.md +++ b/docs/web_interface.md @@ -85,3 +85,26 @@ The frontend files are served directly from `src/journal_utilities/interface/sta - `src/journal_utilities/interface/app.py`: Main application logic. - `src/journal_utilities/interface/static/app.js`: Core frontend logic (routing, rendering). - `src/journal_utilities/interface/static/index.html`: Base layout. + +## Request admission limits + +Chat, streaming chat and clear-session requests must be JSON objects with +string `session_id` (default `default`, at most 200 characters), string `message` +(default empty, at most 32000 characters), and boolean `use_rag` (default true). +Chat and streaming require a nonblank message. Clear validates provided fields +but does not require a message. Malformed JSON or an empty required message +returns 400; invalid shapes/types return 422; byte or field limits return 413 +before a provider request or streaming response starts. + +The application retains at most 387424 body bytes before JSON decoding and +rejects an over-limit chunk before appending it. Streamed bytes are checked even +with missing or understated Content-Length. This does not bound upstream +server/ASGI transport allocations. + +Video pagination requires offset >= 0 and limit 1..200. Search requires limit +1..100; internal unlimited search remains available through `limit=None`. + +`run.py serve` uses the configured corpus and refuses occupied ports without +terminating another process. Browser fixtures must use canonical 11-character +YouTube IDs; external thumbnails, fonts and embedded video still need network +access independently of the locally served transcript. diff --git a/run.py b/run.py index 92a3b595..111b799d 100644 --- a/run.py +++ b/run.py @@ -24,7 +24,6 @@ import json import logging import os -import signal import socket import subprocess import sys @@ -227,12 +226,19 @@ def cmd_export(config: configparser.ConfigParser, _args: argparse.Namespace) -> print("⚠ No transcripts found to export") return 1 + failures = 0 + outcomes = 0 for fmt_name, fmt_results in results.items(): ok = sum(1 for r in fmt_results if r.status == "success") skip = sum(1 for r in fmt_results if r.status == "skipped") fail = sum(1 for r in fmt_results if r.status == "failed") + failures += fail + outcomes += len(fmt_results) print(f" {fmt_name}: {ok} exported, {skip} skipped, {fail} failed") + if failures or not outcomes: + print("\n✗ Export incomplete") + return 1 print("\n✓ Export complete") return 0 @@ -287,36 +293,6 @@ def _port_in_use(port: int) -> bool: return True -def _free_port(port: int) -> bool: - """Try to kill whatever is occupying *port*. Return True on success.""" - try: - result = subprocess.run( - ["lsof", "-ti", f":{port}"], - capture_output=True, - text=True, - timeout=5, - ) - pids = result.stdout.strip().split() - if not pids: - return True - - my_pid = str(os.getpid()) - for pid in pids: - if pid == my_pid: - continue - logger.info("Killing stale process %s on port %d", pid, port) - os.kill(int(pid), signal.SIGTERM) - - # Brief wait for cleanup - import time as _t - - _t.sleep(0.5) - return not _port_in_use(port) - except Exception as exc: - logger.warning("Could not free port %d: %s", port, exc) - return False - - def cmd_serve(config: configparser.ConfigParser, _args: argparse.Namespace) -> int: """Start the web interface.""" import uvicorn @@ -327,23 +303,19 @@ def cmd_serve(config: configparser.ConfigParser, _args: argparse.Namespace) -> i port_str = get_str(config, "interface", "port", "8000") port = int(port_str) - # ── Auto-resolve port conflicts ──────────────────────────────── + # An existing listener belongs to its owner; never kill or reclaim it. if _port_in_use(port): - logger.warning("Port %d already in use — attempting to free it", port) - if _free_port(port): - logger.info("Port %d freed successfully", port) - else: - logger.error("Cannot free port %d — aborting serve", port) - print(f" ✗ Port {port} is occupied and could not be freed.") - print(f" Try: lsof -ti :{port} | xargs kill -9") - return 1 + logger.error("Port %d is already occupied; listener left untouched", port) + print(f" ✗ Port {port} is occupied. Stop its owner explicitly or choose another port.") + return 1 print(f"Starting web interface at http://{host}:{port}") # Configure uvicorn logging to match ours if needed, # but uvicorn has its own config. We'll let it handle itself. try: - app = create_app() + data_dir = PROJECT_ROOT / get_str(config, "general", "data_dir", "data/output") + app = create_app(data_dir=data_dir) # log_config=None tells uvicorn to use the existing logging configuration # which allows our root logger (with FileHandler) to capture the logs. uvicorn.run(app, host=host, port=port, log_config=None) @@ -535,8 +507,9 @@ def cmd_default(config: configparser.ConfigParser, args: argparse.Namespace) -> print("━" * 62) rc = cmd_export(config, args) if rc != 0: - logger.warning("Export step returned non-zero: %d", rc) - print(f" ⚠ Export finished with warnings (exit code {rc})\n") + logger.error("Export step failed: %d", rc) + print(f" ✗ Export failed (exit code {rc}) — aborting pipeline\n") + return rc else: print() diff --git a/src/journal_utilities/interface/app.py b/src/journal_utilities/interface/app.py index 800b14a5..634f4090 100644 --- a/src/journal_utilities/interface/app.py +++ b/src/journal_utilities/interface/app.py @@ -18,6 +18,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, StreamingResponse from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel, ConfigDict, ValidationError from journal_utilities.interface.chat_engine import ChatEngine from journal_utilities.interface.data_loader import DataLoader @@ -30,9 +31,48 @@ STATIC_DIR = Path(__file__).parent / "static" -# Upper bound on a single chat message; the body is otherwise read unbounded -# and forwarded to Ollama (memory + LLM-cost DoS). MAX_MESSAGE_LENGTH = 32_000 +MAX_SESSION_LENGTH = 200 +# JSON can escape one non-BMP character as two six-byte Unicode escapes. +MAX_CHAT_BODY_BYTES = 12 * (MAX_MESSAGE_LENGTH + MAX_SESSION_LENGTH) + 1024 + + +class _ChatRequest(BaseModel): + model_config = ConfigDict(strict=True) + + session_id: str = "default" + message: str = "" + use_rag: bool = True + + +async def _read_chat_request(request: Request, *, require_message: bool) -> _ChatRequest: + """Bound retained bytes before JSON decoding, independent of Content-Length.""" + declared = request.headers.get("content-length") + if declared is not None: + try: + length = int(declared) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid Content-Length") from exc + if length < 0: + raise HTTPException(status_code=400, detail="Invalid Content-Length") + if length > MAX_CHAT_BODY_BYTES: + raise HTTPException(status_code=413, detail="Chat request body too large") + body = bytearray() + async for chunk in request.stream(): + if len(body) + len(chunk) > MAX_CHAT_BODY_BYTES: + raise HTTPException(status_code=413, detail="Chat request body too large") + body.extend(chunk) + try: + payload = _ChatRequest.model_validate_json(body) + except ValidationError as exc: + errors = exc.errors(include_input=False, include_context=False, include_url=False) + status = 400 if any(error["type"] == "json_invalid" for error in errors) else 422 + raise HTTPException(status_code=status, detail="Invalid chat request") from exc + if len(payload.message) > MAX_MESSAGE_LENGTH or len(payload.session_id) > MAX_SESSION_LENGTH: + raise HTTPException(status_code=413, detail="Message or session id too long") + if require_message and not payload.message.strip(): + raise HTTPException(status_code=400, detail="Message cannot be empty") + return payload def create_app(data_dir: Path | None = None) -> FastAPI: @@ -84,8 +124,8 @@ async def get_videos( has_transcript: bool | None = None, sort_by: str = "upload_date", reverse: bool = True, - offset: int = 0, - limit: int = Query(default=50, le=200), + offset: int = Query(default=0, ge=0), + limit: int = Query(default=50, ge=1, le=200), q: str | None = None, ) -> dict[str, Any]: """Get paginated, filterable list of videos.""" @@ -252,7 +292,7 @@ async def get_categories() -> dict[str, Any]: @app.get("/api/search") async def search( q: str = Query(..., min_length=2), - limit: int = Query(default=20, le=100), + limit: int = Query(default=20, ge=1, le=100), ) -> dict[str, Any]: """Full-text search across all transcripts.""" results = loader.search_index.search(q, limit=limit) @@ -282,47 +322,25 @@ async def chat_status() -> dict[str, Any]: @app.post("/api/chat") async def chat_message(request: Request) -> dict[str, Any]: - """Send a chat message (non-streaming).""" - body = await request.json() - session_id = body.get("session_id", "default") - message = body.get("message", "") - use_rag = body.get("use_rag", True) - - if not message.strip(): - raise HTTPException(status_code=400, detail="Message cannot be empty") - if len(message) > MAX_MESSAGE_LENGTH or len(session_id) > 200: - raise HTTPException(status_code=413, detail="Message or session id too long") - - return await chat_engine.chat(session_id, message, use_rag=use_rag) + """Send a validated, bounded chat message (non-streaming).""" + payload = await _read_chat_request(request, require_message=True) + return await chat_engine.chat(payload.session_id, payload.message, use_rag=payload.use_rag) @app.post("/api/chat/stream") async def chat_stream(request: Request) -> StreamingResponse: - """Send a chat message with streaming response (SSE).""" - body = await request.json() - session_id = body.get("session_id", "default") - message = body.get("message", "") - use_rag = body.get("use_rag", True) - - if not message.strip(): - raise HTTPException(status_code=400, detail="Message cannot be empty") - if len(message) > MAX_MESSAGE_LENGTH or len(session_id) > 200: - raise HTTPException(status_code=413, detail="Message or session id too long") - + """Validate completely before opening a streaming response.""" + payload = await _read_chat_request(request, require_message=True) return StreamingResponse( - chat_engine.chat_stream(session_id, message, use_rag=use_rag), + chat_engine.chat_stream(payload.session_id, payload.message, use_rag=payload.use_rag), media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - }, + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, ) @app.post("/api/chat/clear") async def chat_clear(request: Request) -> dict[str, str]: - """Clear a chat session.""" - body = await request.json() - session_id = body.get("session_id", "default") - chat_engine.clear_session(session_id) + """Clear a validated session without calling a provider.""" + payload = await _read_chat_request(request, require_message=False) + chat_engine.clear_session(payload.session_id) return {"status": "cleared"} # ------------------------------------------------------------------ diff --git a/src/journal_utilities/rag/adapters/entity_adapter.py b/src/journal_utilities/rag/adapters/entity_adapter.py index 3136d126..6df0e59f 100644 --- a/src/journal_utilities/rag/adapters/entity_adapter.py +++ b/src/journal_utilities/rag/adapters/entity_adapter.py @@ -15,7 +15,7 @@ class EntityAdapter: """Converts CoreEntities to simple Entity and Relationship models for database storage.""" @staticmethod - def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: + def convert_core_entities(core: CoreEntities, *, source: str) -> Sequence[Entity]: """Convert CoreEntities to a flat list of Entity objects. Args: @@ -30,6 +30,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for concept in core.concepts: entities.append( Entity( + source=source, name=concept.name, type="concept", description=concept.definition, @@ -47,6 +48,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for researcher in core.researchers: entities.append( Entity( + source=source, name=researcher.name, type="researcher", description=researcher.contribution or "", @@ -64,6 +66,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for citation in core.citations: entities.append( Entity( + source=source, name=citation.title, type="citation", description=citation.context or "", @@ -84,6 +87,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for term in core.technical_terms: entities.append( Entity( + source=source, name=term.term, type="technical_term", description=term.explanation, @@ -101,6 +105,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for idx, insight in enumerate(core.key_insights): entities.append( Entity( + source=source, name=f"Insight {idx + 1}", type="key_insight", description=insight, @@ -114,7 +119,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: return entities @staticmethod - def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: + def convert_detailed_analysis(detailed: DetailedAnalysis, *, source: str) -> Sequence[Entity]: """Convert DetailedAnalysis to a flat list of Entity objects. Args: @@ -129,6 +134,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for method in detailed.methods_techniques: entities.append( Entity( + source=source, name=method.name, type="method_technique", description=method.description, @@ -146,6 +152,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for notation in detailed.mathematical_notation: entities.append( Entity( + source=source, name=notation.notation, type="mathematical_notation", description=notation.meaning, @@ -162,6 +169,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for equation in detailed.equations: entities.append( Entity( + source=source, name=equation.name or equation.formula[:50], type="equation", description=equation.context or "", @@ -178,6 +186,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for tool in detailed.tools_resources: entities.append( Entity( + source=source, name=tool.name, type="tool_resource", description=tool.purpose or "", @@ -195,6 +204,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for problem in detailed.research_problems: entities.append( Entity( + source=source, name=problem.problem[:100], # Use first 100 chars as name type="research_problem", description=problem.problem, @@ -212,6 +222,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for application in detailed.applications: entities.append( Entity( + source=source, name=f"{application.domain} Application", type="application", description=application.description, @@ -228,7 +239,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: return entities @staticmethod - def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: + def extract_relationships(core: CoreEntities, *, source: str) -> Sequence[Relationship]: """Extract relationships from CoreEntities structure. Args: @@ -244,6 +255,9 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: for related_concept in concept.related_concepts: relationships.append( Relationship( + transcript_source=source, + source_type="concept", + target_type="concept", source=concept.name, target=related_concept, relationship_type="related_to", @@ -262,6 +276,9 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: if author in researcher_names: relationships.append( Relationship( + transcript_source=source, + source_type="researcher", + target_type="citation", source=author, target=citation.title, relationship_type="authored", @@ -276,31 +293,11 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: @classmethod def convert_extracted_entities( - cls, extracted: ExtractedEntities + cls, extracted: ExtractedEntities, *, source: str ) -> tuple[Sequence[Entity], Sequence[Relationship]]: - """Convert complete ExtractedEntities to entities and relationships. - - Args: - extracted: Complete extraction result - - Returns: - Tuple of (entities, relationships) - """ - entities = list(cls.convert_core_entities(extracted.core)) - + """Use caller-owned provenance, never a scientific extraction's metadata.""" + entities = list(cls.convert_core_entities(extracted.core, source=source)) if extracted.detailed: - entities.extend(cls.convert_detailed_analysis(extracted.detailed)) - - # Stamp the transcript source onto every entity so the graph layer can - # scope merges per transcript — preventing "Insight 1" (or a truncated - # formula name) from one video silently merging with another's (M10). - source = getattr(extracted.core, "source", None) - if source: - for entity in entities: - meta = dict(entity.metadata) - meta.setdefault("source", source) - entity.metadata = meta - - relationships = cls.extract_relationships(extracted.core) - + entities.extend(cls.convert_detailed_analysis(extracted.detailed, source=source)) + relationships = cls.extract_relationships(extracted.core, source=source) return entities, relationships diff --git a/src/journal_utilities/rag/graph/surreal_client.py b/src/journal_utilities/rag/graph/surreal_client.py index 6e55e271..9e2fd5a5 100644 --- a/src/journal_utilities/rag/graph/surreal_client.py +++ b/src/journal_utilities/rag/graph/surreal_client.py @@ -1,34 +1,67 @@ -"""SurrealDB client for storing entities and relationships.""" +"""Source-qualified SurrealDB storage; legacy attribution is never inferred.""" +import hashlib +import json from collections.abc import Mapping, Sequence from datetime import UTC, datetime from typing import Any, cast -import structlog -from surrealdb import AsyncSurreal, AsyncSurrealConnection +from surrealdb import AsyncSurreal, AsyncSurrealConnection, RecordID, StringRecordId from journal_utilities.rag.models import Entity, Relationship, Transcript from journal_utilities.rag.settings import settings -logger = structlog.get_logger(__name__) - - -def _first_statement(result: Sequence[Any]) -> list[Any]: - """Rows produced by the first statement of a ``query()`` response. - - surrealdb 3.x answers ``query()`` with one entry per statement, so a - single-statement query arrives as ``[rows]``. Every caller here issues one - statement and wants those rows, which is the shape the pre-3.x client - returned directly. - """ - if not result: - return [] - rows = result[0] - return rows if isinstance(rows, list) else [] +def _first_statement(result: object) -> object: + """The installed SDK 3.x returns one entry per statement.""" + if not isinstance(result, list) or len(result) != 1: + raise RuntimeError("Unexpected SurrealDB statement result") + return result[0] + + +def _single(rows: Sequence[Mapping[str, Any]], identity: str) -> Mapping[str, Any] | None: + if len(rows) > 1: + raise RuntimeError(f"Ambiguous {identity}; explicit data review required") + return rows[0] if rows else None + + +def _utc(value: datetime) -> datetime: + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + + +_SCHEMA = """ +DEFINE TABLE IF NOT EXISTS entity SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON entity TYPE string; +DEFINE FIELD IF NOT EXISTS type ON entity TYPE string; +DEFINE FIELD IF NOT EXISTS source ON entity TYPE option; +DEFINE FIELD IF NOT EXISTS description ON entity TYPE option; +DEFINE FIELD IF NOT EXISTS confidence ON entity TYPE number; +DEFINE FIELD IF NOT EXISTS mentions ON entity TYPE number; +DEFINE FIELD IF NOT EXISTS context ON entity TYPE array; +DEFINE FIELD OVERWRITE metadata ON entity TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS created_at ON entity TYPE datetime; +DEFINE INDEX IF NOT EXISTS entity_source_type_name ON entity FIELDS source, type, name UNIQUE; +DEFINE TABLE IF NOT EXISTS relates SCHEMAFULL TYPE RELATION; +DEFINE FIELD IF NOT EXISTS source ON relates TYPE option; +DEFINE FIELD IF NOT EXISTS relationship_type ON relates TYPE string; +DEFINE FIELD IF NOT EXISTS description ON relates TYPE option; +DEFINE FIELD IF NOT EXISTS confidence ON relates TYPE number; +DEFINE FIELD IF NOT EXISTS context ON relates TYPE option; +DEFINE FIELD OVERWRITE metadata ON relates TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS created_at ON relates TYPE datetime; +DEFINE INDEX IF NOT EXISTS relates_source_endpoints ON relates FIELDS source, in, out, relationship_type; +DEFINE TABLE IF NOT EXISTS transcript SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS title ON transcript TYPE string; +DEFINE FIELD IF NOT EXISTS date ON transcript TYPE datetime; +DEFINE FIELD IF NOT EXISTS content ON transcript TYPE string; +DEFINE FIELD IF NOT EXISTS source ON transcript TYPE option; +DEFINE FIELD OVERWRITE metadata ON transcript TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS processed ON transcript TYPE bool; +DEFINE FIELD IF NOT EXISTS created_at ON transcript TYPE datetime; +""" class SurrealDBClient: - """Client for interacting with SurrealDB graph database.""" + """Persist identities using trusted transcript source, entity type and name.""" def __init__( self, @@ -38,459 +71,232 @@ def __init__( username: str | None = None, password: str | None = None, ) -> None: - """Initialize the SurrealDB client. - - Args: - url: SurrealDB connection URL. - namespace: Database namespace. - database: Database name. - username: Authentication username. - password: Authentication password. - """ self.url = url or settings.surrealdb_url self.namespace = namespace or settings.surrealdb_namespace self.database = database or settings.surrealdb_database self.username = username or settings.surrealdb_username self.password = password or settings.surrealdb_password self.db: AsyncSurrealConnection | None = None - logger.info( - "Initialized SurrealDBClient", - namespace=self.namespace, - database=self.database, - ) + self.migration_summary: dict[str, int] | None = None + + async def connect(self, *, migrate_source_identity: bool = False) -> None: + """Refuse legacy schema unless migration was explicitly selected. - async def connect(self) -> None: - """Connect to SurrealDB.""" + Migration requires a quiescent database. It changes schema, not legacy + data: absent provenance stays absent and ambiguous attribution fails. + """ + self.db = AsyncSurreal(self.url) try: - self.db = AsyncSurreal(self.url) await self.db.connect() - await self.db.signin({"user": self.username, "pass": self.password}) + await self.db.signin({"username": self.username, "password": self.password}) await self.db.use(self.namespace, self.database) - logger.info("Connected to SurrealDB") - await self._initialize_schema() - except Exception as e: - logger.error("Failed to connect to SurrealDB", error=str(e)) + await self._initialize_schema(migrate_source_identity=migrate_source_identity) + except Exception: + await self.disconnect() raise async def disconnect(self) -> None: - """Disconnect from SurrealDB.""" - if self.db: - await self.db.close() - logger.info("Disconnected from SurrealDB") + if self.db is not None: + db, self.db = self.db, None + await db.close() - async def _initialize_schema(self) -> None: - """Initialize database schema.""" - if not self.db: + async def _initialize_schema(self, *, migrate_source_identity: bool = False) -> None: + if self.db is None: raise RuntimeError("Database not connected") - - # Define entity table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS entity SCHEMAFULL; - DEFINE FIELD IF NOT EXISTS name ON TABLE entity TYPE string; - DEFINE FIELD IF NOT EXISTS type ON TABLE entity TYPE string; - DEFINE FIELD IF NOT EXISTS description ON TABLE entity TYPE option; - DEFINE FIELD IF NOT EXISTS confidence ON TABLE entity TYPE number; - DEFINE FIELD IF NOT EXISTS mentions ON TABLE entity TYPE number; - DEFINE FIELD IF NOT EXISTS context ON TABLE entity TYPE array; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE entity TYPE object; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE entity TYPE datetime; - DEFINE INDEX IF NOT EXISTS entity_name_type ON TABLE entity COLUMNS name, type UNIQUE; - """ - ) - - # Define relationship table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS relates SCHEMAFULL TYPE RELATION; - DEFINE FIELD IF NOT EXISTS relationship_type ON TABLE relates TYPE string; - DEFINE FIELD IF NOT EXISTS description ON TABLE relates TYPE option; - DEFINE FIELD IF NOT EXISTS confidence ON TABLE relates TYPE number; - DEFINE FIELD IF NOT EXISTS context ON TABLE relates TYPE option; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE relates TYPE object; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE relates TYPE datetime; - """ + info = cast(Mapping[str, Any], _first_statement(await self.db.query("INFO FOR DB;"))) + tables = info["tables"] + indexes = {} + if "entity" in tables: + entity_info = cast(Mapping[str, Any], _first_statement(await self.db.query("INFO FOR TABLE entity;"))) + indexes = entity_info["indexes"] + legacy = ("entity" in tables or "relates" in tables) and ( + "entity_source_type_name" not in indexes or "entity_name_type" in indexes ) - - # Define transcript table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS transcript SCHEMAFULL; - DEFINE FIELD IF NOT EXISTS title ON TABLE transcript TYPE string; - DEFINE FIELD IF NOT EXISTS date ON TABLE transcript TYPE datetime; - DEFINE FIELD IF NOT EXISTS content ON TABLE transcript TYPE string; - DEFINE FIELD IF NOT EXISTS source ON TABLE transcript TYPE option; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE transcript TYPE object; - DEFINE FIELD IF NOT EXISTS processed ON TABLE transcript TYPE bool; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE transcript TYPE datetime; - """ - ) - - logger.info("Initialized database schema") - - async def create_entity(self, entity: Entity) -> str: - """Create or update an entity in the database. - - Args: - entity: Entity to create or update. - - Returns: - Entity ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Check if entity exists. Generic/truncated names (e.g. "Insight 1") - # collide across transcripts, so merge is scoped to the entity's - # source when it is known — otherwise a "Insight 1" from two - # different videos would silently merge into one record (M10). - entity_type = entity.type.value if hasattr(entity.type, "value") else entity.type - source = (entity.metadata or {}).get("source") - if source: - existing = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type" - " AND source = $source", - {"name": entity.name, "type": entity_type, "source": source}, - ) - ) - else: - existing = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type", - {"name": entity.name, "type": entity_type}, - ) - ) - - if existing: - # Update existing entity - current = existing[0] - entity_id = current["id"] - await self.db.update( - entity_id, - { - "mentions": entity.mentions, - "confidence": max(entity.confidence, current["confidence"]), - "context": list(set(entity.context + current["context"])), - "description": entity.description or current["description"], - }, - ) - logger.debug("Updated entity", entity_id=entity_id) - return str(entity_id) - else: - # Create new entity - payload: dict[str, Any] = { - "name": entity.name, - "type": entity_type, - "description": entity.description, - "confidence": entity.confidence, - "mentions": entity.mentions, - "context": entity.context, - "metadata": entity.metadata, - "created_at": datetime.now(UTC).isoformat(), - } - if source: - payload["source"] = source - # ``create`` on a table answers with the created records. - created = cast(list[dict[str, Any]], await self.db.create("entity", payload)) - entity_id = created[0]["id"] - logger.debug("Created entity", entity_id=entity_id) - return str(entity_id) - - except Exception as e: - logger.error("Failed to create entity", entity=entity.name, error=str(e)) - raise - - async def create_relationship(self, relationship: Relationship) -> str: - """Create a relationship between entities. - - Args: - relationship: Relationship to create. - - Returns: - Relationship ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Get entity IDs (support both old and new field names) - source_name = ( - relationship.source_entity - if hasattr(relationship, "source_entity") - else relationship.source - ) - target_name = ( - relationship.target_entity - if hasattr(relationship, "target_entity") - else relationship.target - ) - - source = _first_statement( - await self.db.query( - "SELECT id FROM entity WHERE name = $name LIMIT 1", - {"name": source_name}, - ) - ) - target = _first_statement( - await self.db.query( - "SELECT id FROM entity WHERE name = $name LIMIT 1", - {"name": target_name}, - ) - ) - - if not source or not target: - raise ValueError("Source or target entity not found") - - source_id = source[0]["id"] - target_id = target[0]["id"] - - # Create relationship - result = _first_statement( - await self.db.query( - f""" - RELATE {source_id}->relates->{target_id} CONTENT {{ - relationship_type: $type, - description: $description, - confidence: $confidence, - context: $context, - metadata: $metadata, - created_at: $created_at - }} - """, - { - "type": relationship.relationship_type, - "description": relationship.description, - "confidence": relationship.confidence, - "context": relationship.context, - "metadata": relationship.metadata, - "created_at": datetime.now(UTC).isoformat(), - }, - ) - ) - - rel_id = result[0]["id"] - logger.debug("Created relationship", relationship_id=rel_id) - return str(rel_id) - - except Exception as e: - logger.error( - "Failed to create relationship", - source=source_name, - target=target_name, - error=str(e), - ) - raise - - async def create_transcript(self, transcript: Transcript) -> str: - """Store a transcript in the database. - - Args: - transcript: Transcript to store. - - Returns: - Transcript ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - created = cast( - list[dict[str, Any]], - await self.db.create( - "transcript", - { - "title": transcript.title, - "date": transcript.date.isoformat(), - "content": transcript.content, - "source": transcript.source, - "metadata": transcript.metadata, - "processed": transcript.processed, - "created_at": transcript.created_at.isoformat(), - }, - ), + if legacy and not migrate_source_identity: + raise RuntimeError( + "Legacy graph schema: review a backup, stop other writers, then explicitly " + "run --migrate-source-identity. No legacy provenance will be assigned." ) - transcript_id = created[0]["id"] - logger.info("Created transcript", transcript_id=transcript_id) - return str(transcript_id) - - except Exception as e: - logger.error("Failed to create transcript", title=transcript.title, error=str(e)) - raise - - async def get_entity_by_name( - self, name: str, entity_type: str | None = None - ) -> dict[str, Any] | None: - """Get an entity by name. - - Args: - name: Entity name. - entity_type: Optional entity type filter. - - Returns: - Entity data or None if not found. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - if entity_type: - rows = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type LIMIT 1", - {"name": name, "type": entity_type}, - ) - ) - else: - rows = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name LIMIT 1", - {"name": name}, - ) - ) - - if rows: - return cast(dict[str, Any], rows[0]) + if migrate_source_identity: + self.migration_summary = await self._admit_migration(tables) + remove_old = "REMOVE INDEX entity_name_type ON entity;" if "entity_name_type" in indexes else "" + # A failed index definition must not leave a partially migrated schema. + await self.db.query("BEGIN TRANSACTION;" + _SCHEMA + remove_old + "COMMIT TRANSACTION;") + + async def _admit_migration(self, tables: Mapping[str, Any]) -> dict[str, int]: + entities = list(await self.query("SELECT * FROM entity;")) if "entity" in tables else [] + edges = list(await self.query("SELECT * FROM relates;")) if "relates" in tables else [] + transcripts = list(await self.query("SELECT * FROM transcript;")) if "transcript" in tables else [] + by_id = {str(row["id"]): row for row in entities} + identities: set[tuple[str | None, str, str]] = set() + unknown = 0 + for row in entities: + source = self._legacy_source(row) + unknown += source is None + identity = (source, row["type"], row["name"]) + if identity in identities: + raise RuntimeError("Ambiguous legacy entity identity; migration refused") + identities.add(identity) + edge_keys: set[tuple[str, str, str, str]] = set() + unknown_edges = 0 + for row in edges: + source = self._legacy_source(row) + if source is None: + unknown_edges += 1 + continue + for endpoint in ("in", "out"): + entity = by_id.get(str(row[endpoint])) + if entity is None or self._legacy_source(entity) != source: + raise RuntimeError("Ambiguous legacy relationship attribution; migration refused") + key = (source, str(row["in"]), str(row["out"]), row["relationship_type"]) + if key in edge_keys: + raise RuntimeError("Ambiguous legacy relationship identity; migration refused") + edge_keys.add(key) + sources: set[str] = set() + for row in transcripts: + source = self._legacy_source(row) + if source is not None: + if source in sources: + raise RuntimeError("Ambiguous legacy transcript source; migration refused") + sources.add(source) + return {"legacy_entities_preserved": len(entities), "unknown_source_entities": unknown, + "legacy_relationships_preserved": len(edges), "unknown_source_relationships": unknown_edges} + + @staticmethod + def _legacy_source(row: Mapping[str, Any]) -> str | None: + source = row.get("source") + if source is None or source == "": + # A metadata claim alone is not trusted provenance. Keep it untouched. return None - - except Exception as e: - logger.error("Failed to get entity", name=name, error=str(e)) - raise + if not isinstance(source, str) or not source.strip(): + raise RuntimeError("Ambiguous legacy source value; migration refused") + claim = (row.get("metadata") or {}).get("source") + if claim is not None and claim != source: + raise RuntimeError("Conflicting legacy source claims; migration refused") + return source async def query( self, query: str, params: Mapping[str, Any] | None = None ) -> Sequence[Mapping[str, Any]]: - """Execute a custom SurrealQL query. - - Args: - query: SurrealQL query string. - params: Optional query parameters. - - Returns: - Query results. - """ - if not self.db: + if self.db is None: raise RuntimeError("Database not connected") + rows = _first_statement(await self.db.query(query, dict(params) if params else None)) + if not isinstance(rows, list): + raise RuntimeError("Expected SurrealDB records") + return cast(Sequence[Mapping[str, Any]], rows) - try: - result = await self.db.query(query, dict(params) if params else None) - return cast(Sequence[Mapping[str, Any]], _first_statement(result)) - except Exception as e: - logger.error("Query failed", query=query, error=str(e)) - raise + async def _record(self, record_id: str) -> Mapping[str, Any] | None: + return _single(await self.query("SELECT * FROM $id;", {"id": StringRecordId(record_id)}), record_id) - async def get_transcript_by_source(self, source_path: str) -> dict[str, Any] | None: - """Get a transcript by its source path. + async def _merge(self, record_id: str, changes: Mapping[str, Any]) -> None: + rows = await self.query( + "UPDATE $id MERGE $changes;", {"id": StringRecordId(record_id), "changes": dict(changes)} + ) + if _single(rows, record_id) is None: + raise RuntimeError(f"Record disappeared during update: {record_id}") - Args: - source_path: Path to the source transcript file. + async def get_entity_by_name( + self, name: str, *, entity_type: str, source: str + ) -> Mapping[str, Any] | None: + if not source.strip(): + raise ValueError("A trusted transcript source is required") + return _single(await self.query( + "SELECT * FROM entity WHERE source = $source AND type = $type AND name = $name;", + {"source": source, "type": entity_type, "name": name}, + ), "source-qualified entity") - Returns: - Transcript data or None if not found. - """ - if not self.db: + async def create_entity(self, entity: Entity) -> str: + if self.db is None: raise RuntimeError("Database not connected") - - try: - rows = _first_statement( - await self.db.query( - "SELECT * FROM transcript WHERE source = $source LIMIT 1", - {"source": source_path}, - ) - ) - - if rows: - return cast(dict[str, Any], rows[0]) - return None - - except Exception as e: - logger.error("Failed to get transcript by source", source=source_path, error=str(e)) - raise - - async def get_unprocessed_transcript_paths(self) -> Sequence[str]: - """Get paths of all unprocessed transcripts. - - Returns: - List of source paths for unprocessed transcripts. - """ - if not self.db: + existing = await self.get_entity_by_name(entity.name, entity_type=entity.type, source=entity.source) + payload: dict[str, Any] = entity.model_dump() + if existing: + # MERGE, not replacement: preserve identity and original creation time. + payload["confidence"] = max(entity.confidence, existing["confidence"]) + payload["context"] = list(dict.fromkeys([*existing["context"], *entity.context])) + payload["description"] = entity.description or existing.get("description", "") + entity_id = str(existing["id"]) + await self._merge(entity_id, payload) + return entity_id + payload["created_at"] = datetime.now(UTC) + row = await self.db.create("entity", payload) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Entity creation returned no record") + return str(row["id"]) + + async def create_relationship( + self, relationship: Relationship, *, source_id: str, target_id: str + ) -> str: + if self.db is None: raise RuntimeError("Database not connected") + for record_id, name, entity_type in ( + (source_id, relationship.source, relationship.source_type), + (target_id, relationship.target, relationship.target_type), + ): + endpoint = await self._record(record_id) + if endpoint is None or ( + endpoint.get("source"), endpoint.get("type"), endpoint.get("name") + ) != (relationship.transcript_source, entity_type, name): + raise ValueError("Relationship endpoint does not match trusted source/type/name") + params = {"source": relationship.transcript_source, "in": StringRecordId(source_id), + "out": StringRecordId(target_id), "type": relationship.relationship_type} + existing = _single(await self.query( + "SELECT * FROM relates WHERE source = $source AND in = $in AND out = $out " + "AND relationship_type = $type;", params + ), "source-qualified relationship") + payload: dict[str, Any] = {"source": relationship.transcript_source, + "relationship_type": relationship.relationship_type, + "description": relationship.description, "confidence": relationship.confidence, + "context": relationship.context, "metadata": relationship.metadata} + if existing: + edge_id = str(existing["id"]) + await self._merge(edge_id, payload) + return edge_id + # Deterministic record identity also prevents duplicate edges on concurrent retries. + key = json.dumps([relationship.transcript_source, source_id, target_id, + relationship.relationship_type], ensure_ascii=False, separators=(",", ":")) + edge = RecordID("relates", hashlib.sha256(key.encode("utf-8")).hexdigest()) + if await self._record(str(edge)) is not None: + raise RuntimeError("Relationship identity collision; refusing to overwrite") + payload["created_at"] = datetime.now(UTC) + row = await self.db.relate(StringRecordId(source_id), edge, StringRecordId(target_id), payload) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Relationship creation returned no record") + return str(row["id"]) - try: - rows = _first_statement( - await self.db.query("SELECT source FROM transcript WHERE processed = false") - ) + async def create_transcript(self, transcript: Transcript) -> str: + if self.db is None: + raise RuntimeError("Database not connected") + payload = transcript.model_dump() + payload.update(date=_utc(transcript.date), created_at=_utc(transcript.created_at), processed=False) + row = await self.db.create("transcript", payload) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Transcript creation returned no record") + return str(row["id"]) - if rows: - return [rec["source"] for rec in rows if rec.get("source")] - return [] + async def get_transcript_by_source(self, source_path: str) -> Mapping[str, Any] | None: + return _single(await self.query("SELECT * FROM transcript WHERE source = $source;", + {"source": source_path}), "transcript source") - except Exception as e: - logger.error("Failed to get unprocessed transcript paths", error=str(e)) - raise + async def get_unprocessed_transcript_paths(self) -> Sequence[str]: + rows = await self.query("SELECT source FROM transcript WHERE processed = false;") + paths = [] + for row in rows: + source = row.get("source") + if not isinstance(source, str) or not source.strip(): + raise RuntimeError("Pending transcript has no trusted source path; explicit review required") + paths.append(source) + return paths async def update_transcript_status(self, transcript_id: str, processed: bool) -> None: - """Update the processed status of a transcript. - - Args: - transcript_id: ID of the transcript to update. - processed: New processed status. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - await self.db.update(transcript_id, {"processed": processed}) - logger.debug( - "Updated transcript status", transcript_id=transcript_id, processed=processed - ) - - except Exception as e: - logger.error( - "Failed to update transcript status", - transcript_id=transcript_id, - error=str(e), - ) - raise + await self._merge(transcript_id, {"processed": processed}) async def get_or_create_transcript(self, transcript: Transcript) -> str: - """Get existing transcript by source path or create new one. - - Args: - transcript: Transcript to get or create. - - Returns: - Transcript ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Check if transcript exists by source path - existing = await self.get_transcript_by_source(transcript.source) - - if existing: - transcript_id = existing["id"] - logger.debug("Found existing transcript", transcript_id=transcript_id) - # Update content and metadata if needed - await self.db.update( - transcript_id, - { - "title": transcript.title, - "content": transcript.content, - "metadata": transcript.metadata, - }, - ) - return str(transcript_id) - else: - # Create new transcript - return await self.create_transcript(transcript) - - except Exception as e: - logger.error( - "Failed to get or create transcript", - source=transcript.source, - error=str(e), - ) - raise + existing = await self.get_transcript_by_source(transcript.source) + if existing: + transcript_id = str(existing["id"]) + # Do not rewrite historical date/source/created_at on a retry. + await self._merge(transcript_id, {"title": transcript.title, "content": transcript.content, + "metadata": transcript.metadata, "processed": False}) + return transcript_id + return await self.create_transcript(transcript) diff --git a/src/journal_utilities/rag/main.py b/src/journal_utilities/rag/main.py index 33cd179f..11ed8206 100644 --- a/src/journal_utilities/rag/main.py +++ b/src/journal_utilities/rag/main.py @@ -1,16 +1,18 @@ -"""Main processing pipeline for JournalRAG.""" +"""Select transcript extraction work explicitly; readiness alone is not a run.""" +import argparse import asyncio +import json import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from pathlib import Path import structlog -from journal_utilities.rag.adapters import EntityAdapter -from journal_utilities.rag.extractors import CohereExtractor -from journal_utilities.rag.graph import SurrealDBClient +from journal_utilities.rag.adapters.entity_adapter import EntityAdapter +from journal_utilities.rag.extractors.cohere_extractor import CohereExtractor +from journal_utilities.rag.graph.surreal_client import SurrealDBClient from journal_utilities.rag.models import Transcript from journal_utilities.rag.settings import settings from journal_utilities.rag.utils import setup_logging @@ -18,264 +20,141 @@ logger = structlog.get_logger(__name__) -def _date_from_filename(name: str) -> datetime: - """Extract a deterministic transcript date from a filename. - - Prefers an embedded ``YYYY[-_/. ]MM[-_/. ]DD``; otherwise falls back to a - fixed epoch (datetime.min) instead of ``datetime.now()`` so reprocessing - the same file is deterministic. - """ - m = re.search(r"(\d{4})[-_/. ](\d{1,2})[-_/. ](\d{1,2})", name) - if m: +def _date_from_filename(filename: str) -> datetime: + """Use a valid filename date, otherwise the deterministic unknown-date sentinel.""" + match = re.search(r"(\d{4})[-_/. ](\d{1,2})[-_/. ](\d{1,2})", filename) + if match: try: - return datetime(int(m.group(1)), int(m.group(2)), int(m.group(3))) + return datetime(*(int(part) for part in match.groups())) except ValueError: pass return datetime.min +def _load_transcript(path: Path, *, source: str) -> Transcript: + return Transcript(title=path.stem, date=_date_from_filename(path.stem), + content=path.read_text(encoding="utf-8"), source=source) + + class JournalRAGPipeline: - """Main pipeline for extracting entities from transcripts and storing in graph database.""" + """A transcript is complete only after every required entity and edge persists.""" def __init__(self) -> None: - """Initialize the pipeline.""" setup_logging(debug=settings.debug) - self.extractor = CohereExtractor() + # Selection/migration/empty queues must not require or invoke a provider. + self.extractor: CohereExtractor | None = None self.db_client = SurrealDBClient() - logger.info("Initialized JournalRAG pipeline") async def connect(self) -> None: - """Connect to the database.""" await self.db_client.connect() async def disconnect(self) -> None: - """Disconnect from the database.""" await self.db_client.disconnect() async def process_transcript(self, transcript: Transcript) -> Mapping[str, int]: - """Process a single transcript. - - This method: - 1. Gets or creates the transcript record in the database - 2. Extracts entities using CoreEntities schema - 3. Converts CoreEntities to simple Entity model for storage - 4. Stores entities in the database - 5. Extracts relationships from CoreEntities structure - 6. Stores relationships in the database - 7. Updates transcript as processed - - Args: - transcript: The transcript to process. - - Returns: - Dictionary with processing statistics. - """ - logger.info("Processing transcript", title=transcript.title, source=transcript.source) - - # Get or create transcript record (idempotent) transcript_id = await self.db_client.get_or_create_transcript(transcript) - logger.info("Transcript record ready", transcript_id=transcript_id) - - # Extract entities using new CoreEntities schema - core_entities = self.extractor.extract_core_entities(transcript.content) - logger.info("Extracted core entities from transcript") - - # Convert CoreEntities to simple Entity model for storage - entities = EntityAdapter.convert_core_entities(core_entities) - logger.info("Converted to storage format", entity_count=len(entities)) - - # Store entities - entity_count = 0 + await self.db_client.update_transcript_status(transcript_id, processed=False) + if self.extractor is None: + self.extractor = CohereExtractor() + core = self.extractor.extract_core_entities(transcript.content) + entities = EntityAdapter.convert_core_entities(core, source=transcript.source) + relationships = EntityAdapter.extract_relationships(core, source=transcript.source) + identities = {(entity.type, entity.name) for entity in entities} + for relationship in relationships: + for identity in ((relationship.source_type, relationship.source), + (relationship.target_type, relationship.target)): + if identity not in identities: + raise ValueError(f"Required relationship endpoint missing: {identity!r}") + entity_ids = {} for entity in entities: - await self.db_client.create_entity(entity) - entity_count += 1 - - # Extract relationships from CoreEntities structure - relationships = EntityAdapter.extract_relationships(core_entities) - logger.info("Extracted relationships", relationship_count=len(relationships)) - - # Store relationships - relationship_count = 0 + entity_ids[(entity.type, entity.name)] = await self.db_client.create_entity(entity) for relationship in relationships: - try: - await self.db_client.create_relationship(relationship) - relationship_count += 1 - except Exception as e: - logger.warning( - "Failed to create relationship", - source=relationship.source, - target=relationship.target, - error=str(e), - ) - continue - - # Mark transcript as processed + # Returned IDs, not a fresh name-only search, choose the endpoints. + await self.db_client.create_relationship( + relationship, + source_id=entity_ids[(relationship.source_type, relationship.source)], + target_id=entity_ids[(relationship.target_type, relationship.target)], + ) await self.db_client.update_transcript_status(transcript_id, processed=True) - - logger.info( - "Completed processing transcript", - title=transcript.title, - entities=entity_count, - relationships=relationship_count, - ) - - return { - "entities": entity_count, - "relationships": relationship_count, - } + return {"entities": len(entity_ids), "relationships": len(relationships)} async def process_transcript_file(self, file_path: Path) -> Mapping[str, int]: - """Process a transcript from a file. - - Args: - file_path: Path to the transcript file. - - Returns: - Dictionary with processing statistics. - """ - logger.info("Processing transcript file", file_path=str(file_path)) - - # Read transcript content - content = file_path.read_text(encoding="utf-8") - - # Create transcript object (deterministic date from the filename) - transcript = Transcript( - title=file_path.stem, - date=_date_from_filename(file_path.name), - content=content, - source=str(file_path), - ) - - return await self.process_transcript(transcript) + return await self.process_transcript(_load_transcript(file_path, source=str(file_path))) async def process_from_database(self, base_path: Path | None = None) -> Mapping[str, int]: - """Process all unprocessed transcripts from the database. - - This orchestrator method: - 1. Queries SurrealDB for unprocessed transcript paths - 2. Loads each transcript from the local repository - 3. Processes each transcript (extract entities, store in DB) - 4. Handles errors gracefully to continue processing - - Args: - base_path: Optional base path for transcript files. If provided, paths from - the database will be resolved relative to this base. If None, paths - are assumed to be absolute. - - Returns: - Dictionary with processing statistics including: - - total: Total transcripts found in DB - - processed: Number successfully processed - - failed: Number that failed - - total_entities: Total entities extracted - - total_relationships: Total relationships extracted - """ - logger.info("Starting batch processing from database") - - # Get unprocessed transcript paths from database - transcript_paths = await self.db_client.get_unprocessed_transcript_paths() - logger.info("Found unprocessed transcripts", count=len(transcript_paths)) - - if not transcript_paths: - logger.info("No unprocessed transcripts found") - return { - "total": 0, - "processed": 0, - "failed": 0, - "total_entities": 0, - "total_relationships": 0, - } - - # Process each transcript - processed_count = 0 - failed_count = 0 - total_entities = 0 - total_relationships = 0 - - for source_path in transcript_paths: + paths = await self.db_client.get_unprocessed_transcript_paths() + totals = {"total": len(paths), "processed": 0, "failed": 0, + "total_entities": 0, "total_relationships": 0} + for source in paths: + path = base_path / source if base_path is not None else Path(source) try: - # Resolve file path - if base_path: - file_path = base_path / source_path - else: - file_path = Path(source_path) - - # Check if file exists - if not file_path.exists(): - logger.warning("Transcript file not found", path=str(file_path)) - failed_count += 1 - continue - - # Load transcript content - content = file_path.read_text(encoding="utf-8") - - # Create transcript object - transcript = Transcript( - title=file_path.stem, - date=datetime.now(), # You may want to extract date from filename - content=content, - source=source_path, - ) - - # Process transcript - stats = await self.process_transcript(transcript) - - # Update totals - processed_count += 1 - total_entities += stats["entities"] - total_relationships += stats["relationships"] - - logger.info( - "Processed transcript successfully", - path=source_path, - entities=stats["entities"], - relationships=stats["relationships"], - ) - - except Exception as e: - logger.error( - "Failed to process transcript", - path=source_path, - error=str(e), - exc_info=True, - ) - failed_count += 1 - continue - - logger.info( - "Batch processing complete", - total=len(transcript_paths), - processed=processed_count, - failed=failed_count, - total_entities=total_entities, - total_relationships=total_relationships, - ) - - return { - "total": len(transcript_paths), - "processed": processed_count, - "failed": failed_count, - "total_entities": total_entities, - "total_relationships": total_relationships, - } - - -async def main() -> None: - """Main entry point.""" + stats = await self.process_transcript(_load_transcript(path, source=source)) + totals["processed"] += 1 + totals["total_entities"] += stats["entities"] + totals["total_relationships"] += stats["relationships"] + except Exception as exc: + totals["failed"] += 1 + logger.error("Transcript incomplete; remains pending", source=source, error=str(exc)) + return totals + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--file", nargs="+", type=Path, dest="files", help="Extract these transcript files") + selection.add_argument("--pending", action="store_true", help="Extract pending database source paths") + selection.add_argument("--migrate-source-identity", action="store_true", + help="Explicit schema migration; quiescent database and backup required; no attribution backfill") + parser.add_argument("--base-path", type=Path, help="Resolve pending relative source paths here") + return parser + + +async def _run(args: argparse.Namespace) -> int: + if args.migrate_source_identity: + client = SurrealDBClient() + try: + await client.connect(migrate_source_identity=True) + print(json.dumps({"migration": client.migration_summary})) + return 0 + except Exception as exc: + logger.error("Source identity migration refused or failed", error=str(exc)) + return 1 + finally: + await client.disconnect() pipeline = JournalRAGPipeline() - try: await pipeline.connect() - logger.info("Pipeline ready") - - # Example: Process a transcript file - # transcript_path = Path("path/to/transcript.txt") - # stats = await pipeline.process_transcript_file(transcript_path) - # logger.info("Processing complete", stats=stats) - + stats: dict[str, int] + if args.pending: + stats = dict(await pipeline.process_from_database(base_path=args.base_path)) + else: + stats = {"total": len(args.files), "processed": 0, "failed": 0, + "total_entities": 0, "total_relationships": 0} + for path in args.files: + try: + result = await pipeline.process_transcript_file(path) + stats["processed"] += 1 + stats["total_entities"] += result["entities"] + stats["total_relationships"] += result["relationships"] + except Exception as exc: + stats["failed"] += 1 + logger.error("Transcript incomplete", source=str(path), error=str(exc)) + print(json.dumps(stats)) + return 1 if stats["failed"] else 0 + except Exception as exc: + logger.error("Extraction run failed", error=str(exc)) + return 1 finally: await pipeline.disconnect() +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if args.base_path is not None and not args.pending: + parser.error("--base-path is only valid with --pending") + return asyncio.run(_run(args)) + + if __name__ == "__main__": - asyncio.run(main()) + raise SystemExit(main()) diff --git a/src/journal_utilities/rag/models/entities.py b/src/journal_utilities/rag/models/entities.py index 9069cb4f..573035d6 100644 --- a/src/journal_utilities/rag/models/entities.py +++ b/src/journal_utilities/rag/models/entities.py @@ -268,6 +268,7 @@ class Transcript(BaseModel): class Entity(BaseModel): """Simple generic entity for database storage.""" + source: str = Field(..., min_length=1, pattern=r"\S", description="Trusted transcript source") name: str = Field(..., description="Entity name") type: str = Field(..., description="Entity type (concept, researcher, citation, etc.)") description: str = Field(default="", description="Description or definition") @@ -280,6 +281,9 @@ class Entity(BaseModel): class Relationship(BaseModel): """Simple generic relationship for database storage.""" + transcript_source: str = Field(..., min_length=1, pattern=r"\S", description="Trusted transcript source") + source_type: str = Field(..., description="Source entity type") + target_type: str = Field(..., description="Target entity type") source: str = Field(..., description="Source entity name") target: str = Field(..., description="Target entity name") relationship_type: str = Field(..., description="Type of relationship") diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 4748f3f9..496359b4 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -2,9 +2,10 @@ ## Test Suite Overview -The suite is a live snapshot (currently ~475 passing, 3 API-gated skips, snapshot -of `uv run pytest tests/ -q` on 2026-08-02); run -`uv run pytest tests/ -q` for the current count. +Counts and coverage are generated measurements, not maintained prose. See +[`../docs/_generated/gate-arms.md`](../docs/_generated/gate-arms.md) for the +recorded source, runtime, results and limitations. Regenerate only with the +documented gate prerequisites available; historical receipts are not current proof. ## Running Tests @@ -60,6 +61,12 @@ async def test_real_extraction(): # ... ``` +SDK doubles must match the installed SDK's actual return shapes. They cannot +qualify database schema or persistence behavior. The disposable-database +acceptance in `journal_utilities/rag/test_source_identity.py` requires explicit +service admission and checks migration, source isolation, failure recovery and +replay against a real server. A skipped database case is not qualification. + ## Adding Tests 1. Pipeline tests: `tests/journal_utilities/test_*.py` diff --git a/tests/journal_utilities/rag/integration/test_pipeline_integration.py b/tests/journal_utilities/rag/integration/test_pipeline_integration.py index ad88127b..1e1e787d 100644 --- a/tests/journal_utilities/rag/integration/test_pipeline_integration.py +++ b/tests/journal_utilities/rag/integration/test_pipeline_integration.py @@ -10,7 +10,7 @@ import os from datetime import datetime from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, call import pytest @@ -27,7 +27,7 @@ @pytest.fixture def sample_transcript_file() -> Path: """Get path to sample transcript file.""" - return Path(__file__).parent.parent.parent / "fixtures" / "sample_transcript.txt" + return Path(__file__).parent.parent.parent.parent / "fixtures" / "sample_transcript.txt" @pytest.fixture @@ -48,7 +48,7 @@ def mock_db_client(): """Mock database client to avoid needing SurrealDB.""" mock = AsyncMock() mock.get_or_create_transcript.return_value = "transcript:test-001" - mock.create_entity.return_value = "entity:test-001" + mock.create_entity.side_effect = lambda entity: f"entity:{entity.type}-{entity.name}" mock.create_relationship.return_value = "relates:test-001" mock.update_transcript_status.return_value = None return mock @@ -91,9 +91,10 @@ async def test_process_transcript_with_real_extractor( # Verify database interactions occurred mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True - ) + mock_db_client.update_transcript_status.assert_has_awaits([ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ]) # Verify stats from real extraction assert "entities" in stats diff --git a/tests/journal_utilities/rag/test_source_identity.py b/tests/journal_utilities/rag/test_source_identity.py new file mode 100644 index 00000000..81f02c64 --- /dev/null +++ b/tests/journal_utilities/rag/test_source_identity.py @@ -0,0 +1,154 @@ +"""Opt-in real SurrealDB schema acceptance. No Cohere requests. + +Requires an existing disposable service, explicit JOURNAL_TEST_SURREALDB_ALLOW_DISPOSABLE=1, +JOURNAL_TEST_SURREALDB_URL, JOURNAL_TEST_SURREALDB_NAMESPACE, USERNAME and PASSWORD. +Creates/removes only a fresh UUID-named database in that namespace. A skip is not proof. +""" + +import os +from datetime import UTC, datetime +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest + +pytest.importorskip("surrealdb") +pytest.importorskip("structlog") +pytest.importorskip("cohere") + +from surrealdb import AsyncSurreal, RecordID + +from journal_utilities.rag.graph.surreal_client import SurrealDBClient +from journal_utilities.rag.main import JournalRAGPipeline +from journal_utilities.rag.models import Concept, ConceptCategory, CoreEntities, Relationship, Transcript + + +@pytest.fixture +async def disposable_database(): + prefix = "JOURNAL_TEST_SURREALDB_" + required = ("URL", "NAMESPACE", "USERNAME", "PASSWORD") + if os.environ.get(prefix + "ALLOW_DISPOSABLE") != "1" or not all(os.environ.get(prefix + name) for name in required): + pytest.skip("Explicit disposable existing SurrealDB service credentials required; no DB proof") + options = {name.lower(): os.environ[prefix + name] for name in required} + options["database"] = "journal_group07_test_" + uuid4().hex + raw = AsyncSurreal(options["url"]) + await raw.connect() + try: + await raw.signin({"username": options["username"], "password": options["password"]}) + await raw.use(options["namespace"], options["database"]) + yield options, raw + finally: + # Generated identifier contains only this literal prefix and UUID hex. + await raw.query(f"REMOVE DATABASE IF EXISTS {options['database']};") + await raw.close() + + +async def test_legacy_migration_source_identity_retry_and_dates(disposable_database, monkeypatch): + options, raw = disposable_database + # Historical schema: global name/type uniqueness, no source field on graph records. + await raw.query(""" + DEFINE TABLE entity SCHEMAFULL; + DEFINE FIELD name ON entity TYPE string; + DEFINE FIELD type ON entity TYPE string; + DEFINE FIELD description ON entity TYPE option; + DEFINE FIELD confidence ON entity TYPE number; + DEFINE FIELD mentions ON entity TYPE number; + DEFINE FIELD context ON entity TYPE array; + DEFINE FIELD metadata ON entity TYPE object; + DEFINE FIELD created_at ON entity TYPE datetime; + DEFINE INDEX entity_name_type ON entity FIELDS name, type UNIQUE; + DEFINE TABLE relates SCHEMAFULL TYPE RELATION; + DEFINE FIELD relationship_type ON relates TYPE string; + DEFINE FIELD description ON relates TYPE option; + DEFINE FIELD confidence ON relates TYPE number; + DEFINE FIELD context ON relates TYPE option; + DEFINE FIELD metadata ON relates TYPE object; + DEFINE FIELD created_at ON relates TYPE datetime; + """) + payload = {"description": "Legacy", "confidence": 1.0, "mentions": 1, "context": [], + "metadata": {}, "created_at": datetime(2000, 1, 1, tzinfo=UTC)} + await raw.create(RecordID("entity", "legacy_insight"), dict(payload, name="Insight 1", type="key_insight")) + await raw.create(RecordID("entity", "legacy_concept"), dict(payload, name="Shared", type="concept")) + await raw.relate(RecordID("entity", "legacy_concept"), RecordID("relates", "legacy"), + RecordID("entity", "legacy_insight"), + {"relationship_type": "related_to", "description": "Legacy", "confidence": 1.0, + "context": "", "metadata": {}, "created_at": datetime(2000, 1, 1, tzinfo=UTC)}) + legacy_entities = (await raw.query("SELECT * FROM entity ORDER BY id;"))[0] + legacy_edges = (await raw.query("SELECT * FROM relates ORDER BY id;"))[0] + refused = SurrealDBClient(**options) + with pytest.raises(RuntimeError, match="--migrate-source-identity"): + await refused.connect() + client = SurrealDBClient(**options) + await client.connect(migrate_source_identity=True) + try: + assert client.migration_summary["unknown_source_entities"] == 2 + assert client.migration_summary["unknown_source_relationships"] == 1 + assert (await raw.query("SELECT * FROM entity ORDER BY id;"))[0] == legacy_entities + assert (await raw.query("SELECT * FROM relates ORDER BY id;"))[0] == legacy_edges + info = (await raw.query("INFO FOR TABLE entity;"))[0] + assert "entity_name_type" not in info["indexes"] + assert "entity_source_type_name" in info["indexes"] + core = CoreEntities(concepts=[ + Concept(name="Shared", definition="A", category=ConceptCategory.THEORETICAL, related_concepts=["Target"]), + Concept(name="Target", definition="B", category=ConceptCategory.THEORETICAL, related_concepts=["Shared"]), + ], researchers=[], citations=[], technical_terms=[], key_insights=["An insight"]) + pipeline = JournalRAGPipeline() + pipeline.db_client = client + pipeline.extractor = MagicMock() + pipeline.extractor.extract_core_entities.return_value = core + a = Transcript(title="A", date=datetime.min, source="a.txt", content="A") + b = Transcript(title="B", date=datetime(2025, 10, 30), source="b.txt", content="B") + await pipeline.process_transcript(a) + relate = client.db.relate + calls = 0 + + def fail_second_edge(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("Injected external edge write failure") + return relate(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(client.db, "relate", fail_second_edge) + with pytest.raises(RuntimeError, match="edge write failure"): + await pipeline.process_transcript(b) + pending = await client.get_transcript_by_source("b.txt") + assert pending["processed"] is False and pending["content"] == "B" + assert pending["source"] == "b.txt" + original_date = pending["date"] + await pipeline.process_transcript(b.model_copy(update={"date": datetime.min})) + assert (await client.get_transcript_by_source("b.txt"))["date"] == original_date + before_replay = await client.query("SELECT * FROM entity;") + edge_ids = {str(row["id"]) for row in await client.query("SELECT * FROM relates;")} + await pipeline.process_transcript(a) + await pipeline.process_transcript(b) + entities = await client.query("SELECT * FROM entity;") + edges = await client.query("SELECT * FROM relates;") + assert {str(row["id"]) for row in entities} == {str(row["id"]) for row in before_replay} + assert {str(row["id"]) for row in edges} == edge_ids + assert len(entities) == 8 and len(edges) == 5 + by_id = {str(row["id"]): row for row in entities} + for edge in edges: + if edge.get("source"): + assert by_id[str(edge["in"])]["source"] == edge["source"] == by_id[str(edge["out"])]["source"] + assert sorted((row for row in entities if not row.get("source")), key=lambda row: str(row["id"])) == legacy_entities + assert [row for row in edges if not row.get("source")] == legacy_edges + assert {row["source"] for row in entities if row["name"] == "Insight 1" and row.get("source")} == {"a.txt", "b.txt"} + assert (await client.get_transcript_by_source("a.txt"))["processed"] is True + assert (await client.get_transcript_by_source("b.txt"))["processed"] is True + a_entity = await client.get_entity_by_name("Shared", entity_type="concept", source="a.txt") + b_entity = await client.get_entity_by_name("Target", entity_type="concept", source="b.txt") + edge = Relationship(transcript_source="a.txt", source="Shared", source_type="concept", + target="Target", target_type="concept", relationship_type="related_to") + with pytest.raises(ValueError, match="trusted source/type/name"): + await client.create_relationship(edge, source_id=str(a_entity["id"]), target_id=str(b_entity["id"])) + # Explicit migration must refuse contradictory claims without changing data. + await raw.query("UPDATE $id MERGE { metadata: { source: 'contradictory.txt' } };", {"id": a_entity["id"]}) + snapshot = await client.query("SELECT * FROM entity ORDER BY id;") + ambiguous = SurrealDBClient(**options) + with pytest.raises(RuntimeError, match="Conflicting legacy source claims"): + await ambiguous.connect(migrate_source_identity=True) + assert await client.query("SELECT * FROM entity ORDER BY id;") == snapshot + finally: + await client.disconnect() diff --git a/tests/journal_utilities/rag/unit/test_entity_adapter.py b/tests/journal_utilities/rag/unit/test_entity_adapter.py index a32ab1e6..dd61e284 100644 --- a/tests/journal_utilities/rag/unit/test_entity_adapter.py +++ b/tests/journal_utilities/rag/unit/test_entity_adapter.py @@ -2,6 +2,9 @@ Unit tests for EntityAdapter conversion functions. """ +import pytest +from pydantic import ValidationError + from journal_utilities.rag.adapters.entity_adapter import EntityAdapter from journal_utilities.rag.models import Entity from journal_utilities.rag.models.entities import ( @@ -26,7 +29,7 @@ def test_convert_empty_entities(self): concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=[] ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) == 0 @@ -47,7 +50,7 @@ def test_convert_concepts(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 concept_entity = next((e for e in entities if e.name == "Active Inference"), None) @@ -66,7 +69,7 @@ def test_convert_researchers(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 researcher_entity = next((e for e in entities if e.name == "Karl Friston"), None) @@ -90,7 +93,7 @@ def test_convert_citations(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 citation_entity = next((e for e in entities if "Free Energy Principle" in e.name), None) @@ -113,7 +116,7 @@ def test_convert_technical_terms(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 term_entity = next((e for e in entities if e.name == "Variational Free Energy"), None) @@ -130,7 +133,7 @@ def test_convert_key_insights(self): key_insights=["Active Inference unifies perception and action"], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 # The adapter names insights as "Insight 1", "Insight 2", etc. @@ -156,7 +159,7 @@ def test_convert_all_types(self): key_insights=["Insight1"], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") # Should have at least 5 entities (one of each type) assert len(entities) >= 5 @@ -178,7 +181,7 @@ def test_extract_relationships_empty(self): concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=[] ) - relationships = EntityAdapter.extract_relationships(core) + relationships = EntityAdapter.extract_relationships(core, source="transcripts/a.txt") assert len(relationships) == 0 @@ -199,7 +202,7 @@ def test_extract_relationships_from_concepts(self): key_insights=[], ) - relationships = EntityAdapter.extract_relationships(core) + relationships = EntityAdapter.extract_relationships(core, source="transcripts/a.txt") # Should have 2 relationships from related_concepts assert len(relationships) >= 2 @@ -211,6 +214,7 @@ class TestEntityModel: def test_entity_creation(self): """Test basic entity creation.""" entity = Entity( + source="transcripts/a.txt", name="Test Entity", type="concept", description="A test", @@ -225,16 +229,15 @@ def test_entity_creation(self): assert entity.mentions == 3 assert len(entity.context) == 2 - def test_entity_default_values(self): - """Test entity has sensible defaults.""" - entity = Entity(name="Minimal", type="concept", confidence=0.8, mentions=1, context=[]) - - assert entity.description == "" - assert entity.metadata == {} + @pytest.mark.parametrize("source", [None, "", " "]) + def test_entity_requires_trusted_source(self, source): + with pytest.raises(ValidationError): + Entity(name="Insight 1", type="key_insight", source=source) def test_entity_with_metadata(self): """Test entity with custom metadata.""" entity = Entity( + source="transcripts/a.txt", name="Test", type="concept", confidence=0.9, @@ -245,3 +248,15 @@ def test_entity_with_metadata(self): assert entity.metadata["source"] == "transcript" assert entity.metadata["importance"] == "high" + + +def test_caller_source_overrides_extraction_metadata(sample_core_entities): + from journal_utilities.rag.models import ExtractedEntities + + extracted = ExtractedEntities(core=sample_core_entities, metadata={"source": "untrusted"}) + entities, relationships = EntityAdapter.convert_extracted_entities(extracted, source="trusted.txt") + assert {entity.source for entity in entities} == {"trusted.txt"} + assert {relationship.transcript_source for relationship in relationships} == {"trusted.txt"} + assert {(relationship.source_type, relationship.target_type) for relationship in relationships} == { + ("researcher", "citation") + } diff --git a/tests/journal_utilities/rag/unit/test_pipeline.py b/tests/journal_utilities/rag/unit/test_pipeline.py index f0186e90..5da8d211 100644 --- a/tests/journal_utilities/rag/unit/test_pipeline.py +++ b/tests/journal_utilities/rag/unit/test_pipeline.py @@ -6,7 +6,7 @@ from datetime import datetime from pathlib import Path -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, call import pytest @@ -15,7 +15,7 @@ pytest.importorskip("cohere") pytest.importorskip("surrealdb") -from journal_utilities.rag.main import JournalRAGPipeline +from journal_utilities.rag.main import JournalRAGPipeline, main from journal_utilities.rag.models import CoreEntities, Transcript @@ -43,7 +43,7 @@ def mock_db_client(): """Mock database client.""" mock = AsyncMock() mock.get_or_create_transcript.return_value = "transcript:test-001" - mock.create_entity.return_value = "entity:test-001" + mock.create_entity.side_effect = lambda entity: f"entity:{entity.source}-{entity.type}-{entity.name}" mock.create_relationship.return_value = "relates:test-001" mock.update_transcript_status.return_value = None return mock @@ -83,9 +83,10 @@ async def test_process_transcript_with_file( # Verify database interactions mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True - ) + mock_db_client.update_transcript_status.assert_has_awaits([ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ]) # Verify extractor was called mock_extractor.extract_core_entities.assert_called_once_with(sample_transcript.content) @@ -135,7 +136,7 @@ async def test_process_transcript_handles_relationship_errors( mock_db_client, mock_extractor, ) -> None: - """Test that process_transcript handles relationship creation errors gracefully.""" + """A required edge failure is visible, stays pending, and can be retried.""" # Create pipeline with mocked dependencies pipeline = JournalRAGPipeline() pipeline.db_client = mock_db_client @@ -144,14 +145,17 @@ async def test_process_transcript_handles_relationship_errors( # Make relationship creation fail - the sample fixture only generates 1 relationship mock_db_client.create_relationship.side_effect = ValueError("Entity not found") - # Process the transcript - should not raise exception + with pytest.raises(ValueError, match="Entity not found"): + await pipeline.process_transcript(sample_transcript) + assert mock_db_client.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] + mock_db_client.create_relationship.side_effect = None stats = await pipeline.process_transcript(sample_transcript) - - # Should have processed all entities but failed on relationships - assert "entities" in stats - assert "relationships" in stats - # Should have 0 successful relationships since we made it fail - assert stats["relationships"] == 0 + assert stats["relationships"] == 1 + assert mock_db_client.update_transcript_status.await_args == call( + "transcript:test-001", processed=True + ) @pytest.mark.asyncio @@ -171,9 +175,10 @@ async def test_process_transcript_updates_transcript_status( await pipeline.process_transcript(sample_transcript) # Verify transcript was marked as processed - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True - ) + mock_db_client.update_transcript_status.assert_has_awaits([ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ]) @pytest.mark.asyncio @@ -234,3 +239,144 @@ async def test_process_transcript_converts_entities_correctly( entity_names = {call[0][0].name for call in entity_calls} assert "Active Inference" in entity_names assert "Karl Friston" in entity_names + + +@pytest.mark.parametrize("filename, expected", [ + ("talk_2025-10-30.txt", datetime(2025, 10, 30)), + ("undated.txt", datetime.min), + ("talk_2025-02-30.txt", datetime.min), +]) +async def test_file_and_pending_loaders_share_date_and_source( + tmp_path, filename, expected, mock_db_client, mock_extractor +): + path = tmp_path / filename + path.write_text("Active Inference", encoding="utf-8") + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + await pipeline.process_transcript_file(path) + direct = mock_db_client.get_or_create_transcript.await_args.args[0] + mock_db_client.get_unprocessed_transcript_paths.return_value = [filename] + stats = await pipeline.process_from_database(base_path=tmp_path) + pending = mock_db_client.get_or_create_transcript.await_args.args[0] + assert stats["processed"] == 1 and stats["failed"] == 0 + assert direct.date == pending.date == expected + assert direct.source == str(path) + assert pending.source == filename + + +async def test_required_missing_endpoint_is_not_discarded( + sample_transcript, sample_core_entities, mock_db_client, mock_extractor +): + sample_core_entities.concepts[0].related_concepts = ["Not extracted"] + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + with pytest.raises(ValueError, match="Required relationship endpoint missing"): + await pipeline.process_transcript(sample_transcript) + assert mock_db_client.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] + mock_db_client.create_relationship.assert_not_awaited() + + +async def test_source_and_type_choose_returned_endpoint_ids( + sample_transcript, mock_db_client, mock_extractor +): + from journal_utilities.rag.models import Citation, CitationType, Concept, ConceptCategory, Researcher, ResearcherRole + + mock_extractor.extract_core_entities.return_value = CoreEntities( + concepts=[Concept(name="Same", definition="Concept", category=ConceptCategory.THEORETICAL, + related_concepts=["Same"])], + researchers=[Researcher(name="Same", role=ResearcherRole.SPEAKER)], + citations=[Citation(title="Same", type=CitationType.PAPER, authors=["Same"])], + technical_terms=[], key_insights=["One"], + ) + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + for source in ("a.txt", "b.txt"): + await pipeline.process_transcript(sample_transcript.model_copy(update={"source": source})) + edges = mock_db_client.create_relationship.await_args_list + assert len(edges) == 4 + for invocation in edges: + edge = invocation.args[0] + assert invocation.kwargs == { + "source_id": f"entity:{edge.transcript_source}-{edge.source_type}-{edge.source}", + "target_id": f"entity:{edge.transcript_source}-{edge.target_type}-{edge.target}", + } + assert {invocation.args[0].source for invocation in mock_db_client.create_entity.await_args_list} == {"a.txt", "b.txt"} + + +@pytest.fixture +def cli_boundaries(monkeypatch, mock_db_client, mock_extractor): + monkeypatch.setattr("journal_utilities.rag.main.SurrealDBClient", lambda: mock_db_client) + monkeypatch.setattr("journal_utilities.rag.main.CohereExtractor", lambda: mock_extractor) + return mock_db_client, mock_extractor + + +def test_cli_requires_work_selection(cli_boundaries): + db, extractor = cli_boundaries + with pytest.raises(SystemExit) as exc: + main([]) + assert exc.value.code == 2 + db.connect.assert_not_awaited() + extractor.extract_core_entities.assert_not_called() + + +def test_cli_files_perform_work_and_report_missing_file(tmp_path, cli_boundaries): + db, extractor = cli_boundaries + path = tmp_path / "2025-10-30.txt" + path.write_text("Active Inference", encoding="utf-8") + assert main(["--file", str(path), str(tmp_path / "missing.txt")]) == 1 + extractor.extract_core_entities.assert_called_once_with("Active Inference") + assert db.update_transcript_status.await_args == call("transcript:test-001", processed=True) + db.disconnect.assert_awaited_once() + + +def test_cli_pending_edge_failure_returns_failure_and_disconnects(tmp_path, cli_boundaries): + db, extractor = cli_boundaries + (tmp_path / "pending.txt").write_text("Pending", encoding="utf-8") + db.get_unprocessed_transcript_paths.return_value = ["pending.txt"] + db.create_relationship.side_effect = RuntimeError("edge write failed") + assert main(["--pending", "--base-path", str(tmp_path)]) == 1 + extractor.extract_core_entities.assert_called_once_with("Pending") + assert db.update_transcript_status.await_args_list == [call("transcript:test-001", processed=False)] + db.disconnect.assert_awaited_once() + + +def test_cli_empty_pending_does_not_invoke_provider(cli_boundaries): + db, extractor = cli_boundaries + db.get_unprocessed_transcript_paths.return_value = [] + assert main(["--pending"]) == 0 + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +def test_cli_connect_failure_still_disconnects(cli_boundaries): + db, extractor = cli_boundaries + db.connect.side_effect = RuntimeError("offline") + assert main(["--pending"]) == 1 + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +def test_cli_explicit_migration_does_not_extract(cli_boundaries): + db, extractor = cli_boundaries + db.migration_summary = {"unknown_source_entities": 2} + assert main(["--migrate-source-identity"]) == 0 + db.connect.assert_awaited_once_with(migrate_source_identity=True) + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +async def test_no_required_relationships_can_complete(sample_transcript, mock_db_client, mock_extractor): + mock_extractor.extract_core_entities.return_value = CoreEntities( + concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=["An insight"] + ) + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + assert await pipeline.process_transcript(sample_transcript) == {"entities": 1, "relationships": 0} + mock_db_client.create_relationship.assert_not_awaited() + assert mock_db_client.update_transcript_status.await_args == call("transcript:test-001", processed=True) diff --git a/tests/journal_utilities/rag/unit/test_surreal_client.py b/tests/journal_utilities/rag/unit/test_surreal_client.py index 28cf8df3..ef9e60ef 100644 --- a/tests/journal_utilities/rag/unit/test_surreal_client.py +++ b/tests/journal_utilities/rag/unit/test_surreal_client.py @@ -1,13 +1,10 @@ -""" -Unit tests for SurrealDBClient with mocks. -""" +"""Storage admission/error contracts at the SDK boundary; not real-schema proof.""" -from datetime import UTC, datetime -from unittest.mock import AsyncMock, patch +from datetime import datetime +from unittest.mock import AsyncMock import pytest -# Requires the rag extra; skip cleanly when it is not installed. pytest.importorskip("surrealdb") pytest.importorskip("structlog") @@ -15,304 +12,101 @@ from journal_utilities.rag.models import Entity, Relationship, Transcript -class TestSurrealDBClientInit: - """Tests for SurrealDBClient initialization.""" - - def test_default_initialization(self): - """Test client initializes with default settings.""" - with patch("journal_utilities.rag.graph.surreal_client.settings") as mock_settings: - mock_settings.surrealdb_url = "ws://localhost:8080" - mock_settings.surrealdb_namespace = "test" - mock_settings.surrealdb_database = "test" - mock_settings.surrealdb_username = "root" - mock_settings.surrealdb_password = "root" - - client = SurrealDBClient() - - assert client.url == "ws://localhost:8080" - assert client.namespace == "test" - assert client.database == "test" - - def test_custom_initialization(self): - """Test client with custom parameters.""" - client = SurrealDBClient( - url="ws://custom:9000", - namespace="custom_ns", - database="custom_db", - username="user", - password="pass", - ) - - assert client.url == "ws://custom:9000" - assert client.namespace == "custom_ns" - assert client.database == "custom_db" - - -class TestSurrealDBClientConnection: - """Tests for connection management.""" - - @pytest.fixture - def mock_surreal(self): - """Create a mock Surreal instance.""" - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.close = AsyncMock() - mock.query = AsyncMock(return_value=[]) - mock.create = AsyncMock(return_value=[{"id": "test:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.mark.asyncio - async def test_connect_success(self, mock_surreal): - """Test successful connection.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - - await client.connect() - - mock_surreal.connect.assert_called_once() - mock_surreal.signin.assert_called_once_with({"user": "user", "pass": "pass"}) - mock_surreal.use.assert_called_once_with("ns", "db") - - @pytest.mark.asyncio - async def test_disconnect(self, mock_surreal): - """Test disconnection.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - - await client.connect() - await client.disconnect() - - mock_surreal.close.assert_called_once() - - -class TestSurrealDBClientEntities: - """Tests for entity operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[]]) - mock.create = AsyncMock(return_value=[{"id": "entity:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.fixture - def sample_entity(self): - return Entity( - name="Test Entity", - type="concept", - description="A test entity", - confidence=0.9, - mentions=5, - context=["test context"], - ) - - @pytest.mark.asyncio - async def test_create_new_entity(self, mock_surreal, sample_entity): - """Test creating a new entity.""" - mock_surreal.query = AsyncMock(return_value=[[]]) # No existing - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_entity(sample_entity) - - assert result == "entity:1" - mock_surreal.create.assert_called_once() - - @pytest.mark.asyncio - async def test_update_existing_entity(self, mock_surreal, sample_entity): - """Test updating an existing entity.""" - mock_surreal.query = AsyncMock( - return_value=[ - [ - { - "id": "entity:existing", - "confidence": 0.8, - "context": ["old context"], - "description": "Old description", - } - ] - ] - ) - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_entity(sample_entity) - - assert result == "entity:existing" - mock_surreal.update.assert_called_once() - - -class TestSurrealDBClientTranscripts: - """Tests for transcript operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[]]) - mock.create = AsyncMock(return_value=[{"id": "transcript:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.fixture - def sample_transcript(self): - return Transcript( - title="Test Transcript", - date=datetime.now(UTC), - content="Test content", - source="/path/to/file.txt", - ) - - @pytest.mark.asyncio - async def test_create_transcript(self, mock_surreal, sample_transcript): - """Test creating a transcript.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_transcript(sample_transcript) - - assert result == "transcript:1" - mock_surreal.create.assert_called_once() - - @pytest.mark.asyncio - async def test_get_unprocessed_transcripts(self, mock_surreal): - """Test getting unprocessed transcript paths.""" - mock_surreal.query = AsyncMock( - return_value=[[{"source": "/path/1.txt"}, {"source": "/path/2.txt"}]] - ) - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - paths = await client.get_unprocessed_transcript_paths() - - assert len(paths) == 2 - assert "/path/1.txt" in paths - assert "/path/2.txt" in paths - - -class TestSurrealDBClientRelationships: - """Tests for relationship operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[{"id": "entity:1"}]]) - return mock - - @pytest.fixture - def sample_relationship(self): - return Relationship( - source="Entity A", target="Entity B", relationship_type="relates_to", confidence=0.85 - ) - - @pytest.mark.asyncio - async def test_create_relationship_entities_not_found(self, mock_surreal, sample_relationship): - """Test creating relationship when entities not found.""" - mock_surreal.query = AsyncMock(return_value=[[]]) # No entities found - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - with pytest.raises(ValueError, match="Source or target entity not found"): - await client.create_relationship(sample_relationship) - - -class TestSurrealDBClientErrors: - """Tests for error handling.""" - - def test_query_without_connection(self): - """Test that queries fail without connection.""" - client = SurrealDBClient( - url="ws://test:8080", namespace="ns", database="db", username="user", password="pass" - ) - - # db is None until connect() is called - assert client.db is None - - @pytest.mark.asyncio - async def test_create_entity_not_connected(self): - """Test create_entity raises error when not connected.""" - client = SurrealDBClient( - url="ws://test:8080", namespace="ns", database="db", username="user", password="pass" - ) - - entity = Entity(name="Test", type="concept", confidence=0.9, mentions=1, context=["test"]) - - with pytest.raises(RuntimeError, match="Database not connected"): - await client.create_entity(entity) +@pytest.fixture +def connected(): + client = SurrealDBClient() + client.db = AsyncMock() + client.db.query.return_value = [[]] + return client, client.db + + + + +async def test_ambiguous_entity_does_not_select_first_or_write(connected): + client, db = connected + db.query.return_value = [[{"id": "entity:a"}, {"id": "entity:b"}]] + with pytest.raises(RuntimeError, match="Ambiguous"): + await client.create_entity(Entity(source="a.txt", type="concept", name="Shared")) + db.create.assert_not_awaited() + + +async def test_cross_source_or_wrong_type_endpoints_are_refused(connected): + client, db = connected + db.query.return_value = [[{"id": "entity:wrong", "source": "other.txt", "type": "concept", "name": "Shared"}]] + edge = Relationship(transcript_source="a.txt", source="Shared", target="Target", + source_type="concept", target_type="concept", relationship_type="related_to") + with pytest.raises(ValueError, match="trusted source/type/name"): + await client.create_relationship(edge, source_id="entity:wrong", target_id="entity:target") + db.relate.assert_not_awaited() + + +async def test_transcript_update_does_not_replace_date_or_identity(connected): + client, db = connected + db.query.side_effect = [ + [[{"id": "transcript:old", "source": "a.txt", "date": datetime(2000, 1, 1)}]], + [[{"id": "transcript:old"}]], + ] + transcript = Transcript(title="Updated", date=datetime.min, source="a.txt", content="New text") + assert await client.get_or_create_transcript(transcript) == "transcript:old" + changes = db.query.await_args.args[1]["changes"] + assert changes == {"title": "Updated", "content": "New text", "metadata": {}, "processed": False} + db.create.assert_not_awaited() + + +async def test_disappeared_status_target_is_not_reported_complete(connected): + client, _ = connected + with pytest.raises(RuntimeError, match="disappeared"): + await client.update_transcript_status("transcript:gone", processed=True) + + +async def test_pending_without_source_is_not_silently_skipped(connected): + client, db = connected + db.query.return_value = [[{"source": None}]] + with pytest.raises(RuntimeError, match="no trusted source"): + await client.get_unprocessed_transcript_paths() + + +async def test_legacy_schema_requires_explicit_admission(monkeypatch): + db = AsyncMock() + db.query.side_effect = [ + [{"tables": {"entity": "SCHEMAFULL"}}], + [{"indexes": {"entity_name_type": "name,type UNIQUE"}}], + ] + monkeypatch.setattr("journal_utilities.rag.graph.surreal_client.AsyncSurreal", lambda _: db) + client = SurrealDBClient() + with pytest.raises(RuntimeError, match="--migrate-source-identity"): + await client.connect() + db.close.assert_awaited_once() + assert client.db is None + + +@pytest.mark.parametrize("entities,edges", [ + ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt", "metadata": {"source": "b.txt"}}], []), + ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}, + {"id": "entity:b", "name": "Shared", "type": "concept", "source": "a.txt"}], []), + ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}], + [{"id": "relates:x", "source": "b.txt", "in": "entity:a", "out": "entity:a", "relationship_type": "related_to"}]), +]) +async def test_explicit_migration_refuses_ambiguous_attribution(monkeypatch, entities, edges): + db = AsyncMock() + db.query.side_effect = [ + [{"tables": {"entity": "SCHEMAFULL", "relates": "RELATION"}}], + [{"indexes": {"entity_name_type": "name,type UNIQUE"}}], + [entities], [edges], + ] + monkeypatch.setattr("journal_utilities.rag.graph.surreal_client.AsyncSurreal", lambda _: db) + client = SurrealDBClient() + with pytest.raises(RuntimeError, match="[Aa]mbiguous|Conflicting"): + await client.connect(migrate_source_identity=True) + # No schema/data write was reached after the four evidence reads. + assert db.query.await_count == 4 + db.close.assert_awaited_once() + + +async def test_metadata_only_source_stays_unknown(connected): + client, db = connected + db.query.return_value = [[{"id": "entity:old", "name": "Insight 1", "type": "key_insight", + "metadata": {"source": "untrusted-claim.txt"}}]] + summary = await client._admit_migration({"entity": "SCHEMAFULL"}) + assert summary["legacy_entities_preserved"] == summary["unknown_source_entities"] == 1 + db.create.assert_not_awaited() diff --git a/tests/journal_utilities/test_app.py b/tests/journal_utilities/test_app.py index d64e90fe..c3f90be3 100644 --- a/tests/journal_utilities/test_app.py +++ b/tests/journal_utilities/test_app.py @@ -12,16 +12,51 @@ # Requires the interface extra; skip cleanly when it is not installed. pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") from fastapi.testclient import TestClient from journal_utilities.interface import chat_engine -from journal_utilities.interface.app import create_app +from journal_utilities.interface.app import ( + MAX_CHAT_BODY_BYTES, + MAX_MESSAGE_LENGTH, + MAX_SESSION_LENGTH, + create_app, +) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def ollama_transport(monkeypatch): + """Run real ChatEngine logic, with only HTTP transport replaced; never sockets.""" + state = {"requests": [], "fail": False} + client_type = httpx.AsyncClient + + def handle(request): + state["requests"].append(request) + if state["fail"]: + raise httpx.ConnectError("Connection to http://private-host:11434 failed", request=request) + if request.url.path == "/api/tags": + return httpx.Response(200, json={"models": [{"name": chat_engine.OLLAMA_MODEL}]}) + assert request.url.path == "/api/chat" + payload = json.loads(request.content) + if payload["stream"]: + return httpx.Response(200, content=( + json.dumps({"message": {"content": "Bounded answer"}, "done": False}) + "\n" + + json.dumps({"done": True}) + "\n" + ), headers={"content-type": "application/x-ndjson"}) + return httpx.Response(200, json={"message": {"content": "Bounded answer"}}) + + def client(*args, **kwargs): + kwargs["transport"] = httpx.MockTransport(handle) + return client_type(*args, **kwargs) + + monkeypatch.setattr(chat_engine.httpx, "AsyncClient", client) + return state + + @pytest.fixture def sample_data_dir(tmp_path: Path) -> Path: """Create a temporary data directory with realistic sample data.""" @@ -265,8 +300,8 @@ def test_chat_status(self, test_client: TestClient): resp = test_client.get("/api/chat/status") assert resp.status_code == 200 data = resp.json() - # If Ollama is not running, this should still succeed - assert "available" in data or "error" in data + assert data["available"] is True + assert data["current_model"] == chat_engine.OLLAMA_MODEL def test_chat_empty_message_rejected(self, test_client: TestClient): """POST /api/chat rejects empty messages.""" @@ -384,30 +419,18 @@ def test_search_no_results(self, test_client: TestClient): class TestChatEndpointsAdditional: """Additional chat endpoint tests.""" - def test_chat_with_valid_message_returns_response( - self, test_client: TestClient, monkeypatch: pytest.MonkeyPatch - ): - """POST /api/chat with a valid message returns a graceful response. - - With no Ollama reachable the endpoint returns 200 with a sanitized - generic error — it must not leak raw exception detail. The endpoint is - pointed at a dead port so a developer machine running `ollama serve` - still exercises the offline path instead of a live model's answer. - """ - monkeypatch.setattr(chat_engine, "OLLAMA_BASE_URL", "http://127.0.0.1:1") - resp = test_client.post( - "/api/chat", - json={ - "session_id": "test_session", - "message": "What is active inference?", - }, - ) + def test_chat_with_valid_message_returns_response(self, test_client: TestClient): + resp = test_client.post("/api/chat", json={"message": "What is active inference?"}) + assert resp.status_code == 200 + assert resp.json()["response"] == "Bounded answer" + + def test_transport_failure_is_sanitized(self, test_client: TestClient, ollama_transport): + ollama_transport["fail"] = True + resp = test_client.post("/api/chat", json={"message": "What is active inference?"}) assert resp.status_code == 200 - body = resp.json() - assert "response" in body - # Internal exception text (host/port/connection) must never be echoed. - assert "Connection" not in body.get("response", "") - assert "http" not in body.get("response", "").lower() + assert resp.json()["error"] is True + assert "private-host" not in resp.text + assert "http://" not in resp.text def test_chat_rejects_oversized_message(self, test_client: TestClient): """An over-long message is rejected (413), not forwarded to Ollama.""" @@ -416,3 +439,116 @@ def test_chat_rejects_oversized_message(self, test_client: TestClient): json={"session_id": "s", "message": "x" * 40_000}, ) assert resp.status_code == 413 + + +CHAT_ROUTES = ["/api/chat", "/api/chat/stream", "/api/chat/clear"] + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize("body,status", [ + (b"null", 422), (b"[]", 422), (b"3", 422), (b'"text"', 422), (b"true", 422), + (b"{broken", 400), (b"\xff", 400), + (b'{"message": null}', 422), (b'{"message": []}', 422), (b'{"message": 1}', 422), + (b'{"message": "hello", "session_id": null}', 422), + (b'{"message": "hello", "session_id": {}}', 422), + (b'{"message": "hello", "session_id": 1}', 422), + (b'{"message": "hello", "use_rag": "false"}', 422), + (b'{"message": "hello", "use_rag": 0}', 422), + (b'{"message": "hello", "use_rag": null}', 422), +]) +def test_invalid_chat_payloads_are_client_errors(test_client, ollama_transport, route, body, status): + response = test_client.post(route, content=body, headers={"content-type": "application/json"}) + assert response.status_code == status + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize("route", ["/api/chat", "/api/chat/stream"]) +def test_exact_field_bounds_allow_escaped_unicode(test_client, route): + payload = {"message": "😀" * MAX_MESSAGE_LENGTH, "session_id": "s" * MAX_SESSION_LENGTH, "use_rag": False} + response = test_client.post(route, content=json.dumps(payload), headers={"content-type": "application/json"}) + assert response.status_code == 200 + if route.endswith("/stream"): + events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: ")] + assert [event["type"] for event in events] == ["context", "token", "done"] + assert events[1]["content"] == "Bounded answer" + else: + assert response.json()["response"] == "Bounded answer" + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize("field,length", [("message", MAX_MESSAGE_LENGTH + 1), ("session_id", MAX_SESSION_LENGTH + 1)]) +def test_field_limit_plus_one_never_reaches_transport(test_client, ollama_transport, route, field, length): + payload = {"message": "hello", "session_id": "session", "use_rag": False, field: "x" * length} + assert test_client.post(route, json=payload).status_code == 413 + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize("extra,status", [(0, 200), (1, 413)]) +def test_exact_body_limit(test_client, extra, status): + message = b'{"message":"hello","use_rag":false}' + body = message + b" " * (MAX_CHAT_BODY_BYTES + extra - len(message)) + assert test_client.post("/api/chat", content=body).status_code == status + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize("declared", [None, b"1", str(MAX_CHAT_BODY_BYTES + 1).encode()]) +@pytest.mark.parametrize("single_chunk", [False, True]) +async def test_streamed_body_stops_before_full_accumulation( + sample_data_dir, ollama_transport, route, declared, single_chunk +): + app = create_app(data_dir=sample_data_dir) + chunks = ([b"x" * (MAX_CHAT_BODY_BYTES + 1), b"unread"] if single_chunk else + [b"x" * (MAX_CHAT_BODY_BYTES - 5), b"0123456789", b"unread"]) + consumed = 0 + sent = [] + headers = [(b"content-type", b"application/json")] + if declared is not None: + headers.append((b"content-length", declared)) + scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.4"}, + "http_version": "1.1", "method": "POST", "scheme": "http", "path": route, + "raw_path": route.encode(), "root_path": "", "query_string": b"", "headers": headers, + "server": ("test", 80), "client": ("test", 1234)} + + async def receive(): + nonlocal consumed + if consumed >= len(chunks) - 1: + pytest.fail("request drained beyond the byte cap") + chunk = chunks[consumed] + consumed += 1 + return {"type": "http.request", "body": chunk, "more_body": True} + + async def send(message): + sent.append(message) + + await app(scope, receive, send) + assert next(message["status"] for message in sent if message["type"] == "http.response.start") == 413 + assert consumed == (0 if declared == str(MAX_CHAT_BODY_BYTES + 1).encode() else (1 if single_chunk else 2)) + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize("url", [ + "/api/videos?offset=-1", "/api/videos?limit=0", "/api/videos?limit=-1", "/api/videos?limit=201", + "/api/videos?q=active&offset=-1", "/api/search?q=active&limit=0", + "/api/search?q=active&limit=-1", "/api/search?q=active&limit=101", +]) +def test_pagination_rejects_invalid_bounds(test_client, url): + assert test_client.get(url).status_code == 422 + + +def test_pagination_upper_bounds_remain_valid(test_client): + videos = test_client.get("/api/videos?offset=0&limit=200") + assert videos.status_code == 200 + assert videos.json()["total"] == 3 + results = test_client.get("/api/search?q=Bayesian&limit=100") + assert results.status_code == 200 + assert results.json()["results"][0]["video_id"] == "vid_bbb22222222" + + +def test_clear_removes_history_without_provider_call(test_client, ollama_transport): + assert test_client.post("/api/chat", json={"session_id": "s", "message": "first", "use_rag": False}).status_code == 200 + before = len(ollama_transport["requests"]) + assert test_client.post("/api/chat/clear", json={"session_id": "s"}).json() == {"status": "cleared"} + assert len(ollama_transport["requests"]) == before + assert test_client.post("/api/chat", json={"session_id": "s", "message": "second", "use_rag": False}).status_code == 200 + payload = json.loads(ollama_transport["requests"][-1].content) + assert [message["content"] for message in payload["messages"] if message["role"] == "user"] == ["second"] diff --git a/tests/journal_utilities/test_run.py b/tests/journal_utilities/test_run.py index 26d9b410..32cfeed6 100644 --- a/tests/journal_utilities/test_run.py +++ b/tests/journal_utilities/test_run.py @@ -214,6 +214,24 @@ def test_parser_log_level_override(self): class TestMain: + @pytest.fixture(autouse=True) + def isolated_runner_logging(self, tmp_path, monkeypatch): + import logging + + import run + + root = logging.getLogger() + handlers, level = list(root.handlers), root.level + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + try: + yield + finally: + for handler in list(root.handlers): + if handler not in handlers: + handler.close() + root.handlers[:] = handlers + root.setLevel(level) + def test_config_command_returns_zero(self, sample_config_file): rc = main(["--config", str(sample_config_file), "config"]) assert rc == 0 @@ -302,3 +320,157 @@ def test_default_shows_clickable_url(self, config, capsys, monkeypatch): assert "http://127.0.0.1:9000" in captured.out assert "Open in browser" in captured.out assert rc == 0 + + +@pytest.fixture +def export_corpus(config, tmp_path, monkeypatch): + import run + + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + transcripts = tmp_path / "data" / "output" / "transcripts" + transcripts.mkdir(parents=True) + config.set("export", "markdown", "false") + return config, transcripts, tmp_path / "data" / "export" / "plaintext" + + +def test_export_empty_directory_is_incomplete(export_corpus): + import argparse + + import run + + cfg, _, _ = export_corpus + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + + +@pytest.mark.parametrize("bad_bytes", [b" ", b"\xff\xfe"]) +def test_export_mixed_success_and_failure_is_nonzero(export_corpus, bad_bytes): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "good.txt").write_text("A valid transcript", encoding="utf-8") + (transcripts / "bad.txt").write_bytes(bad_bytes) + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + assert "A valid transcript" in (output / "good.txt").read_text(encoding="utf-8") + assert not (output / "bad.txt").exists() + + +def test_export_read_permission_failure_is_nonzero(export_corpus, monkeypatch): + import argparse + + import run + + cfg, transcripts, _ = export_corpus + blocked = transcripts / "blocked.txt" + blocked.write_text("Cannot read", encoding="utf-8") + read_text = Path.read_text + + def guarded_read(path, *args, **kwargs): + if path == blocked: + raise PermissionError("Access denied") + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", guarded_read) + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + + +def test_export_success_and_existing_outputs_remain_successful(export_corpus): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "good.txt").write_text("A valid transcript", encoding="utf-8") + assert run.cmd_export(cfg, argparse.Namespace()) == 0 + saved = (output / "good.txt").read_bytes() + assert run.cmd_export(cfg, argparse.Namespace()) == 0 + assert (output / "good.txt").read_bytes() == saved + + +@pytest.mark.parametrize("command", ["cmd_full", "cmd_default"]) +def test_required_export_failure_stops_pipeline(export_corpus, monkeypatch, command): + import argparse + + import run + + cfg, transcripts, _ = export_corpus + (transcripts / "empty.txt").write_text("", encoding="utf-8") + monkeypatch.setattr(run, "cmd_download", lambda *_: 0) + monkeypatch.setattr(run, "cmd_test", lambda *_: pytest.fail("tests started after failed export")) + monkeypatch.setattr(run, "cmd_serve", lambda *_: pytest.fail("server started after failed export")) + assert getattr(run, command)(cfg, argparse.Namespace()) == 1 + + +def test_occupied_listener_survives_serve_refusal(config, monkeypatch): + import argparse + import socket + + import run + + uvicorn = pytest.importorskip("uvicorn") + pytest.importorskip("fastapi") + monkeypatch.setattr(run.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("listener discovery attempted")) + monkeypatch.setattr(run.os, "kill", lambda *_args, **_kwargs: pytest.fail("listener termination attempted")) + monkeypatch.setattr(uvicorn, "run", lambda *_args, **_kwargs: pytest.fail("occupied server started")) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(2) + port = listener.getsockname()[1] + config.set("interface", "port", str(port)) + assert run.cmd_serve(config, argparse.Namespace()) == 1 + with socket.create_connection(("127.0.0.1", port), timeout=2) as probe: + accepted, _ = listener.accept() + with accepted: + probe.sendall(b"still alive") + assert accepted.recv(11) == b"still alive" + + +@pytest.mark.parametrize("absolute", [False, True]) +def test_serve_loads_configured_corpus(config, tmp_path, monkeypatch, absolute): + import argparse + import json + + import run + + uvicorn = pytest.importorskip("uvicorn") + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + chosen = tmp_path / "chosen" + chosen.mkdir() + (chosen / "channel_videos.json").write_text(json.dumps({"videos": [ + {"id": "chosen12345", "title": "Chosen corpus", "upload_date": "20250101"} + ]}), encoding="utf-8") + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + config.set("general", "data_dir", str(chosen) if absolute else "chosen") + config.set("interface", "port", "0") + observed = [] + + def serve_boundary(app, **_kwargs): + with TestClient(app) as client: + response = client.get("/api/videos") + observed.extend(video["id"] for video in response.json()["videos"]) + + monkeypatch.setattr(uvicorn, "run", serve_boundary) + assert run.cmd_serve(config, argparse.Namespace()) == 0 + assert observed == ["chosen12345"] + + +def test_export_writer_failure_is_nonzero(export_corpus, monkeypatch): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "blocked.txt").write_text("Readable input", encoding="utf-8") + write_text = Path.write_text + + def guarded_write(path, *args, **kwargs): + if path == output / "blocked.txt": + raise OSError("Output device unavailable") + return write_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", guarded_write) + assert run.cmd_export(cfg, argparse.Namespace()) == 1 From ce1beb617ee5f74757ff0a4ad5f86150970c9ade Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman Date: Mon, 16 Nov 2026 12:49:50 -0800 Subject: [PATCH 2/3] fix(rag): preserve dotted dates in full transcript filenames Address independent review P2 without changing transcript titles or historical-date replay semantics. Cover dotted filenames with and without a separate extension in both file and pending loaders. --- docs/rag.md | 5 +++-- src/journal_utilities/rag/main.py | 2 +- tests/journal_utilities/rag/unit/test_pipeline.py | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/rag.md b/docs/rag.md index 31fd26b8..671446c3 100644 --- a/docs/rag.md +++ b/docs/rag.md @@ -85,8 +85,9 @@ Relationship endpoints are checked against the trusted source-qualified entity identity. Missing required endpoints or failed edge writes leave the transcript pending. Retry and replay reuse persisted records rather than duplicating edges. Generic names or metadata claims never establish provenance for unknown-source -legacy records. File and database loaders use a valid filename date, otherwise -`datetime.min`; retries preserve stored historical dates. +legacy records. File and database loaders parse dates from the full filename, +including dotted dates without a separate extension; invalid or absent dates use +`datetime.min`. Titles still use the filename stem; retries preserve stored historical dates. ## Legacy schema migration diff --git a/src/journal_utilities/rag/main.py b/src/journal_utilities/rag/main.py index 11ed8206..78ec8807 100644 --- a/src/journal_utilities/rag/main.py +++ b/src/journal_utilities/rag/main.py @@ -32,7 +32,7 @@ def _date_from_filename(filename: str) -> datetime: def _load_transcript(path: Path, *, source: str) -> Transcript: - return Transcript(title=path.stem, date=_date_from_filename(path.stem), + return Transcript(title=path.stem, date=_date_from_filename(path.name), content=path.read_text(encoding="utf-8"), source=source) diff --git a/tests/journal_utilities/rag/unit/test_pipeline.py b/tests/journal_utilities/rag/unit/test_pipeline.py index 5da8d211..95cb3027 100644 --- a/tests/journal_utilities/rag/unit/test_pipeline.py +++ b/tests/journal_utilities/rag/unit/test_pipeline.py @@ -243,6 +243,8 @@ async def test_process_transcript_converts_entities_correctly( @pytest.mark.parametrize("filename, expected", [ ("talk_2025-10-30.txt", datetime(2025, 10, 30)), + ("talk_2025.10.30", datetime(2025, 10, 30)), + ("talk_2025.10.30.txt", datetime(2025, 10, 30)), ("undated.txt", datetime.min), ("talk_2025-02-30.txt", datetime.min), ]) From 9f93f9a3cf2c8ddf682613d04e451bf680b27f85 Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman Date: Mon, 16 Nov 2026 12:56:54 -0800 Subject: [PATCH 3/3] fix: satisfy journal software lint and typing gates Apply only exact source-layout repairs diagnosed by final native gate 74595. Unpack the known three date components before datetime construction for strict typing. --- .../rag/graph/surreal_client.py | 121 ++++++++++---- src/journal_utilities/rag/main.py | 54 +++++-- src/journal_utilities/rag/models/entities.py | 4 +- .../integration/test_pipeline_integration.py | 10 +- .../rag/test_source_identity.py | 111 ++++++++++--- .../rag/unit/test_entity_adapter.py | 10 +- .../rag/unit/test_pipeline.py | 84 +++++++--- .../rag/unit/test_surreal_client.py | 81 ++++++++-- tests/journal_utilities/test_app.py | 150 +++++++++++++----- tests/journal_utilities/test_run.py | 29 +++- 10 files changed, 488 insertions(+), 166 deletions(-) diff --git a/src/journal_utilities/rag/graph/surreal_client.py b/src/journal_utilities/rag/graph/surreal_client.py index 9e2fd5a5..e519ee2f 100644 --- a/src/journal_utilities/rag/graph/surreal_client.py +++ b/src/journal_utilities/rag/graph/surreal_client.py @@ -11,6 +11,7 @@ from journal_utilities.rag.models import Entity, Relationship, Transcript from journal_utilities.rag.settings import settings + def _first_statement(result: object) -> object: """The installed SDK 3.x returns one entry per statement.""" if not isinstance(result, list) or len(result) != 1: @@ -107,7 +108,9 @@ async def _initialize_schema(self, *, migrate_source_identity: bool = False) -> tables = info["tables"] indexes = {} if "entity" in tables: - entity_info = cast(Mapping[str, Any], _first_statement(await self.db.query("INFO FOR TABLE entity;"))) + entity_info = cast( + Mapping[str, Any], _first_statement(await self.db.query("INFO FOR TABLE entity;")) + ) indexes = entity_info["indexes"] legacy = ("entity" in tables or "relates" in tables) and ( "entity_source_type_name" not in indexes or "entity_name_type" in indexes @@ -119,14 +122,18 @@ async def _initialize_schema(self, *, migrate_source_identity: bool = False) -> ) if migrate_source_identity: self.migration_summary = await self._admit_migration(tables) - remove_old = "REMOVE INDEX entity_name_type ON entity;" if "entity_name_type" in indexes else "" + remove_old = ( + "REMOVE INDEX entity_name_type ON entity;" if "entity_name_type" in indexes else "" + ) # A failed index definition must not leave a partially migrated schema. await self.db.query("BEGIN TRANSACTION;" + _SCHEMA + remove_old + "COMMIT TRANSACTION;") async def _admit_migration(self, tables: Mapping[str, Any]) -> dict[str, int]: entities = list(await self.query("SELECT * FROM entity;")) if "entity" in tables else [] edges = list(await self.query("SELECT * FROM relates;")) if "relates" in tables else [] - transcripts = list(await self.query("SELECT * FROM transcript;")) if "transcript" in tables else [] + transcripts = ( + list(await self.query("SELECT * FROM transcript;")) if "transcript" in tables else [] + ) by_id = {str(row["id"]): row for row in entities} identities: set[tuple[str | None, str, str]] = set() unknown = 0 @@ -147,7 +154,9 @@ async def _admit_migration(self, tables: Mapping[str, Any]) -> dict[str, int]: for endpoint in ("in", "out"): entity = by_id.get(str(row[endpoint])) if entity is None or self._legacy_source(entity) != source: - raise RuntimeError("Ambiguous legacy relationship attribution; migration refused") + raise RuntimeError( + "Ambiguous legacy relationship attribution; migration refused" + ) key = (source, str(row["in"]), str(row["out"]), row["relationship_type"]) if key in edge_keys: raise RuntimeError("Ambiguous legacy relationship identity; migration refused") @@ -159,8 +168,12 @@ async def _admit_migration(self, tables: Mapping[str, Any]) -> dict[str, int]: if source in sources: raise RuntimeError("Ambiguous legacy transcript source; migration refused") sources.add(source) - return {"legacy_entities_preserved": len(entities), "unknown_source_entities": unknown, - "legacy_relationships_preserved": len(edges), "unknown_source_relationships": unknown_edges} + return { + "legacy_entities_preserved": len(entities), + "unknown_source_entities": unknown, + "legacy_relationships_preserved": len(edges), + "unknown_source_relationships": unknown_edges, + } @staticmethod def _legacy_source(row: Mapping[str, Any]) -> str | None: @@ -186,11 +199,14 @@ async def query( return cast(Sequence[Mapping[str, Any]], rows) async def _record(self, record_id: str) -> Mapping[str, Any] | None: - return _single(await self.query("SELECT * FROM $id;", {"id": StringRecordId(record_id)}), record_id) + return _single( + await self.query("SELECT * FROM $id;", {"id": StringRecordId(record_id)}), record_id + ) async def _merge(self, record_id: str, changes: Mapping[str, Any]) -> None: rows = await self.query( - "UPDATE $id MERGE $changes;", {"id": StringRecordId(record_id), "changes": dict(changes)} + "UPDATE $id MERGE $changes;", + {"id": StringRecordId(record_id), "changes": dict(changes)}, ) if _single(rows, record_id) is None: raise RuntimeError(f"Record disappeared during update: {record_id}") @@ -200,15 +216,20 @@ async def get_entity_by_name( ) -> Mapping[str, Any] | None: if not source.strip(): raise ValueError("A trusted transcript source is required") - return _single(await self.query( - "SELECT * FROM entity WHERE source = $source AND type = $type AND name = $name;", - {"source": source, "type": entity_type, "name": name}, - ), "source-qualified entity") + return _single( + await self.query( + "SELECT * FROM entity WHERE source = $source AND type = $type AND name = $name;", + {"source": source, "type": entity_type, "name": name}, + ), + "source-qualified entity", + ) async def create_entity(self, entity: Entity) -> str: if self.db is None: raise RuntimeError("Database not connected") - existing = await self.get_entity_by_name(entity.name, entity_type=entity.type, source=entity.source) + existing = await self.get_entity_by_name( + entity.name, entity_type=entity.type, source=entity.source + ) payload: dict[str, Any] = entity.model_dump() if existing: # MERGE, not replacement: preserve identity and original creation time. @@ -235,31 +256,50 @@ async def create_relationship( ): endpoint = await self._record(record_id) if endpoint is None or ( - endpoint.get("source"), endpoint.get("type"), endpoint.get("name") + endpoint.get("source"), + endpoint.get("type"), + endpoint.get("name"), ) != (relationship.transcript_source, entity_type, name): raise ValueError("Relationship endpoint does not match trusted source/type/name") - params = {"source": relationship.transcript_source, "in": StringRecordId(source_id), - "out": StringRecordId(target_id), "type": relationship.relationship_type} - existing = _single(await self.query( - "SELECT * FROM relates WHERE source = $source AND in = $in AND out = $out " - "AND relationship_type = $type;", params - ), "source-qualified relationship") - payload: dict[str, Any] = {"source": relationship.transcript_source, - "relationship_type": relationship.relationship_type, - "description": relationship.description, "confidence": relationship.confidence, - "context": relationship.context, "metadata": relationship.metadata} + params = { + "source": relationship.transcript_source, + "in": StringRecordId(source_id), + "out": StringRecordId(target_id), + "type": relationship.relationship_type, + } + existing = _single( + await self.query( + "SELECT * FROM relates WHERE source = $source AND in = $in AND out = $out " + "AND relationship_type = $type;", + params, + ), + "source-qualified relationship", + ) + payload: dict[str, Any] = { + "source": relationship.transcript_source, + "relationship_type": relationship.relationship_type, + "description": relationship.description, + "confidence": relationship.confidence, + "context": relationship.context, + "metadata": relationship.metadata, + } if existing: edge_id = str(existing["id"]) await self._merge(edge_id, payload) return edge_id # Deterministic record identity also prevents duplicate edges on concurrent retries. - key = json.dumps([relationship.transcript_source, source_id, target_id, - relationship.relationship_type], ensure_ascii=False, separators=(",", ":")) + key = json.dumps( + [relationship.transcript_source, source_id, target_id, relationship.relationship_type], + ensure_ascii=False, + separators=(",", ":"), + ) edge = RecordID("relates", hashlib.sha256(key.encode("utf-8")).hexdigest()) if await self._record(str(edge)) is not None: raise RuntimeError("Relationship identity collision; refusing to overwrite") payload["created_at"] = datetime.now(UTC) - row = await self.db.relate(StringRecordId(source_id), edge, StringRecordId(target_id), payload) + row = await self.db.relate( + StringRecordId(source_id), edge, StringRecordId(target_id), payload + ) if not isinstance(row, Mapping) or "id" not in row: raise RuntimeError("Relationship creation returned no record") return str(row["id"]) @@ -268,15 +308,21 @@ async def create_transcript(self, transcript: Transcript) -> str: if self.db is None: raise RuntimeError("Database not connected") payload = transcript.model_dump() - payload.update(date=_utc(transcript.date), created_at=_utc(transcript.created_at), processed=False) + payload.update( + date=_utc(transcript.date), created_at=_utc(transcript.created_at), processed=False + ) row = await self.db.create("transcript", payload) if not isinstance(row, Mapping) or "id" not in row: raise RuntimeError("Transcript creation returned no record") return str(row["id"]) async def get_transcript_by_source(self, source_path: str) -> Mapping[str, Any] | None: - return _single(await self.query("SELECT * FROM transcript WHERE source = $source;", - {"source": source_path}), "transcript source") + return _single( + await self.query( + "SELECT * FROM transcript WHERE source = $source;", {"source": source_path} + ), + "transcript source", + ) async def get_unprocessed_transcript_paths(self) -> Sequence[str]: rows = await self.query("SELECT source FROM transcript WHERE processed = false;") @@ -284,7 +330,9 @@ async def get_unprocessed_transcript_paths(self) -> Sequence[str]: for row in rows: source = row.get("source") if not isinstance(source, str) or not source.strip(): - raise RuntimeError("Pending transcript has no trusted source path; explicit review required") + raise RuntimeError( + "Pending transcript has no trusted source path; explicit review required" + ) paths.append(source) return paths @@ -296,7 +344,14 @@ async def get_or_create_transcript(self, transcript: Transcript) -> str: if existing: transcript_id = str(existing["id"]) # Do not rewrite historical date/source/created_at on a retry. - await self._merge(transcript_id, {"title": transcript.title, "content": transcript.content, - "metadata": transcript.metadata, "processed": False}) + await self._merge( + transcript_id, + { + "title": transcript.title, + "content": transcript.content, + "metadata": transcript.metadata, + "processed": False, + }, + ) return transcript_id return await self.create_transcript(transcript) diff --git a/src/journal_utilities/rag/main.py b/src/journal_utilities/rag/main.py index 78ec8807..1dba637a 100644 --- a/src/journal_utilities/rag/main.py +++ b/src/journal_utilities/rag/main.py @@ -25,15 +25,20 @@ def _date_from_filename(filename: str) -> datetime: match = re.search(r"(\d{4})[-_/. ](\d{1,2})[-_/. ](\d{1,2})", filename) if match: try: - return datetime(*(int(part) for part in match.groups())) + year, month, day = (int(part) for part in match.groups()) + return datetime(year, month, day) except ValueError: pass return datetime.min def _load_transcript(path: Path, *, source: str) -> Transcript: - return Transcript(title=path.stem, date=_date_from_filename(path.name), - content=path.read_text(encoding="utf-8"), source=source) + return Transcript( + title=path.stem, + date=_date_from_filename(path.name), + content=path.read_text(encoding="utf-8"), + source=source, + ) class JournalRAGPipeline: @@ -61,8 +66,10 @@ async def process_transcript(self, transcript: Transcript) -> Mapping[str, int]: relationships = EntityAdapter.extract_relationships(core, source=transcript.source) identities = {(entity.type, entity.name) for entity in entities} for relationship in relationships: - for identity in ((relationship.source_type, relationship.source), - (relationship.target_type, relationship.target)): + for identity in ( + (relationship.source_type, relationship.source), + (relationship.target_type, relationship.target), + ): if identity not in identities: raise ValueError(f"Required relationship endpoint missing: {identity!r}") entity_ids = {} @@ -83,8 +90,13 @@ async def process_transcript_file(self, file_path: Path) -> Mapping[str, int]: async def process_from_database(self, base_path: Path | None = None) -> Mapping[str, int]: paths = await self.db_client.get_unprocessed_transcript_paths() - totals = {"total": len(paths), "processed": 0, "failed": 0, - "total_entities": 0, "total_relationships": 0} + totals = { + "total": len(paths), + "processed": 0, + "failed": 0, + "total_entities": 0, + "total_relationships": 0, + } for source in paths: path = base_path / source if base_path is not None else Path(source) try: @@ -94,17 +106,26 @@ async def process_from_database(self, base_path: Path | None = None) -> Mapping[ totals["total_relationships"] += stats["relationships"] except Exception as exc: totals["failed"] += 1 - logger.error("Transcript incomplete; remains pending", source=source, error=str(exc)) + logger.error( + "Transcript incomplete; remains pending", source=source, error=str(exc) + ) return totals def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) selection = parser.add_mutually_exclusive_group(required=True) - selection.add_argument("--file", nargs="+", type=Path, dest="files", help="Extract these transcript files") - selection.add_argument("--pending", action="store_true", help="Extract pending database source paths") - selection.add_argument("--migrate-source-identity", action="store_true", - help="Explicit schema migration; quiescent database and backup required; no attribution backfill") + selection.add_argument( + "--file", nargs="+", type=Path, dest="files", help="Extract these transcript files" + ) + selection.add_argument( + "--pending", action="store_true", help="Extract pending database source paths" + ) + selection.add_argument( + "--migrate-source-identity", + action="store_true", + help="Explicit schema migration; quiescent database and backup required; no attribution backfill", + ) parser.add_argument("--base-path", type=Path, help="Resolve pending relative source paths here") return parser @@ -128,8 +149,13 @@ async def _run(args: argparse.Namespace) -> int: if args.pending: stats = dict(await pipeline.process_from_database(base_path=args.base_path)) else: - stats = {"total": len(args.files), "processed": 0, "failed": 0, - "total_entities": 0, "total_relationships": 0} + stats = { + "total": len(args.files), + "processed": 0, + "failed": 0, + "total_entities": 0, + "total_relationships": 0, + } for path in args.files: try: result = await pipeline.process_transcript_file(path) diff --git a/src/journal_utilities/rag/models/entities.py b/src/journal_utilities/rag/models/entities.py index 573035d6..b95f6603 100644 --- a/src/journal_utilities/rag/models/entities.py +++ b/src/journal_utilities/rag/models/entities.py @@ -281,7 +281,9 @@ class Entity(BaseModel): class Relationship(BaseModel): """Simple generic relationship for database storage.""" - transcript_source: str = Field(..., min_length=1, pattern=r"\S", description="Trusted transcript source") + transcript_source: str = Field( + ..., min_length=1, pattern=r"\S", description="Trusted transcript source" + ) source_type: str = Field(..., description="Source entity type") target_type: str = Field(..., description="Target entity type") source: str = Field(..., description="Source entity name") diff --git a/tests/journal_utilities/rag/integration/test_pipeline_integration.py b/tests/journal_utilities/rag/integration/test_pipeline_integration.py index 1e1e787d..224c6ca3 100644 --- a/tests/journal_utilities/rag/integration/test_pipeline_integration.py +++ b/tests/journal_utilities/rag/integration/test_pipeline_integration.py @@ -91,10 +91,12 @@ async def test_process_transcript_with_real_extractor( # Verify database interactions occurred mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_has_awaits([ - call("transcript:test-001", processed=False), - call("transcript:test-001", processed=True), - ]) + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] + ) # Verify stats from real extraction assert "entities" in stats diff --git a/tests/journal_utilities/rag/test_source_identity.py b/tests/journal_utilities/rag/test_source_identity.py index 81f02c64..f124d20d 100644 --- a/tests/journal_utilities/rag/test_source_identity.py +++ b/tests/journal_utilities/rag/test_source_identity.py @@ -20,15 +20,25 @@ from journal_utilities.rag.graph.surreal_client import SurrealDBClient from journal_utilities.rag.main import JournalRAGPipeline -from journal_utilities.rag.models import Concept, ConceptCategory, CoreEntities, Relationship, Transcript +from journal_utilities.rag.models import ( + Concept, + ConceptCategory, + CoreEntities, + Relationship, + Transcript, +) @pytest.fixture async def disposable_database(): prefix = "JOURNAL_TEST_SURREALDB_" required = ("URL", "NAMESPACE", "USERNAME", "PASSWORD") - if os.environ.get(prefix + "ALLOW_DISPOSABLE") != "1" or not all(os.environ.get(prefix + name) for name in required): - pytest.skip("Explicit disposable existing SurrealDB service credentials required; no DB proof") + if os.environ.get(prefix + "ALLOW_DISPOSABLE") != "1" or not all( + os.environ.get(prefix + name) for name in required + ): + pytest.skip( + "Explicit disposable existing SurrealDB service credentials required; no DB proof" + ) options = {name.lower(): os.environ[prefix + name] for name in required} options["database"] = "journal_group07_test_" + uuid4().hex raw = AsyncSurreal(options["url"]) @@ -65,14 +75,33 @@ async def test_legacy_migration_source_identity_retry_and_dates(disposable_datab DEFINE FIELD metadata ON relates TYPE object; DEFINE FIELD created_at ON relates TYPE datetime; """) - payload = {"description": "Legacy", "confidence": 1.0, "mentions": 1, "context": [], - "metadata": {}, "created_at": datetime(2000, 1, 1, tzinfo=UTC)} - await raw.create(RecordID("entity", "legacy_insight"), dict(payload, name="Insight 1", type="key_insight")) - await raw.create(RecordID("entity", "legacy_concept"), dict(payload, name="Shared", type="concept")) - await raw.relate(RecordID("entity", "legacy_concept"), RecordID("relates", "legacy"), - RecordID("entity", "legacy_insight"), - {"relationship_type": "related_to", "description": "Legacy", "confidence": 1.0, - "context": "", "metadata": {}, "created_at": datetime(2000, 1, 1, tzinfo=UTC)}) + payload = { + "description": "Legacy", + "confidence": 1.0, + "mentions": 1, + "context": [], + "metadata": {}, + "created_at": datetime(2000, 1, 1, tzinfo=UTC), + } + await raw.create( + RecordID("entity", "legacy_insight"), dict(payload, name="Insight 1", type="key_insight") + ) + await raw.create( + RecordID("entity", "legacy_concept"), dict(payload, name="Shared", type="concept") + ) + await raw.relate( + RecordID("entity", "legacy_concept"), + RecordID("relates", "legacy"), + RecordID("entity", "legacy_insight"), + { + "relationship_type": "related_to", + "description": "Legacy", + "confidence": 1.0, + "context": "", + "metadata": {}, + "created_at": datetime(2000, 1, 1, tzinfo=UTC), + }, + ) legacy_entities = (await raw.query("SELECT * FROM entity ORDER BY id;"))[0] legacy_edges = (await raw.query("SELECT * FROM relates ORDER BY id;"))[0] refused = SurrealDBClient(**options) @@ -88,10 +117,26 @@ async def test_legacy_migration_source_identity_retry_and_dates(disposable_datab info = (await raw.query("INFO FOR TABLE entity;"))[0] assert "entity_name_type" not in info["indexes"] assert "entity_source_type_name" in info["indexes"] - core = CoreEntities(concepts=[ - Concept(name="Shared", definition="A", category=ConceptCategory.THEORETICAL, related_concepts=["Target"]), - Concept(name="Target", definition="B", category=ConceptCategory.THEORETICAL, related_concepts=["Shared"]), - ], researchers=[], citations=[], technical_terms=[], key_insights=["An insight"]) + core = CoreEntities( + concepts=[ + Concept( + name="Shared", + definition="A", + category=ConceptCategory.THEORETICAL, + related_concepts=["Target"], + ), + Concept( + name="Target", + definition="B", + category=ConceptCategory.THEORETICAL, + related_concepts=["Shared"], + ), + ], + researchers=[], + citations=[], + technical_terms=[], + key_insights=["An insight"], + ) pipeline = JournalRAGPipeline() pipeline.db_client = client pipeline.extractor = MagicMock() @@ -131,20 +176,42 @@ def fail_second_edge(*args, **kwargs): by_id = {str(row["id"]): row for row in entities} for edge in edges: if edge.get("source"): - assert by_id[str(edge["in"])]["source"] == edge["source"] == by_id[str(edge["out"])]["source"] - assert sorted((row for row in entities if not row.get("source")), key=lambda row: str(row["id"])) == legacy_entities + assert ( + by_id[str(edge["in"])]["source"] + == edge["source"] + == by_id[str(edge["out"])]["source"] + ) + assert ( + sorted( + (row for row in entities if not row.get("source")), key=lambda row: str(row["id"]) + ) + == legacy_entities + ) assert [row for row in edges if not row.get("source")] == legacy_edges - assert {row["source"] for row in entities if row["name"] == "Insight 1" and row.get("source")} == {"a.txt", "b.txt"} + assert { + row["source"] for row in entities if row["name"] == "Insight 1" and row.get("source") + } == {"a.txt", "b.txt"} assert (await client.get_transcript_by_source("a.txt"))["processed"] is True assert (await client.get_transcript_by_source("b.txt"))["processed"] is True a_entity = await client.get_entity_by_name("Shared", entity_type="concept", source="a.txt") b_entity = await client.get_entity_by_name("Target", entity_type="concept", source="b.txt") - edge = Relationship(transcript_source="a.txt", source="Shared", source_type="concept", - target="Target", target_type="concept", relationship_type="related_to") + edge = Relationship( + transcript_source="a.txt", + source="Shared", + source_type="concept", + target="Target", + target_type="concept", + relationship_type="related_to", + ) with pytest.raises(ValueError, match="trusted source/type/name"): - await client.create_relationship(edge, source_id=str(a_entity["id"]), target_id=str(b_entity["id"])) + await client.create_relationship( + edge, source_id=str(a_entity["id"]), target_id=str(b_entity["id"]) + ) # Explicit migration must refuse contradictory claims without changing data. - await raw.query("UPDATE $id MERGE { metadata: { source: 'contradictory.txt' } };", {"id": a_entity["id"]}) + await raw.query( + "UPDATE $id MERGE { metadata: { source: 'contradictory.txt' } };", + {"id": a_entity["id"]}, + ) snapshot = await client.query("SELECT * FROM entity ORDER BY id;") ambiguous = SurrealDBClient(**options) with pytest.raises(RuntimeError, match="Conflicting legacy source claims"): diff --git a/tests/journal_utilities/rag/unit/test_entity_adapter.py b/tests/journal_utilities/rag/unit/test_entity_adapter.py index dd61e284..a9dbdae2 100644 --- a/tests/journal_utilities/rag/unit/test_entity_adapter.py +++ b/tests/journal_utilities/rag/unit/test_entity_adapter.py @@ -254,9 +254,11 @@ def test_caller_source_overrides_extraction_metadata(sample_core_entities): from journal_utilities.rag.models import ExtractedEntities extracted = ExtractedEntities(core=sample_core_entities, metadata={"source": "untrusted"}) - entities, relationships = EntityAdapter.convert_extracted_entities(extracted, source="trusted.txt") + entities, relationships = EntityAdapter.convert_extracted_entities( + extracted, source="trusted.txt" + ) assert {entity.source for entity in entities} == {"trusted.txt"} assert {relationship.transcript_source for relationship in relationships} == {"trusted.txt"} - assert {(relationship.source_type, relationship.target_type) for relationship in relationships} == { - ("researcher", "citation") - } + assert { + (relationship.source_type, relationship.target_type) for relationship in relationships + } == {("researcher", "citation")} diff --git a/tests/journal_utilities/rag/unit/test_pipeline.py b/tests/journal_utilities/rag/unit/test_pipeline.py index 95cb3027..06e586d5 100644 --- a/tests/journal_utilities/rag/unit/test_pipeline.py +++ b/tests/journal_utilities/rag/unit/test_pipeline.py @@ -43,7 +43,9 @@ def mock_db_client(): """Mock database client.""" mock = AsyncMock() mock.get_or_create_transcript.return_value = "transcript:test-001" - mock.create_entity.side_effect = lambda entity: f"entity:{entity.source}-{entity.type}-{entity.name}" + mock.create_entity.side_effect = lambda entity: ( + f"entity:{entity.source}-{entity.type}-{entity.name}" + ) mock.create_relationship.return_value = "relates:test-001" mock.update_transcript_status.return_value = None return mock @@ -83,10 +85,12 @@ async def test_process_transcript_with_file( # Verify database interactions mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_has_awaits([ - call("transcript:test-001", processed=False), - call("transcript:test-001", processed=True), - ]) + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] + ) # Verify extractor was called mock_extractor.extract_core_entities.assert_called_once_with(sample_transcript.content) @@ -175,10 +179,12 @@ async def test_process_transcript_updates_transcript_status( await pipeline.process_transcript(sample_transcript) # Verify transcript was marked as processed - mock_db_client.update_transcript_status.assert_has_awaits([ - call("transcript:test-001", processed=False), - call("transcript:test-001", processed=True), - ]) + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] + ) @pytest.mark.asyncio @@ -241,13 +247,16 @@ async def test_process_transcript_converts_entities_correctly( assert "Karl Friston" in entity_names -@pytest.mark.parametrize("filename, expected", [ - ("talk_2025-10-30.txt", datetime(2025, 10, 30)), - ("talk_2025.10.30", datetime(2025, 10, 30)), - ("talk_2025.10.30.txt", datetime(2025, 10, 30)), - ("undated.txt", datetime.min), - ("talk_2025-02-30.txt", datetime.min), -]) +@pytest.mark.parametrize( + "filename, expected", + [ + ("talk_2025-10-30.txt", datetime(2025, 10, 30)), + ("talk_2025.10.30", datetime(2025, 10, 30)), + ("talk_2025.10.30.txt", datetime(2025, 10, 30)), + ("undated.txt", datetime.min), + ("talk_2025-02-30.txt", datetime.min), + ], +) async def test_file_and_pending_loaders_share_date_and_source( tmp_path, filename, expected, mock_db_client, mock_extractor ): @@ -285,14 +294,28 @@ async def test_required_missing_endpoint_is_not_discarded( async def test_source_and_type_choose_returned_endpoint_ids( sample_transcript, mock_db_client, mock_extractor ): - from journal_utilities.rag.models import Citation, CitationType, Concept, ConceptCategory, Researcher, ResearcherRole + from journal_utilities.rag.models import ( + Citation, + CitationType, + Concept, + ConceptCategory, + Researcher, + ResearcherRole, + ) mock_extractor.extract_core_entities.return_value = CoreEntities( - concepts=[Concept(name="Same", definition="Concept", category=ConceptCategory.THEORETICAL, - related_concepts=["Same"])], + concepts=[ + Concept( + name="Same", + definition="Concept", + category=ConceptCategory.THEORETICAL, + related_concepts=["Same"], + ) + ], researchers=[Researcher(name="Same", role=ResearcherRole.SPEAKER)], citations=[Citation(title="Same", type=CitationType.PAPER, authors=["Same"])], - technical_terms=[], key_insights=["One"], + technical_terms=[], + key_insights=["One"], ) pipeline = JournalRAGPipeline() pipeline.db_client = mock_db_client @@ -307,7 +330,9 @@ async def test_source_and_type_choose_returned_endpoint_ids( "source_id": f"entity:{edge.transcript_source}-{edge.source_type}-{edge.source}", "target_id": f"entity:{edge.transcript_source}-{edge.target_type}-{edge.target}", } - assert {invocation.args[0].source for invocation in mock_db_client.create_entity.await_args_list} == {"a.txt", "b.txt"} + assert { + invocation.args[0].source for invocation in mock_db_client.create_entity.await_args_list + } == {"a.txt", "b.txt"} @pytest.fixture @@ -343,7 +368,9 @@ def test_cli_pending_edge_failure_returns_failure_and_disconnects(tmp_path, cli_ db.create_relationship.side_effect = RuntimeError("edge write failed") assert main(["--pending", "--base-path", str(tmp_path)]) == 1 extractor.extract_core_entities.assert_called_once_with("Pending") - assert db.update_transcript_status.await_args_list == [call("transcript:test-001", processed=False)] + assert db.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] db.disconnect.assert_awaited_once() @@ -372,13 +399,20 @@ def test_cli_explicit_migration_does_not_extract(cli_boundaries): db.disconnect.assert_awaited_once() -async def test_no_required_relationships_can_complete(sample_transcript, mock_db_client, mock_extractor): +async def test_no_required_relationships_can_complete( + sample_transcript, mock_db_client, mock_extractor +): mock_extractor.extract_core_entities.return_value = CoreEntities( concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=["An insight"] ) pipeline = JournalRAGPipeline() pipeline.db_client = mock_db_client pipeline.extractor = mock_extractor - assert await pipeline.process_transcript(sample_transcript) == {"entities": 1, "relationships": 0} + assert await pipeline.process_transcript(sample_transcript) == { + "entities": 1, + "relationships": 0, + } mock_db_client.create_relationship.assert_not_awaited() - assert mock_db_client.update_transcript_status.await_args == call("transcript:test-001", processed=True) + assert mock_db_client.update_transcript_status.await_args == call( + "transcript:test-001", processed=True + ) diff --git a/tests/journal_utilities/rag/unit/test_surreal_client.py b/tests/journal_utilities/rag/unit/test_surreal_client.py index ef9e60ef..85ae57ee 100644 --- a/tests/journal_utilities/rag/unit/test_surreal_client.py +++ b/tests/journal_utilities/rag/unit/test_surreal_client.py @@ -20,8 +20,6 @@ def connected(): return client, client.db - - async def test_ambiguous_entity_does_not_select_first_or_write(connected): client, db = connected db.query.return_value = [[{"id": "entity:a"}, {"id": "entity:b"}]] @@ -32,9 +30,17 @@ async def test_ambiguous_entity_does_not_select_first_or_write(connected): async def test_cross_source_or_wrong_type_endpoints_are_refused(connected): client, db = connected - db.query.return_value = [[{"id": "entity:wrong", "source": "other.txt", "type": "concept", "name": "Shared"}]] - edge = Relationship(transcript_source="a.txt", source="Shared", target="Target", - source_type="concept", target_type="concept", relationship_type="related_to") + db.query.return_value = [ + [{"id": "entity:wrong", "source": "other.txt", "type": "concept", "name": "Shared"}] + ] + edge = Relationship( + transcript_source="a.txt", + source="Shared", + target="Target", + source_type="concept", + target_type="concept", + relationship_type="related_to", + ) with pytest.raises(ValueError, match="trusted source/type/name"): await client.create_relationship(edge, source_id="entity:wrong", target_id="entity:target") db.relate.assert_not_awaited() @@ -49,7 +55,12 @@ async def test_transcript_update_does_not_replace_date_or_identity(connected): transcript = Transcript(title="Updated", date=datetime.min, source="a.txt", content="New text") assert await client.get_or_create_transcript(transcript) == "transcript:old" changes = db.query.await_args.args[1]["changes"] - assert changes == {"title": "Updated", "content": "New text", "metadata": {}, "processed": False} + assert changes == { + "title": "Updated", + "content": "New text", + "metadata": {}, + "processed": False, + } db.create.assert_not_awaited() @@ -80,19 +91,49 @@ async def test_legacy_schema_requires_explicit_admission(monkeypatch): assert client.db is None -@pytest.mark.parametrize("entities,edges", [ - ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt", "metadata": {"source": "b.txt"}}], []), - ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}, - {"id": "entity:b", "name": "Shared", "type": "concept", "source": "a.txt"}], []), - ([{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}], - [{"id": "relates:x", "source": "b.txt", "in": "entity:a", "out": "entity:a", "relationship_type": "related_to"}]), -]) +@pytest.mark.parametrize( + "entities,edges", + [ + ( + [ + { + "id": "entity:a", + "name": "Shared", + "type": "concept", + "source": "a.txt", + "metadata": {"source": "b.txt"}, + } + ], + [], + ), + ( + [ + {"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}, + {"id": "entity:b", "name": "Shared", "type": "concept", "source": "a.txt"}, + ], + [], + ), + ( + [{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}], + [ + { + "id": "relates:x", + "source": "b.txt", + "in": "entity:a", + "out": "entity:a", + "relationship_type": "related_to", + } + ], + ), + ], +) async def test_explicit_migration_refuses_ambiguous_attribution(monkeypatch, entities, edges): db = AsyncMock() db.query.side_effect = [ [{"tables": {"entity": "SCHEMAFULL", "relates": "RELATION"}}], [{"indexes": {"entity_name_type": "name,type UNIQUE"}}], - [entities], [edges], + [entities], + [edges], ] monkeypatch.setattr("journal_utilities.rag.graph.surreal_client.AsyncSurreal", lambda _: db) client = SurrealDBClient() @@ -105,8 +146,16 @@ async def test_explicit_migration_refuses_ambiguous_attribution(monkeypatch, ent async def test_metadata_only_source_stays_unknown(connected): client, db = connected - db.query.return_value = [[{"id": "entity:old", "name": "Insight 1", "type": "key_insight", - "metadata": {"source": "untrusted-claim.txt"}}]] + db.query.return_value = [ + [ + { + "id": "entity:old", + "name": "Insight 1", + "type": "key_insight", + "metadata": {"source": "untrusted-claim.txt"}, + } + ] + ] summary = await client._admit_migration({"entity": "SCHEMAFULL"}) assert summary["legacy_entities_preserved"] == summary["unknown_source_entities"] == 1 db.create.assert_not_awaited() diff --git a/tests/journal_utilities/test_app.py b/tests/journal_utilities/test_app.py index c3f90be3..1d47490f 100644 --- a/tests/journal_utilities/test_app.py +++ b/tests/journal_utilities/test_app.py @@ -37,16 +37,24 @@ def ollama_transport(monkeypatch): def handle(request): state["requests"].append(request) if state["fail"]: - raise httpx.ConnectError("Connection to http://private-host:11434 failed", request=request) + raise httpx.ConnectError( + "Connection to http://private-host:11434 failed", request=request + ) if request.url.path == "/api/tags": return httpx.Response(200, json={"models": [{"name": chat_engine.OLLAMA_MODEL}]}) assert request.url.path == "/api/chat" payload = json.loads(request.content) if payload["stream"]: - return httpx.Response(200, content=( - json.dumps({"message": {"content": "Bounded answer"}, "done": False}) + "\n" + - json.dumps({"done": True}) + "\n" - ), headers={"content-type": "application/x-ndjson"}) + return httpx.Response( + 200, + content=( + json.dumps({"message": {"content": "Bounded answer"}, "done": False}) + + "\n" + + json.dumps({"done": True}) + + "\n" + ), + headers={"content-type": "application/x-ndjson"}, + ) return httpx.Response(200, json={"message": {"content": "Bounded answer"}}) def client(*args, **kwargs): @@ -445,18 +453,30 @@ def test_chat_rejects_oversized_message(self, test_client: TestClient): @pytest.mark.parametrize("route", CHAT_ROUTES) -@pytest.mark.parametrize("body,status", [ - (b"null", 422), (b"[]", 422), (b"3", 422), (b'"text"', 422), (b"true", 422), - (b"{broken", 400), (b"\xff", 400), - (b'{"message": null}', 422), (b'{"message": []}', 422), (b'{"message": 1}', 422), - (b'{"message": "hello", "session_id": null}', 422), - (b'{"message": "hello", "session_id": {}}', 422), - (b'{"message": "hello", "session_id": 1}', 422), - (b'{"message": "hello", "use_rag": "false"}', 422), - (b'{"message": "hello", "use_rag": 0}', 422), - (b'{"message": "hello", "use_rag": null}', 422), -]) -def test_invalid_chat_payloads_are_client_errors(test_client, ollama_transport, route, body, status): +@pytest.mark.parametrize( + "body,status", + [ + (b"null", 422), + (b"[]", 422), + (b"3", 422), + (b'"text"', 422), + (b"true", 422), + (b"{broken", 400), + (b"\xff", 400), + (b'{"message": null}', 422), + (b'{"message": []}', 422), + (b'{"message": 1}', 422), + (b'{"message": "hello", "session_id": null}', 422), + (b'{"message": "hello", "session_id": {}}', 422), + (b'{"message": "hello", "session_id": 1}', 422), + (b'{"message": "hello", "use_rag": "false"}', 422), + (b'{"message": "hello", "use_rag": 0}', 422), + (b'{"message": "hello", "use_rag": null}', 422), + ], +) +def test_invalid_chat_payloads_are_client_errors( + test_client, ollama_transport, route, body, status +): response = test_client.post(route, content=body, headers={"content-type": "application/json"}) assert response.status_code == status assert ollama_transport["requests"] == [] @@ -464,11 +484,19 @@ def test_invalid_chat_payloads_are_client_errors(test_client, ollama_transport, @pytest.mark.parametrize("route", ["/api/chat", "/api/chat/stream"]) def test_exact_field_bounds_allow_escaped_unicode(test_client, route): - payload = {"message": "😀" * MAX_MESSAGE_LENGTH, "session_id": "s" * MAX_SESSION_LENGTH, "use_rag": False} - response = test_client.post(route, content=json.dumps(payload), headers={"content-type": "application/json"}) + payload = { + "message": "😀" * MAX_MESSAGE_LENGTH, + "session_id": "s" * MAX_SESSION_LENGTH, + "use_rag": False, + } + response = test_client.post( + route, content=json.dumps(payload), headers={"content-type": "application/json"} + ) assert response.status_code == 200 if route.endswith("/stream"): - events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: ")] + events = [ + json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: ") + ] assert [event["type"] for event in events] == ["context", "token", "done"] assert events[1]["content"] == "Bounded answer" else: @@ -476,8 +504,12 @@ def test_exact_field_bounds_allow_escaped_unicode(test_client, route): @pytest.mark.parametrize("route", CHAT_ROUTES) -@pytest.mark.parametrize("field,length", [("message", MAX_MESSAGE_LENGTH + 1), ("session_id", MAX_SESSION_LENGTH + 1)]) -def test_field_limit_plus_one_never_reaches_transport(test_client, ollama_transport, route, field, length): +@pytest.mark.parametrize( + "field,length", [("message", MAX_MESSAGE_LENGTH + 1), ("session_id", MAX_SESSION_LENGTH + 1)] +) +def test_field_limit_plus_one_never_reaches_transport( + test_client, ollama_transport, route, field, length +): payload = {"message": "hello", "session_id": "session", "use_rag": False, field: "x" * length} assert test_client.post(route, json=payload).status_code == 413 assert ollama_transport["requests"] == [] @@ -497,17 +529,30 @@ async def test_streamed_body_stops_before_full_accumulation( sample_data_dir, ollama_transport, route, declared, single_chunk ): app = create_app(data_dir=sample_data_dir) - chunks = ([b"x" * (MAX_CHAT_BODY_BYTES + 1), b"unread"] if single_chunk else - [b"x" * (MAX_CHAT_BODY_BYTES - 5), b"0123456789", b"unread"]) + chunks = ( + [b"x" * (MAX_CHAT_BODY_BYTES + 1), b"unread"] + if single_chunk + else [b"x" * (MAX_CHAT_BODY_BYTES - 5), b"0123456789", b"unread"] + ) consumed = 0 sent = [] headers = [(b"content-type", b"application/json")] if declared is not None: headers.append((b"content-length", declared)) - scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.4"}, - "http_version": "1.1", "method": "POST", "scheme": "http", "path": route, - "raw_path": route.encode(), "root_path": "", "query_string": b"", "headers": headers, - "server": ("test", 80), "client": ("test", 1234)} + scope = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.4"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": route, + "raw_path": route.encode(), + "root_path": "", + "query_string": b"", + "headers": headers, + "server": ("test", 80), + "client": ("test", 1234), + } async def receive(): nonlocal consumed @@ -521,16 +566,29 @@ async def send(message): sent.append(message) await app(scope, receive, send) - assert next(message["status"] for message in sent if message["type"] == "http.response.start") == 413 - assert consumed == (0 if declared == str(MAX_CHAT_BODY_BYTES + 1).encode() else (1 if single_chunk else 2)) + assert ( + next(message["status"] for message in sent if message["type"] == "http.response.start") + == 413 + ) + assert consumed == ( + 0 if declared == str(MAX_CHAT_BODY_BYTES + 1).encode() else (1 if single_chunk else 2) + ) assert ollama_transport["requests"] == [] -@pytest.mark.parametrize("url", [ - "/api/videos?offset=-1", "/api/videos?limit=0", "/api/videos?limit=-1", "/api/videos?limit=201", - "/api/videos?q=active&offset=-1", "/api/search?q=active&limit=0", - "/api/search?q=active&limit=-1", "/api/search?q=active&limit=101", -]) +@pytest.mark.parametrize( + "url", + [ + "/api/videos?offset=-1", + "/api/videos?limit=0", + "/api/videos?limit=-1", + "/api/videos?limit=201", + "/api/videos?q=active&offset=-1", + "/api/search?q=active&limit=0", + "/api/search?q=active&limit=-1", + "/api/search?q=active&limit=101", + ], +) def test_pagination_rejects_invalid_bounds(test_client, url): assert test_client.get(url).status_code == 422 @@ -545,10 +603,24 @@ def test_pagination_upper_bounds_remain_valid(test_client): def test_clear_removes_history_without_provider_call(test_client, ollama_transport): - assert test_client.post("/api/chat", json={"session_id": "s", "message": "first", "use_rag": False}).status_code == 200 + assert ( + test_client.post( + "/api/chat", json={"session_id": "s", "message": "first", "use_rag": False} + ).status_code + == 200 + ) before = len(ollama_transport["requests"]) - assert test_client.post("/api/chat/clear", json={"session_id": "s"}).json() == {"status": "cleared"} + assert test_client.post("/api/chat/clear", json={"session_id": "s"}).json() == { + "status": "cleared" + } assert len(ollama_transport["requests"]) == before - assert test_client.post("/api/chat", json={"session_id": "s", "message": "second", "use_rag": False}).status_code == 200 + assert ( + test_client.post( + "/api/chat", json={"session_id": "s", "message": "second", "use_rag": False} + ).status_code + == 200 + ) payload = json.loads(ollama_transport["requests"][-1].content) - assert [message["content"] for message in payload["messages"] if message["role"] == "user"] == ["second"] + assert [message["content"] for message in payload["messages"] if message["role"] == "user"] == [ + "second" + ] diff --git a/tests/journal_utilities/test_run.py b/tests/journal_utilities/test_run.py index 32cfeed6..7e186bea 100644 --- a/tests/journal_utilities/test_run.py +++ b/tests/journal_utilities/test_run.py @@ -397,8 +397,12 @@ def test_required_export_failure_stops_pipeline(export_corpus, monkeypatch, comm cfg, transcripts, _ = export_corpus (transcripts / "empty.txt").write_text("", encoding="utf-8") monkeypatch.setattr(run, "cmd_download", lambda *_: 0) - monkeypatch.setattr(run, "cmd_test", lambda *_: pytest.fail("tests started after failed export")) - monkeypatch.setattr(run, "cmd_serve", lambda *_: pytest.fail("server started after failed export")) + monkeypatch.setattr( + run, "cmd_test", lambda *_: pytest.fail("tests started after failed export") + ) + monkeypatch.setattr( + run, "cmd_serve", lambda *_: pytest.fail("server started after failed export") + ) assert getattr(run, command)(cfg, argparse.Namespace()) == 1 @@ -410,9 +414,15 @@ def test_occupied_listener_survives_serve_refusal(config, monkeypatch): uvicorn = pytest.importorskip("uvicorn") pytest.importorskip("fastapi") - monkeypatch.setattr(run.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("listener discovery attempted")) - monkeypatch.setattr(run.os, "kill", lambda *_args, **_kwargs: pytest.fail("listener termination attempted")) - monkeypatch.setattr(uvicorn, "run", lambda *_args, **_kwargs: pytest.fail("occupied server started")) + monkeypatch.setattr( + run.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("listener discovery attempted") + ) + monkeypatch.setattr( + run.os, "kill", lambda *_args, **_kwargs: pytest.fail("listener termination attempted") + ) + monkeypatch.setattr( + uvicorn, "run", lambda *_args, **_kwargs: pytest.fail("occupied server started") + ) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: listener.bind(("127.0.0.1", 0)) listener.listen(1) @@ -440,9 +450,12 @@ def test_serve_loads_configured_corpus(config, tmp_path, monkeypatch, absolute): chosen = tmp_path / "chosen" chosen.mkdir() - (chosen / "channel_videos.json").write_text(json.dumps({"videos": [ - {"id": "chosen12345", "title": "Chosen corpus", "upload_date": "20250101"} - ]}), encoding="utf-8") + (chosen / "channel_videos.json").write_text( + json.dumps( + {"videos": [{"id": "chosen12345", "title": "Chosen corpus", "upload_date": "20250101"}]} + ), + encoding="utf-8", + ) monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) config.set("general", "data_dir", str(chosen) if absolute else "chosen") config.set("interface", "port", "0")