diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 8b86dca..95a3426 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,12 +1,7 @@ # Formatting-only commits excluded from `git blame` (git config: # blame.ignoreRevsFile = .git-blame-ignore-revs). -# -# PLACEHOLDER — the ruff-format-only commit has NOT landed yet. Once the -# black -> ruff format switch is committed (~63 files reformatted, no code -# changes), append its full 40-char SHA below with a one-line note, e.g.: -# -# # style: switch formatter from black to ruff format (formatting-only, no code changes) -# -# -# Then enable it repo-wide: -# git config blame.ignoreRevsFile .git-blame-ignore-revs \ No newline at end of file +# style: apply ruff format across src, tests, scripts (52 files) [E2] +# Reviewed against its parent: identical ASTs and ordered comment tokens. +9cb2642687b5565d752762eb8e6c921b6c21608e + +# Enable locally: git config blame.ignoreRevsFile .git-blame-ignore-revs \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c676cd..54c0168 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # Pre-commit gates — mirror the `gate` job in .github/workflows/test.yml: # ruff check, ruff format, mypy on src (the same arms `run.py gate` runs). -# Replaces black: ruff format is the repo formatter (see [tool.ruff] in -# pyproject.toml); the black config/dev-dep removal is tracked in TODO.md. +# Ruff format is the repo formatter (see [tool.ruff] in pyproject.toml). +# The black configuration and development dependency were removed in E2. repos: - repo: https://github.com/astral-sh/ruff-pre-commit diff --git a/Makefile b/Makefile index abcdd76..1d9887e 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ JOURNAL_DIR ?= ../ActiveInferenceJournal MANIFEST ?= data/output/channel_videos.json +RAG_BASE_PATH ?= . # Load .env file if it exists ifneq (,$(wildcard .env)) @@ -69,8 +70,8 @@ lint: uv run python run.py gate lint format format: - uv run ruff format src/ tests/ - uv run ruff check --fix src/ tests/ + uv run ruff format src/ tests/ scripts/ + uv run ruff check --fix src/ tests/ scripts/ db-start: surreal start --log trace --user root --pass root --bind 0.0.0.0:8080 rocksdb://./data/database @@ -112,7 +113,7 @@ extract-entities: echo "Error: COHERE_API_KEY not found in .env file"; \ exit 1; \ fi - uv run python -m journal_utilities.rag.main + uv run python -m journal_utilities.rag.main --pending --base-path "$(RAG_BASE_PATH)" enumerate-channel: @echo "Enumerating videos on the Active Inference channel..." diff --git a/TODO.md b/TODO.md index 9a816fa..ab6cbde 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,44 @@ **Status:** Maintained **Owner:** Active Inference Institute / Journal-Utilities maintainers -**Last reviewed:** 2026-11-12 (verification pass — Publishing contract migration, gate -re-run on this checkout, pin and formatter state confirmed; see the pass-5 entry below) +**Last reviewed:** 2026-11-13 (source-qualified CLI/API, disposable-database and +browser acceptance; earlier dated passes below retain their historical scope). + +--- + +## Pass 6 — pipeline and graph reliability (2026-11-13) + +- [x] **J1/J3 — Serve ownership and corpus** — occupied ports are refused without + killing their owners. The configured corpus reaches the actual web app; + browser navigation displayed its title and transcript with external media blocked. +- [x] **J2 — Required exports** — empty outcomes and mixed-success failures return + nonzero; required failure stops `full` and default orchestration. +- [x] **J4/J6 — Source identity and replay** — trusted source/type/name identity, + record-ID endpoints and deterministic edges prevent cross-source merging. + Explicit legacy migration preserves unknown-source rows and refuses conflicting + claims; failed edges remain pending and retry preserves historical dates. + Real in-memory SurrealDB 4.0.1 acceptance used SDK 3.0.0, including injected + write failure and replay. No production database was accessed or migrated. +- [x] **J5/A1 — Extraction entry point** — explicit file or pending selection performs + work, reports totals and propagates failures. Empty queues do not construct + the provider. `make extract-entities` selects pending work under `RAG_BASE_PATH`. +- [x] **J7 — Request boundaries** — bounded body ingestion, typed chat fields, + pre-stream errors and bounded public pagination are covered by focused tests. +- [x] **O3/O4/O5/A2 — Maintenance** — reviewed format-only commit recorded for blame; + stale test-count prose and Black commentary removed; both Makefile formatter + operations now include `scripts/`. + +Focused Python 3.12 checks, actual database acceptance and browser navigation +passed. This is not a full-suite, coverage, hosted-CI, live-Cohere or production +migration claim. Historical M10 metadata stamping and ME2 module-path repair +were partial steps, not prior proof of complete identity or extraction behavior. +The dated measurement files remain unchanged. + +Still open: hosted billing, qualified exact-pin GNN admission (no automatic pin +bump), canonical Publishing producer adoption, admitted full-gate measurements, +GPU/live-provider/YouTube-write qualification, the sibling translation migration, +and the owner-scheduled description backlog. Existing dated entries retain the +details of those external and ownership constraints. --- diff --git a/docs/RUN_GUIDE.md b/docs/RUN_GUIDE.md index a951c15..98358cb 100644 --- a/docs/RUN_GUIDE.md +++ b/docs/RUN_GUIDE.md @@ -98,3 +98,15 @@ it runs the live download tests. 2. Add the row to the arms table above and any rule it enforces to the doc it protects. 3. Run `uv run python run.py gate ` and, if the timing page should change, `uv run python run.py gate report`. + +## Listener ownership and corpus selection + +`serve` refuses an occupied port without discovering or signalling its owner. +Stop that owner explicitly or choose another configured port. +`general.data_dir` is passed to the web application: relative paths resolve +against the project root; absolute paths remain absolute. + +`make extract-entities` selects pending database work and resolves relative +transcript paths under `RAG_BASE_PATH` (default `.`). +`make format` runs both Ruff operations over `src/ tests/ scripts/`; it changes +files and is not an acceptance-only command. diff --git a/docs/export.md b/docs/export.md index 18f896e..67b22f2 100644 --- a/docs/export.md +++ b/docs/export.md @@ -76,3 +76,11 @@ html = true # Output directory relative to project root output_dir = data/export ``` + +## Required export completion + +`run.py export` returns nonzero when no outcomes are produced or any requested +export fails, including mixed-success batches. Successfully written or +already-existing nonempty outputs remain available. `full` and the default +pipeline propagate required export failure; the default pipeline does not +continue to tests or serving after that failure, independently of `test_strict`. diff --git a/docs/rag.md b/docs/rag.md index fc9c0c3..671446c 100644 --- a/docs/rag.md +++ b/docs/rag.md @@ -35,15 +35,17 @@ Defines the ontology for extraction. Manages interaction with SurrealDB. -- **Nodes**: Stores entities as graph nodes (e.g., `person:karl_friston`). -- **Edges**: Stores relationships as graph edges (e.g., `authored` -> `concept:free_energy_principle`). -- **Idempotency**: Ensures entities are merged rather than duplicated. +- **Nodes**: Entity records are identified by trusted transcript source, entity type and name. +- **Edges**: Relationships reference the returned entity record IDs, not generic display names. +- **Idempotency**: Scoped records and deterministic relationship identities are reused on replay. - **SDK**: `surrealdb>=3.0.0` (the async-client generation). `SurrealDBClient` opens the connection with `AsyncSurreal`, types it as `AsyncSurrealConnection`, and awaits `connect()` / `signin()` / `use()` / `query()` / `close()`. `query()` answers one entry per statement, so single-statement queries are unwrapped to their rows - (`_first_statement`) before use; `create()` on a table answers with the created records. - Server-side, the SDK accepts SurrealDB 2.x and newer and refuses older servers. + (`_first_statement`) before use. The qualified 3.0.0 SDK returns a single record + mapping from `create()` and singular-endpoint `relate()`, not a list. + The source-identity migration was exercised against SurrealDB 4.0.1; this is + not a qualification of every server version accepted by the SDK. ## Pipeline (`main.py`) @@ -51,15 +53,15 @@ The `JournalRAGPipeline` class orchestrates the process: 1. **`process_transcript(transcript)`**: - Creates a `Transcript` record in DB. - -Calls `extractor.extract_core_entities()`. + - Calls `extractor.extract_core_entities()`. - Converts results via `EntityAdapter`. - Upserts entities and relationships to SurrealDB. ## Usage ```bash -# Run the extraction pipeline on all new transcripts -make extract-entities +# Select pending database transcripts; resolve their relative paths under the corpus +make extract-entities RAG_BASE_PATH=/path/to/corpus ``` ## Configuration @@ -69,3 +71,30 @@ make extract-entities | `COHERE_API_KEY` | API Key for Cohere AI | | `COHERE_MODEL` | Model ID (default `command-a-03-2025`, from `rag/settings.py`) | | `DB_URL` | SurrealDB connection string | + +## Explicit extraction and source identity + +Select files with `python -m journal_utilities.rag.main --file transcript.txt [more.txt ...]` +or pending database work with `python -m journal_utilities.rag.main --pending --base-path /path/to/corpus`. +No selection is a usage error (exit 2). Requested failures return exit 1; +successful runs, including an explicitly selected empty pending queue, return 0 +and print processed/failed/entity/relationship totals. Extraction needs Cohere +credentials and SurrealDB; an empty queue does not construct the provider client. + +Relationship endpoints are checked against the trusted source-qualified entity +identity. Missing required endpoints or failed edge writes leave the transcript +pending. Retry and replay reuse persisted records rather than duplicating edges. +Generic names or metadata claims never establish provenance for unknown-source +legacy records. File and database loaders parse dates from the full filename, +including dotted dates without a separate extension; invalid or absent dates use +`datetime.min`. Titles still use the filename stem; retries preserve stored historical dates. + +## Legacy schema migration + +Take a reviewed backup and stop concurrent writers before explicitly running +`python -m journal_utilities.rag.main --migrate-source-identity`. +Ordinary connection refuses a legacy schema rather than silently migrating it. +The migration changes schema, preserves records and performs no Cohere request. +It refuses ambiguous identities or contradictory attributed endpoints; it does +not backfill provenance. Qualify it against a disposable database before any +production migration. diff --git a/docs/web_interface.md b/docs/web_interface.md index 0022e0b..5ed5d6f 100644 --- a/docs/web_interface.md +++ b/docs/web_interface.md @@ -85,3 +85,26 @@ The frontend files are served directly from `src/journal_utilities/interface/sta - `src/journal_utilities/interface/app.py`: Main application logic. - `src/journal_utilities/interface/static/app.js`: Core frontend logic (routing, rendering). - `src/journal_utilities/interface/static/index.html`: Base layout. + +## Request admission limits + +Chat, streaming chat and clear-session requests must be JSON objects with +string `session_id` (default `default`, at most 200 characters), string `message` +(default empty, at most 32000 characters), and boolean `use_rag` (default true). +Chat and streaming require a nonblank message. Clear validates provided fields +but does not require a message. Malformed JSON or an empty required message +returns 400; invalid shapes/types return 422; byte or field limits return 413 +before a provider request or streaming response starts. + +The application retains at most 387424 body bytes before JSON decoding and +rejects an over-limit chunk before appending it. Streamed bytes are checked even +with missing or understated Content-Length. This does not bound upstream +server/ASGI transport allocations. + +Video pagination requires offset >= 0 and limit 1..200. Search requires limit +1..100; internal unlimited search remains available through `limit=None`. + +`run.py serve` uses the configured corpus and refuses occupied ports without +terminating another process. Browser fixtures must use canonical 11-character +YouTube IDs; external thumbnails, fonts and embedded video still need network +access independently of the locally served transcript. diff --git a/run.py b/run.py index 92a3b59..111b799 100644 --- a/run.py +++ b/run.py @@ -24,7 +24,6 @@ import json import logging import os -import signal import socket import subprocess import sys @@ -227,12 +226,19 @@ def cmd_export(config: configparser.ConfigParser, _args: argparse.Namespace) -> print("⚠ No transcripts found to export") return 1 + failures = 0 + outcomes = 0 for fmt_name, fmt_results in results.items(): ok = sum(1 for r in fmt_results if r.status == "success") skip = sum(1 for r in fmt_results if r.status == "skipped") fail = sum(1 for r in fmt_results if r.status == "failed") + failures += fail + outcomes += len(fmt_results) print(f" {fmt_name}: {ok} exported, {skip} skipped, {fail} failed") + if failures or not outcomes: + print("\n✗ Export incomplete") + return 1 print("\n✓ Export complete") return 0 @@ -287,36 +293,6 @@ def _port_in_use(port: int) -> bool: return True -def _free_port(port: int) -> bool: - """Try to kill whatever is occupying *port*. Return True on success.""" - try: - result = subprocess.run( - ["lsof", "-ti", f":{port}"], - capture_output=True, - text=True, - timeout=5, - ) - pids = result.stdout.strip().split() - if not pids: - return True - - my_pid = str(os.getpid()) - for pid in pids: - if pid == my_pid: - continue - logger.info("Killing stale process %s on port %d", pid, port) - os.kill(int(pid), signal.SIGTERM) - - # Brief wait for cleanup - import time as _t - - _t.sleep(0.5) - return not _port_in_use(port) - except Exception as exc: - logger.warning("Could not free port %d: %s", port, exc) - return False - - def cmd_serve(config: configparser.ConfigParser, _args: argparse.Namespace) -> int: """Start the web interface.""" import uvicorn @@ -327,23 +303,19 @@ def cmd_serve(config: configparser.ConfigParser, _args: argparse.Namespace) -> i port_str = get_str(config, "interface", "port", "8000") port = int(port_str) - # ── Auto-resolve port conflicts ──────────────────────────────── + # An existing listener belongs to its owner; never kill or reclaim it. if _port_in_use(port): - logger.warning("Port %d already in use — attempting to free it", port) - if _free_port(port): - logger.info("Port %d freed successfully", port) - else: - logger.error("Cannot free port %d — aborting serve", port) - print(f" ✗ Port {port} is occupied and could not be freed.") - print(f" Try: lsof -ti :{port} | xargs kill -9") - return 1 + logger.error("Port %d is already occupied; listener left untouched", port) + print(f" ✗ Port {port} is occupied. Stop its owner explicitly or choose another port.") + return 1 print(f"Starting web interface at http://{host}:{port}") # Configure uvicorn logging to match ours if needed, # but uvicorn has its own config. We'll let it handle itself. try: - app = create_app() + data_dir = PROJECT_ROOT / get_str(config, "general", "data_dir", "data/output") + app = create_app(data_dir=data_dir) # log_config=None tells uvicorn to use the existing logging configuration # which allows our root logger (with FileHandler) to capture the logs. uvicorn.run(app, host=host, port=port, log_config=None) @@ -535,8 +507,9 @@ def cmd_default(config: configparser.ConfigParser, args: argparse.Namespace) -> print("━" * 62) rc = cmd_export(config, args) if rc != 0: - logger.warning("Export step returned non-zero: %d", rc) - print(f" ⚠ Export finished with warnings (exit code {rc})\n") + logger.error("Export step failed: %d", rc) + print(f" ✗ Export failed (exit code {rc}) — aborting pipeline\n") + return rc else: print() diff --git a/src/journal_utilities/interface/app.py b/src/journal_utilities/interface/app.py index 800b14a..634f409 100644 --- a/src/journal_utilities/interface/app.py +++ b/src/journal_utilities/interface/app.py @@ -18,6 +18,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, StreamingResponse from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel, ConfigDict, ValidationError from journal_utilities.interface.chat_engine import ChatEngine from journal_utilities.interface.data_loader import DataLoader @@ -30,9 +31,48 @@ STATIC_DIR = Path(__file__).parent / "static" -# Upper bound on a single chat message; the body is otherwise read unbounded -# and forwarded to Ollama (memory + LLM-cost DoS). MAX_MESSAGE_LENGTH = 32_000 +MAX_SESSION_LENGTH = 200 +# JSON can escape one non-BMP character as two six-byte Unicode escapes. +MAX_CHAT_BODY_BYTES = 12 * (MAX_MESSAGE_LENGTH + MAX_SESSION_LENGTH) + 1024 + + +class _ChatRequest(BaseModel): + model_config = ConfigDict(strict=True) + + session_id: str = "default" + message: str = "" + use_rag: bool = True + + +async def _read_chat_request(request: Request, *, require_message: bool) -> _ChatRequest: + """Bound retained bytes before JSON decoding, independent of Content-Length.""" + declared = request.headers.get("content-length") + if declared is not None: + try: + length = int(declared) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid Content-Length") from exc + if length < 0: + raise HTTPException(status_code=400, detail="Invalid Content-Length") + if length > MAX_CHAT_BODY_BYTES: + raise HTTPException(status_code=413, detail="Chat request body too large") + body = bytearray() + async for chunk in request.stream(): + if len(body) + len(chunk) > MAX_CHAT_BODY_BYTES: + raise HTTPException(status_code=413, detail="Chat request body too large") + body.extend(chunk) + try: + payload = _ChatRequest.model_validate_json(body) + except ValidationError as exc: + errors = exc.errors(include_input=False, include_context=False, include_url=False) + status = 400 if any(error["type"] == "json_invalid" for error in errors) else 422 + raise HTTPException(status_code=status, detail="Invalid chat request") from exc + if len(payload.message) > MAX_MESSAGE_LENGTH or len(payload.session_id) > MAX_SESSION_LENGTH: + raise HTTPException(status_code=413, detail="Message or session id too long") + if require_message and not payload.message.strip(): + raise HTTPException(status_code=400, detail="Message cannot be empty") + return payload def create_app(data_dir: Path | None = None) -> FastAPI: @@ -84,8 +124,8 @@ async def get_videos( has_transcript: bool | None = None, sort_by: str = "upload_date", reverse: bool = True, - offset: int = 0, - limit: int = Query(default=50, le=200), + offset: int = Query(default=0, ge=0), + limit: int = Query(default=50, ge=1, le=200), q: str | None = None, ) -> dict[str, Any]: """Get paginated, filterable list of videos.""" @@ -252,7 +292,7 @@ async def get_categories() -> dict[str, Any]: @app.get("/api/search") async def search( q: str = Query(..., min_length=2), - limit: int = Query(default=20, le=100), + limit: int = Query(default=20, ge=1, le=100), ) -> dict[str, Any]: """Full-text search across all transcripts.""" results = loader.search_index.search(q, limit=limit) @@ -282,47 +322,25 @@ async def chat_status() -> dict[str, Any]: @app.post("/api/chat") async def chat_message(request: Request) -> dict[str, Any]: - """Send a chat message (non-streaming).""" - body = await request.json() - session_id = body.get("session_id", "default") - message = body.get("message", "") - use_rag = body.get("use_rag", True) - - if not message.strip(): - raise HTTPException(status_code=400, detail="Message cannot be empty") - if len(message) > MAX_MESSAGE_LENGTH or len(session_id) > 200: - raise HTTPException(status_code=413, detail="Message or session id too long") - - return await chat_engine.chat(session_id, message, use_rag=use_rag) + """Send a validated, bounded chat message (non-streaming).""" + payload = await _read_chat_request(request, require_message=True) + return await chat_engine.chat(payload.session_id, payload.message, use_rag=payload.use_rag) @app.post("/api/chat/stream") async def chat_stream(request: Request) -> StreamingResponse: - """Send a chat message with streaming response (SSE).""" - body = await request.json() - session_id = body.get("session_id", "default") - message = body.get("message", "") - use_rag = body.get("use_rag", True) - - if not message.strip(): - raise HTTPException(status_code=400, detail="Message cannot be empty") - if len(message) > MAX_MESSAGE_LENGTH or len(session_id) > 200: - raise HTTPException(status_code=413, detail="Message or session id too long") - + """Validate completely before opening a streaming response.""" + payload = await _read_chat_request(request, require_message=True) return StreamingResponse( - chat_engine.chat_stream(session_id, message, use_rag=use_rag), + chat_engine.chat_stream(payload.session_id, payload.message, use_rag=payload.use_rag), media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - }, + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, ) @app.post("/api/chat/clear") async def chat_clear(request: Request) -> dict[str, str]: - """Clear a chat session.""" - body = await request.json() - session_id = body.get("session_id", "default") - chat_engine.clear_session(session_id) + """Clear a validated session without calling a provider.""" + payload = await _read_chat_request(request, require_message=False) + chat_engine.clear_session(payload.session_id) return {"status": "cleared"} # ------------------------------------------------------------------ diff --git a/src/journal_utilities/rag/adapters/entity_adapter.py b/src/journal_utilities/rag/adapters/entity_adapter.py index 3136d12..6df0e59 100644 --- a/src/journal_utilities/rag/adapters/entity_adapter.py +++ b/src/journal_utilities/rag/adapters/entity_adapter.py @@ -15,7 +15,7 @@ class EntityAdapter: """Converts CoreEntities to simple Entity and Relationship models for database storage.""" @staticmethod - def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: + def convert_core_entities(core: CoreEntities, *, source: str) -> Sequence[Entity]: """Convert CoreEntities to a flat list of Entity objects. Args: @@ -30,6 +30,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for concept in core.concepts: entities.append( Entity( + source=source, name=concept.name, type="concept", description=concept.definition, @@ -47,6 +48,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for researcher in core.researchers: entities.append( Entity( + source=source, name=researcher.name, type="researcher", description=researcher.contribution or "", @@ -64,6 +66,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for citation in core.citations: entities.append( Entity( + source=source, name=citation.title, type="citation", description=citation.context or "", @@ -84,6 +87,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for term in core.technical_terms: entities.append( Entity( + source=source, name=term.term, type="technical_term", description=term.explanation, @@ -101,6 +105,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: for idx, insight in enumerate(core.key_insights): entities.append( Entity( + source=source, name=f"Insight {idx + 1}", type="key_insight", description=insight, @@ -114,7 +119,7 @@ def convert_core_entities(core: CoreEntities) -> Sequence[Entity]: return entities @staticmethod - def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: + def convert_detailed_analysis(detailed: DetailedAnalysis, *, source: str) -> Sequence[Entity]: """Convert DetailedAnalysis to a flat list of Entity objects. Args: @@ -129,6 +134,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for method in detailed.methods_techniques: entities.append( Entity( + source=source, name=method.name, type="method_technique", description=method.description, @@ -146,6 +152,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for notation in detailed.mathematical_notation: entities.append( Entity( + source=source, name=notation.notation, type="mathematical_notation", description=notation.meaning, @@ -162,6 +169,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for equation in detailed.equations: entities.append( Entity( + source=source, name=equation.name or equation.formula[:50], type="equation", description=equation.context or "", @@ -178,6 +186,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for tool in detailed.tools_resources: entities.append( Entity( + source=source, name=tool.name, type="tool_resource", description=tool.purpose or "", @@ -195,6 +204,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for problem in detailed.research_problems: entities.append( Entity( + source=source, name=problem.problem[:100], # Use first 100 chars as name type="research_problem", description=problem.problem, @@ -212,6 +222,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: for application in detailed.applications: entities.append( Entity( + source=source, name=f"{application.domain} Application", type="application", description=application.description, @@ -228,7 +239,7 @@ def convert_detailed_analysis(detailed: DetailedAnalysis) -> Sequence[Entity]: return entities @staticmethod - def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: + def extract_relationships(core: CoreEntities, *, source: str) -> Sequence[Relationship]: """Extract relationships from CoreEntities structure. Args: @@ -244,6 +255,9 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: for related_concept in concept.related_concepts: relationships.append( Relationship( + transcript_source=source, + source_type="concept", + target_type="concept", source=concept.name, target=related_concept, relationship_type="related_to", @@ -262,6 +276,9 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: if author in researcher_names: relationships.append( Relationship( + transcript_source=source, + source_type="researcher", + target_type="citation", source=author, target=citation.title, relationship_type="authored", @@ -276,31 +293,11 @@ def extract_relationships(core: CoreEntities) -> Sequence[Relationship]: @classmethod def convert_extracted_entities( - cls, extracted: ExtractedEntities + cls, extracted: ExtractedEntities, *, source: str ) -> tuple[Sequence[Entity], Sequence[Relationship]]: - """Convert complete ExtractedEntities to entities and relationships. - - Args: - extracted: Complete extraction result - - Returns: - Tuple of (entities, relationships) - """ - entities = list(cls.convert_core_entities(extracted.core)) - + """Use caller-owned provenance, never a scientific extraction's metadata.""" + entities = list(cls.convert_core_entities(extracted.core, source=source)) if extracted.detailed: - entities.extend(cls.convert_detailed_analysis(extracted.detailed)) - - # Stamp the transcript source onto every entity so the graph layer can - # scope merges per transcript — preventing "Insight 1" (or a truncated - # formula name) from one video silently merging with another's (M10). - source = getattr(extracted.core, "source", None) - if source: - for entity in entities: - meta = dict(entity.metadata) - meta.setdefault("source", source) - entity.metadata = meta - - relationships = cls.extract_relationships(extracted.core) - + entities.extend(cls.convert_detailed_analysis(extracted.detailed, source=source)) + relationships = cls.extract_relationships(extracted.core, source=source) return entities, relationships diff --git a/src/journal_utilities/rag/graph/surreal_client.py b/src/journal_utilities/rag/graph/surreal_client.py index 6e55e27..e519ee2 100644 --- a/src/journal_utilities/rag/graph/surreal_client.py +++ b/src/journal_utilities/rag/graph/surreal_client.py @@ -1,34 +1,68 @@ -"""SurrealDB client for storing entities and relationships.""" +"""Source-qualified SurrealDB storage; legacy attribution is never inferred.""" +import hashlib +import json from collections.abc import Mapping, Sequence from datetime import UTC, datetime from typing import Any, cast -import structlog -from surrealdb import AsyncSurreal, AsyncSurrealConnection +from surrealdb import AsyncSurreal, AsyncSurrealConnection, RecordID, StringRecordId from journal_utilities.rag.models import Entity, Relationship, Transcript from journal_utilities.rag.settings import settings -logger = structlog.get_logger(__name__) - -def _first_statement(result: Sequence[Any]) -> list[Any]: - """Rows produced by the first statement of a ``query()`` response. - - surrealdb 3.x answers ``query()`` with one entry per statement, so a - single-statement query arrives as ``[rows]``. Every caller here issues one - statement and wants those rows, which is the shape the pre-3.x client - returned directly. - """ - if not result: - return [] - rows = result[0] - return rows if isinstance(rows, list) else [] +def _first_statement(result: object) -> object: + """The installed SDK 3.x returns one entry per statement.""" + if not isinstance(result, list) or len(result) != 1: + raise RuntimeError("Unexpected SurrealDB statement result") + return result[0] + + +def _single(rows: Sequence[Mapping[str, Any]], identity: str) -> Mapping[str, Any] | None: + if len(rows) > 1: + raise RuntimeError(f"Ambiguous {identity}; explicit data review required") + return rows[0] if rows else None + + +def _utc(value: datetime) -> datetime: + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + + +_SCHEMA = """ +DEFINE TABLE IF NOT EXISTS entity SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON entity TYPE string; +DEFINE FIELD IF NOT EXISTS type ON entity TYPE string; +DEFINE FIELD IF NOT EXISTS source ON entity TYPE option; +DEFINE FIELD IF NOT EXISTS description ON entity TYPE option; +DEFINE FIELD IF NOT EXISTS confidence ON entity TYPE number; +DEFINE FIELD IF NOT EXISTS mentions ON entity TYPE number; +DEFINE FIELD IF NOT EXISTS context ON entity TYPE array; +DEFINE FIELD OVERWRITE metadata ON entity TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS created_at ON entity TYPE datetime; +DEFINE INDEX IF NOT EXISTS entity_source_type_name ON entity FIELDS source, type, name UNIQUE; +DEFINE TABLE IF NOT EXISTS relates SCHEMAFULL TYPE RELATION; +DEFINE FIELD IF NOT EXISTS source ON relates TYPE option; +DEFINE FIELD IF NOT EXISTS relationship_type ON relates TYPE string; +DEFINE FIELD IF NOT EXISTS description ON relates TYPE option; +DEFINE FIELD IF NOT EXISTS confidence ON relates TYPE number; +DEFINE FIELD IF NOT EXISTS context ON relates TYPE option; +DEFINE FIELD OVERWRITE metadata ON relates TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS created_at ON relates TYPE datetime; +DEFINE INDEX IF NOT EXISTS relates_source_endpoints ON relates FIELDS source, in, out, relationship_type; +DEFINE TABLE IF NOT EXISTS transcript SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS title ON transcript TYPE string; +DEFINE FIELD IF NOT EXISTS date ON transcript TYPE datetime; +DEFINE FIELD IF NOT EXISTS content ON transcript TYPE string; +DEFINE FIELD IF NOT EXISTS source ON transcript TYPE option; +DEFINE FIELD OVERWRITE metadata ON transcript TYPE object FLEXIBLE; +DEFINE FIELD IF NOT EXISTS processed ON transcript TYPE bool; +DEFINE FIELD IF NOT EXISTS created_at ON transcript TYPE datetime; +""" class SurrealDBClient: - """Client for interacting with SurrealDB graph database.""" + """Persist identities using trusted transcript source, entity type and name.""" def __init__( self, @@ -38,459 +72,286 @@ def __init__( username: str | None = None, password: str | None = None, ) -> None: - """Initialize the SurrealDB client. - - Args: - url: SurrealDB connection URL. - namespace: Database namespace. - database: Database name. - username: Authentication username. - password: Authentication password. - """ self.url = url or settings.surrealdb_url self.namespace = namespace or settings.surrealdb_namespace self.database = database or settings.surrealdb_database self.username = username or settings.surrealdb_username self.password = password or settings.surrealdb_password self.db: AsyncSurrealConnection | None = None - logger.info( - "Initialized SurrealDBClient", - namespace=self.namespace, - database=self.database, - ) + self.migration_summary: dict[str, int] | None = None - async def connect(self) -> None: - """Connect to SurrealDB.""" + async def connect(self, *, migrate_source_identity: bool = False) -> None: + """Refuse legacy schema unless migration was explicitly selected. + + Migration requires a quiescent database. It changes schema, not legacy + data: absent provenance stays absent and ambiguous attribution fails. + """ + self.db = AsyncSurreal(self.url) try: - self.db = AsyncSurreal(self.url) await self.db.connect() - await self.db.signin({"user": self.username, "pass": self.password}) + await self.db.signin({"username": self.username, "password": self.password}) await self.db.use(self.namespace, self.database) - logger.info("Connected to SurrealDB") - await self._initialize_schema() - except Exception as e: - logger.error("Failed to connect to SurrealDB", error=str(e)) + await self._initialize_schema(migrate_source_identity=migrate_source_identity) + except Exception: + await self.disconnect() raise async def disconnect(self) -> None: - """Disconnect from SurrealDB.""" - if self.db: - await self.db.close() - logger.info("Disconnected from SurrealDB") + if self.db is not None: + db, self.db = self.db, None + await db.close() - async def _initialize_schema(self) -> None: - """Initialize database schema.""" - if not self.db: + async def _initialize_schema(self, *, migrate_source_identity: bool = False) -> None: + if self.db is None: raise RuntimeError("Database not connected") - - # Define entity table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS entity SCHEMAFULL; - DEFINE FIELD IF NOT EXISTS name ON TABLE entity TYPE string; - DEFINE FIELD IF NOT EXISTS type ON TABLE entity TYPE string; - DEFINE FIELD IF NOT EXISTS description ON TABLE entity TYPE option; - DEFINE FIELD IF NOT EXISTS confidence ON TABLE entity TYPE number; - DEFINE FIELD IF NOT EXISTS mentions ON TABLE entity TYPE number; - DEFINE FIELD IF NOT EXISTS context ON TABLE entity TYPE array; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE entity TYPE object; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE entity TYPE datetime; - DEFINE INDEX IF NOT EXISTS entity_name_type ON TABLE entity COLUMNS name, type UNIQUE; - """ + info = cast(Mapping[str, Any], _first_statement(await self.db.query("INFO FOR DB;"))) + tables = info["tables"] + indexes = {} + if "entity" in tables: + entity_info = cast( + Mapping[str, Any], _first_statement(await self.db.query("INFO FOR TABLE entity;")) + ) + indexes = entity_info["indexes"] + legacy = ("entity" in tables or "relates" in tables) and ( + "entity_source_type_name" not in indexes or "entity_name_type" in indexes ) - - # Define relationship table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS relates SCHEMAFULL TYPE RELATION; - DEFINE FIELD IF NOT EXISTS relationship_type ON TABLE relates TYPE string; - DEFINE FIELD IF NOT EXISTS description ON TABLE relates TYPE option; - DEFINE FIELD IF NOT EXISTS confidence ON TABLE relates TYPE number; - DEFINE FIELD IF NOT EXISTS context ON TABLE relates TYPE option; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE relates TYPE object; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE relates TYPE datetime; - """ + if legacy and not migrate_source_identity: + raise RuntimeError( + "Legacy graph schema: review a backup, stop other writers, then explicitly " + "run --migrate-source-identity. No legacy provenance will be assigned." + ) + if migrate_source_identity: + self.migration_summary = await self._admit_migration(tables) + remove_old = ( + "REMOVE INDEX entity_name_type ON entity;" if "entity_name_type" in indexes else "" ) - - # Define transcript table - await self.db.query( - """ - DEFINE TABLE IF NOT EXISTS transcript SCHEMAFULL; - DEFINE FIELD IF NOT EXISTS title ON TABLE transcript TYPE string; - DEFINE FIELD IF NOT EXISTS date ON TABLE transcript TYPE datetime; - DEFINE FIELD IF NOT EXISTS content ON TABLE transcript TYPE string; - DEFINE FIELD IF NOT EXISTS source ON TABLE transcript TYPE option; - DEFINE FIELD IF NOT EXISTS metadata ON TABLE transcript TYPE object; - DEFINE FIELD IF NOT EXISTS processed ON TABLE transcript TYPE bool; - DEFINE FIELD IF NOT EXISTS created_at ON TABLE transcript TYPE datetime; - """ + # A failed index definition must not leave a partially migrated schema. + await self.db.query("BEGIN TRANSACTION;" + _SCHEMA + remove_old + "COMMIT TRANSACTION;") + + async def _admit_migration(self, tables: Mapping[str, Any]) -> dict[str, int]: + entities = list(await self.query("SELECT * FROM entity;")) if "entity" in tables else [] + edges = list(await self.query("SELECT * FROM relates;")) if "relates" in tables else [] + transcripts = ( + list(await self.query("SELECT * FROM transcript;")) if "transcript" in tables else [] ) - - logger.info("Initialized database schema") - - async def create_entity(self, entity: Entity) -> str: - """Create or update an entity in the database. - - Args: - entity: Entity to create or update. - - Returns: - Entity ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Check if entity exists. Generic/truncated names (e.g. "Insight 1") - # collide across transcripts, so merge is scoped to the entity's - # source when it is known — otherwise a "Insight 1" from two - # different videos would silently merge into one record (M10). - entity_type = entity.type.value if hasattr(entity.type, "value") else entity.type - source = (entity.metadata or {}).get("source") - if source: - existing = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type" - " AND source = $source", - {"name": entity.name, "type": entity_type, "source": source}, - ) - ) - else: - existing = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type", - {"name": entity.name, "type": entity_type}, - ) - ) - - if existing: - # Update existing entity - current = existing[0] - entity_id = current["id"] - await self.db.update( - entity_id, - { - "mentions": entity.mentions, - "confidence": max(entity.confidence, current["confidence"]), - "context": list(set(entity.context + current["context"])), - "description": entity.description or current["description"], - }, - ) - logger.debug("Updated entity", entity_id=entity_id) - return str(entity_id) - else: - # Create new entity - payload: dict[str, Any] = { - "name": entity.name, - "type": entity_type, - "description": entity.description, - "confidence": entity.confidence, - "mentions": entity.mentions, - "context": entity.context, - "metadata": entity.metadata, - "created_at": datetime.now(UTC).isoformat(), - } - if source: - payload["source"] = source - # ``create`` on a table answers with the created records. - created = cast(list[dict[str, Any]], await self.db.create("entity", payload)) - entity_id = created[0]["id"] - logger.debug("Created entity", entity_id=entity_id) - return str(entity_id) - - except Exception as e: - logger.error("Failed to create entity", entity=entity.name, error=str(e)) - raise - - async def create_relationship(self, relationship: Relationship) -> str: - """Create a relationship between entities. - - Args: - relationship: Relationship to create. - - Returns: - Relationship ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Get entity IDs (support both old and new field names) - source_name = ( - relationship.source_entity - if hasattr(relationship, "source_entity") - else relationship.source - ) - target_name = ( - relationship.target_entity - if hasattr(relationship, "target_entity") - else relationship.target - ) - - source = _first_statement( - await self.db.query( - "SELECT id FROM entity WHERE name = $name LIMIT 1", - {"name": source_name}, - ) - ) - target = _first_statement( - await self.db.query( - "SELECT id FROM entity WHERE name = $name LIMIT 1", - {"name": target_name}, - ) - ) - - if not source or not target: - raise ValueError("Source or target entity not found") - - source_id = source[0]["id"] - target_id = target[0]["id"] - - # Create relationship - result = _first_statement( - await self.db.query( - f""" - RELATE {source_id}->relates->{target_id} CONTENT {{ - relationship_type: $type, - description: $description, - confidence: $confidence, - context: $context, - metadata: $metadata, - created_at: $created_at - }} - """, - { - "type": relationship.relationship_type, - "description": relationship.description, - "confidence": relationship.confidence, - "context": relationship.context, - "metadata": relationship.metadata, - "created_at": datetime.now(UTC).isoformat(), - }, - ) - ) - - rel_id = result[0]["id"] - logger.debug("Created relationship", relationship_id=rel_id) - return str(rel_id) - - except Exception as e: - logger.error( - "Failed to create relationship", - source=source_name, - target=target_name, - error=str(e), - ) - raise - - async def create_transcript(self, transcript: Transcript) -> str: - """Store a transcript in the database. - - Args: - transcript: Transcript to store. - - Returns: - Transcript ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - created = cast( - list[dict[str, Any]], - await self.db.create( - "transcript", - { - "title": transcript.title, - "date": transcript.date.isoformat(), - "content": transcript.content, - "source": transcript.source, - "metadata": transcript.metadata, - "processed": transcript.processed, - "created_at": transcript.created_at.isoformat(), - }, - ), - ) - transcript_id = created[0]["id"] - logger.info("Created transcript", transcript_id=transcript_id) - return str(transcript_id) - - except Exception as e: - logger.error("Failed to create transcript", title=transcript.title, error=str(e)) - raise - - async def get_entity_by_name( - self, name: str, entity_type: str | None = None - ) -> dict[str, Any] | None: - """Get an entity by name. - - Args: - name: Entity name. - entity_type: Optional entity type filter. - - Returns: - Entity data or None if not found. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - if entity_type: - rows = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name AND type = $type LIMIT 1", - {"name": name, "type": entity_type}, + by_id = {str(row["id"]): row for row in entities} + identities: set[tuple[str | None, str, str]] = set() + unknown = 0 + for row in entities: + source = self._legacy_source(row) + unknown += source is None + identity = (source, row["type"], row["name"]) + if identity in identities: + raise RuntimeError("Ambiguous legacy entity identity; migration refused") + identities.add(identity) + edge_keys: set[tuple[str, str, str, str]] = set() + unknown_edges = 0 + for row in edges: + source = self._legacy_source(row) + if source is None: + unknown_edges += 1 + continue + for endpoint in ("in", "out"): + entity = by_id.get(str(row[endpoint])) + if entity is None or self._legacy_source(entity) != source: + raise RuntimeError( + "Ambiguous legacy relationship attribution; migration refused" ) - ) - else: - rows = _first_statement( - await self.db.query( - "SELECT * FROM entity WHERE name = $name LIMIT 1", - {"name": name}, - ) - ) - - if rows: - return cast(dict[str, Any], rows[0]) + key = (source, str(row["in"]), str(row["out"]), row["relationship_type"]) + if key in edge_keys: + raise RuntimeError("Ambiguous legacy relationship identity; migration refused") + edge_keys.add(key) + sources: set[str] = set() + for row in transcripts: + source = self._legacy_source(row) + if source is not None: + if source in sources: + raise RuntimeError("Ambiguous legacy transcript source; migration refused") + sources.add(source) + return { + "legacy_entities_preserved": len(entities), + "unknown_source_entities": unknown, + "legacy_relationships_preserved": len(edges), + "unknown_source_relationships": unknown_edges, + } + + @staticmethod + def _legacy_source(row: Mapping[str, Any]) -> str | None: + source = row.get("source") + if source is None or source == "": + # A metadata claim alone is not trusted provenance. Keep it untouched. return None - - except Exception as e: - logger.error("Failed to get entity", name=name, error=str(e)) - raise + if not isinstance(source, str) or not source.strip(): + raise RuntimeError("Ambiguous legacy source value; migration refused") + claim = (row.get("metadata") or {}).get("source") + if claim is not None and claim != source: + raise RuntimeError("Conflicting legacy source claims; migration refused") + return source async def query( self, query: str, params: Mapping[str, Any] | None = None ) -> Sequence[Mapping[str, Any]]: - """Execute a custom SurrealQL query. - - Args: - query: SurrealQL query string. - params: Optional query parameters. - - Returns: - Query results. - """ - if not self.db: + if self.db is None: raise RuntimeError("Database not connected") + rows = _first_statement(await self.db.query(query, dict(params) if params else None)) + if not isinstance(rows, list): + raise RuntimeError("Expected SurrealDB records") + return cast(Sequence[Mapping[str, Any]], rows) + + async def _record(self, record_id: str) -> Mapping[str, Any] | None: + return _single( + await self.query("SELECT * FROM $id;", {"id": StringRecordId(record_id)}), record_id + ) - try: - result = await self.db.query(query, dict(params) if params else None) - return cast(Sequence[Mapping[str, Any]], _first_statement(result)) - except Exception as e: - logger.error("Query failed", query=query, error=str(e)) - raise - - async def get_transcript_by_source(self, source_path: str) -> dict[str, Any] | None: - """Get a transcript by its source path. + async def _merge(self, record_id: str, changes: Mapping[str, Any]) -> None: + rows = await self.query( + "UPDATE $id MERGE $changes;", + {"id": StringRecordId(record_id), "changes": dict(changes)}, + ) + if _single(rows, record_id) is None: + raise RuntimeError(f"Record disappeared during update: {record_id}") - Args: - source_path: Path to the source transcript file. + async def get_entity_by_name( + self, name: str, *, entity_type: str, source: str + ) -> Mapping[str, Any] | None: + if not source.strip(): + raise ValueError("A trusted transcript source is required") + return _single( + await self.query( + "SELECT * FROM entity WHERE source = $source AND type = $type AND name = $name;", + {"source": source, "type": entity_type, "name": name}, + ), + "source-qualified entity", + ) - Returns: - Transcript data or None if not found. - """ - if not self.db: + async def create_entity(self, entity: Entity) -> str: + if self.db is None: raise RuntimeError("Database not connected") - - try: - rows = _first_statement( - await self.db.query( - "SELECT * FROM transcript WHERE source = $source LIMIT 1", - {"source": source_path}, - ) - ) - - if rows: - return cast(dict[str, Any], rows[0]) - return None - - except Exception as e: - logger.error("Failed to get transcript by source", source=source_path, error=str(e)) - raise - - async def get_unprocessed_transcript_paths(self) -> Sequence[str]: - """Get paths of all unprocessed transcripts. - - Returns: - List of source paths for unprocessed transcripts. - """ - if not self.db: + existing = await self.get_entity_by_name( + entity.name, entity_type=entity.type, source=entity.source + ) + payload: dict[str, Any] = entity.model_dump() + if existing: + # MERGE, not replacement: preserve identity and original creation time. + payload["confidence"] = max(entity.confidence, existing["confidence"]) + payload["context"] = list(dict.fromkeys([*existing["context"], *entity.context])) + payload["description"] = entity.description or existing.get("description", "") + entity_id = str(existing["id"]) + await self._merge(entity_id, payload) + return entity_id + payload["created_at"] = datetime.now(UTC) + row = await self.db.create("entity", payload) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Entity creation returned no record") + return str(row["id"]) + + async def create_relationship( + self, relationship: Relationship, *, source_id: str, target_id: str + ) -> str: + if self.db is None: raise RuntimeError("Database not connected") + for record_id, name, entity_type in ( + (source_id, relationship.source, relationship.source_type), + (target_id, relationship.target, relationship.target_type), + ): + endpoint = await self._record(record_id) + if endpoint is None or ( + endpoint.get("source"), + endpoint.get("type"), + endpoint.get("name"), + ) != (relationship.transcript_source, entity_type, name): + raise ValueError("Relationship endpoint does not match trusted source/type/name") + params = { + "source": relationship.transcript_source, + "in": StringRecordId(source_id), + "out": StringRecordId(target_id), + "type": relationship.relationship_type, + } + existing = _single( + await self.query( + "SELECT * FROM relates WHERE source = $source AND in = $in AND out = $out " + "AND relationship_type = $type;", + params, + ), + "source-qualified relationship", + ) + payload: dict[str, Any] = { + "source": relationship.transcript_source, + "relationship_type": relationship.relationship_type, + "description": relationship.description, + "confidence": relationship.confidence, + "context": relationship.context, + "metadata": relationship.metadata, + } + if existing: + edge_id = str(existing["id"]) + await self._merge(edge_id, payload) + return edge_id + # Deterministic record identity also prevents duplicate edges on concurrent retries. + key = json.dumps( + [relationship.transcript_source, source_id, target_id, relationship.relationship_type], + ensure_ascii=False, + separators=(",", ":"), + ) + edge = RecordID("relates", hashlib.sha256(key.encode("utf-8")).hexdigest()) + if await self._record(str(edge)) is not None: + raise RuntimeError("Relationship identity collision; refusing to overwrite") + payload["created_at"] = datetime.now(UTC) + row = await self.db.relate( + StringRecordId(source_id), edge, StringRecordId(target_id), payload + ) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Relationship creation returned no record") + return str(row["id"]) - try: - rows = _first_statement( - await self.db.query("SELECT source FROM transcript WHERE processed = false") - ) - - if rows: - return [rec["source"] for rec in rows if rec.get("source")] - return [] - - except Exception as e: - logger.error("Failed to get unprocessed transcript paths", error=str(e)) - raise - - async def update_transcript_status(self, transcript_id: str, processed: bool) -> None: - """Update the processed status of a transcript. - - Args: - transcript_id: ID of the transcript to update. - processed: New processed status. - """ - if not self.db: + async def create_transcript(self, transcript: Transcript) -> str: + if self.db is None: raise RuntimeError("Database not connected") + payload = transcript.model_dump() + payload.update( + date=_utc(transcript.date), created_at=_utc(transcript.created_at), processed=False + ) + row = await self.db.create("transcript", payload) + if not isinstance(row, Mapping) or "id" not in row: + raise RuntimeError("Transcript creation returned no record") + return str(row["id"]) + + async def get_transcript_by_source(self, source_path: str) -> Mapping[str, Any] | None: + return _single( + await self.query( + "SELECT * FROM transcript WHERE source = $source;", {"source": source_path} + ), + "transcript source", + ) - try: - await self.db.update(transcript_id, {"processed": processed}) - logger.debug( - "Updated transcript status", transcript_id=transcript_id, processed=processed - ) + async def get_unprocessed_transcript_paths(self) -> Sequence[str]: + rows = await self.query("SELECT source FROM transcript WHERE processed = false;") + paths = [] + for row in rows: + source = row.get("source") + if not isinstance(source, str) or not source.strip(): + raise RuntimeError( + "Pending transcript has no trusted source path; explicit review required" + ) + paths.append(source) + return paths - except Exception as e: - logger.error( - "Failed to update transcript status", - transcript_id=transcript_id, - error=str(e), - ) - raise + async def update_transcript_status(self, transcript_id: str, processed: bool) -> None: + await self._merge(transcript_id, {"processed": processed}) async def get_or_create_transcript(self, transcript: Transcript) -> str: - """Get existing transcript by source path or create new one. - - Args: - transcript: Transcript to get or create. - - Returns: - Transcript ID. - """ - if not self.db: - raise RuntimeError("Database not connected") - - try: - # Check if transcript exists by source path - existing = await self.get_transcript_by_source(transcript.source) - - if existing: - transcript_id = existing["id"] - logger.debug("Found existing transcript", transcript_id=transcript_id) - # Update content and metadata if needed - await self.db.update( - transcript_id, - { - "title": transcript.title, - "content": transcript.content, - "metadata": transcript.metadata, - }, - ) - return str(transcript_id) - else: - # Create new transcript - return await self.create_transcript(transcript) - - except Exception as e: - logger.error( - "Failed to get or create transcript", - source=transcript.source, - error=str(e), + existing = await self.get_transcript_by_source(transcript.source) + if existing: + transcript_id = str(existing["id"]) + # Do not rewrite historical date/source/created_at on a retry. + await self._merge( + transcript_id, + { + "title": transcript.title, + "content": transcript.content, + "metadata": transcript.metadata, + "processed": False, + }, ) - raise + return transcript_id + return await self.create_transcript(transcript) diff --git a/src/journal_utilities/rag/main.py b/src/journal_utilities/rag/main.py index 33cd179..1dba637 100644 --- a/src/journal_utilities/rag/main.py +++ b/src/journal_utilities/rag/main.py @@ -1,16 +1,18 @@ -"""Main processing pipeline for JournalRAG.""" +"""Select transcript extraction work explicitly; readiness alone is not a run.""" +import argparse import asyncio +import json import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from pathlib import Path import structlog -from journal_utilities.rag.adapters import EntityAdapter -from journal_utilities.rag.extractors import CohereExtractor -from journal_utilities.rag.graph import SurrealDBClient +from journal_utilities.rag.adapters.entity_adapter import EntityAdapter +from journal_utilities.rag.extractors.cohere_extractor import CohereExtractor +from journal_utilities.rag.graph.surreal_client import SurrealDBClient from journal_utilities.rag.models import Transcript from journal_utilities.rag.settings import settings from journal_utilities.rag.utils import setup_logging @@ -18,264 +20,167 @@ logger = structlog.get_logger(__name__) -def _date_from_filename(name: str) -> datetime: - """Extract a deterministic transcript date from a filename. - - Prefers an embedded ``YYYY[-_/. ]MM[-_/. ]DD``; otherwise falls back to a - fixed epoch (datetime.min) instead of ``datetime.now()`` so reprocessing - the same file is deterministic. - """ - m = re.search(r"(\d{4})[-_/. ](\d{1,2})[-_/. ](\d{1,2})", name) - if m: +def _date_from_filename(filename: str) -> datetime: + """Use a valid filename date, otherwise the deterministic unknown-date sentinel.""" + match = re.search(r"(\d{4})[-_/. ](\d{1,2})[-_/. ](\d{1,2})", filename) + if match: try: - return datetime(int(m.group(1)), int(m.group(2)), int(m.group(3))) + year, month, day = (int(part) for part in match.groups()) + return datetime(year, month, day) except ValueError: pass return datetime.min +def _load_transcript(path: Path, *, source: str) -> Transcript: + return Transcript( + title=path.stem, + date=_date_from_filename(path.name), + content=path.read_text(encoding="utf-8"), + source=source, + ) + + class JournalRAGPipeline: - """Main pipeline for extracting entities from transcripts and storing in graph database.""" + """A transcript is complete only after every required entity and edge persists.""" def __init__(self) -> None: - """Initialize the pipeline.""" setup_logging(debug=settings.debug) - self.extractor = CohereExtractor() + # Selection/migration/empty queues must not require or invoke a provider. + self.extractor: CohereExtractor | None = None self.db_client = SurrealDBClient() - logger.info("Initialized JournalRAG pipeline") async def connect(self) -> None: - """Connect to the database.""" await self.db_client.connect() async def disconnect(self) -> None: - """Disconnect from the database.""" await self.db_client.disconnect() async def process_transcript(self, transcript: Transcript) -> Mapping[str, int]: - """Process a single transcript. - - This method: - 1. Gets or creates the transcript record in the database - 2. Extracts entities using CoreEntities schema - 3. Converts CoreEntities to simple Entity model for storage - 4. Stores entities in the database - 5. Extracts relationships from CoreEntities structure - 6. Stores relationships in the database - 7. Updates transcript as processed - - Args: - transcript: The transcript to process. - - Returns: - Dictionary with processing statistics. - """ - logger.info("Processing transcript", title=transcript.title, source=transcript.source) - - # Get or create transcript record (idempotent) transcript_id = await self.db_client.get_or_create_transcript(transcript) - logger.info("Transcript record ready", transcript_id=transcript_id) - - # Extract entities using new CoreEntities schema - core_entities = self.extractor.extract_core_entities(transcript.content) - logger.info("Extracted core entities from transcript") - - # Convert CoreEntities to simple Entity model for storage - entities = EntityAdapter.convert_core_entities(core_entities) - logger.info("Converted to storage format", entity_count=len(entities)) - - # Store entities - entity_count = 0 + await self.db_client.update_transcript_status(transcript_id, processed=False) + if self.extractor is None: + self.extractor = CohereExtractor() + core = self.extractor.extract_core_entities(transcript.content) + entities = EntityAdapter.convert_core_entities(core, source=transcript.source) + relationships = EntityAdapter.extract_relationships(core, source=transcript.source) + identities = {(entity.type, entity.name) for entity in entities} + for relationship in relationships: + for identity in ( + (relationship.source_type, relationship.source), + (relationship.target_type, relationship.target), + ): + if identity not in identities: + raise ValueError(f"Required relationship endpoint missing: {identity!r}") + entity_ids = {} for entity in entities: - await self.db_client.create_entity(entity) - entity_count += 1 - - # Extract relationships from CoreEntities structure - relationships = EntityAdapter.extract_relationships(core_entities) - logger.info("Extracted relationships", relationship_count=len(relationships)) - - # Store relationships - relationship_count = 0 + entity_ids[(entity.type, entity.name)] = await self.db_client.create_entity(entity) for relationship in relationships: - try: - await self.db_client.create_relationship(relationship) - relationship_count += 1 - except Exception as e: - logger.warning( - "Failed to create relationship", - source=relationship.source, - target=relationship.target, - error=str(e), - ) - continue - - # Mark transcript as processed + # Returned IDs, not a fresh name-only search, choose the endpoints. + await self.db_client.create_relationship( + relationship, + source_id=entity_ids[(relationship.source_type, relationship.source)], + target_id=entity_ids[(relationship.target_type, relationship.target)], + ) await self.db_client.update_transcript_status(transcript_id, processed=True) - - logger.info( - "Completed processing transcript", - title=transcript.title, - entities=entity_count, - relationships=relationship_count, - ) - - return { - "entities": entity_count, - "relationships": relationship_count, - } + return {"entities": len(entity_ids), "relationships": len(relationships)} async def process_transcript_file(self, file_path: Path) -> Mapping[str, int]: - """Process a transcript from a file. - - Args: - file_path: Path to the transcript file. - - Returns: - Dictionary with processing statistics. - """ - logger.info("Processing transcript file", file_path=str(file_path)) - - # Read transcript content - content = file_path.read_text(encoding="utf-8") - - # Create transcript object (deterministic date from the filename) - transcript = Transcript( - title=file_path.stem, - date=_date_from_filename(file_path.name), - content=content, - source=str(file_path), - ) - - return await self.process_transcript(transcript) + return await self.process_transcript(_load_transcript(file_path, source=str(file_path))) async def process_from_database(self, base_path: Path | None = None) -> Mapping[str, int]: - """Process all unprocessed transcripts from the database. - - This orchestrator method: - 1. Queries SurrealDB for unprocessed transcript paths - 2. Loads each transcript from the local repository - 3. Processes each transcript (extract entities, store in DB) - 4. Handles errors gracefully to continue processing - - Args: - base_path: Optional base path for transcript files. If provided, paths from - the database will be resolved relative to this base. If None, paths - are assumed to be absolute. - - Returns: - Dictionary with processing statistics including: - - total: Total transcripts found in DB - - processed: Number successfully processed - - failed: Number that failed - - total_entities: Total entities extracted - - total_relationships: Total relationships extracted - """ - logger.info("Starting batch processing from database") - - # Get unprocessed transcript paths from database - transcript_paths = await self.db_client.get_unprocessed_transcript_paths() - logger.info("Found unprocessed transcripts", count=len(transcript_paths)) - - if not transcript_paths: - logger.info("No unprocessed transcripts found") - return { - "total": 0, - "processed": 0, - "failed": 0, - "total_entities": 0, - "total_relationships": 0, - } - - # Process each transcript - processed_count = 0 - failed_count = 0 - total_entities = 0 - total_relationships = 0 - - for source_path in transcript_paths: + paths = await self.db_client.get_unprocessed_transcript_paths() + totals = { + "total": len(paths), + "processed": 0, + "failed": 0, + "total_entities": 0, + "total_relationships": 0, + } + for source in paths: + path = base_path / source if base_path is not None else Path(source) try: - # Resolve file path - if base_path: - file_path = base_path / source_path - else: - file_path = Path(source_path) - - # Check if file exists - if not file_path.exists(): - logger.warning("Transcript file not found", path=str(file_path)) - failed_count += 1 - continue - - # Load transcript content - content = file_path.read_text(encoding="utf-8") - - # Create transcript object - transcript = Transcript( - title=file_path.stem, - date=datetime.now(), # You may want to extract date from filename - content=content, - source=source_path, - ) - - # Process transcript - stats = await self.process_transcript(transcript) - - # Update totals - processed_count += 1 - total_entities += stats["entities"] - total_relationships += stats["relationships"] - - logger.info( - "Processed transcript successfully", - path=source_path, - entities=stats["entities"], - relationships=stats["relationships"], - ) - - except Exception as e: + stats = await self.process_transcript(_load_transcript(path, source=source)) + totals["processed"] += 1 + totals["total_entities"] += stats["entities"] + totals["total_relationships"] += stats["relationships"] + except Exception as exc: + totals["failed"] += 1 logger.error( - "Failed to process transcript", - path=source_path, - error=str(e), - exc_info=True, + "Transcript incomplete; remains pending", source=source, error=str(exc) ) - failed_count += 1 - continue - - logger.info( - "Batch processing complete", - total=len(transcript_paths), - processed=processed_count, - failed=failed_count, - total_entities=total_entities, - total_relationships=total_relationships, - ) - - return { - "total": len(transcript_paths), - "processed": processed_count, - "failed": failed_count, - "total_entities": total_entities, - "total_relationships": total_relationships, - } - - -async def main() -> None: - """Main entry point.""" + return totals + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument( + "--file", nargs="+", type=Path, dest="files", help="Extract these transcript files" + ) + selection.add_argument( + "--pending", action="store_true", help="Extract pending database source paths" + ) + selection.add_argument( + "--migrate-source-identity", + action="store_true", + help="Explicit schema migration; quiescent database and backup required; no attribution backfill", + ) + parser.add_argument("--base-path", type=Path, help="Resolve pending relative source paths here") + return parser + + +async def _run(args: argparse.Namespace) -> int: + if args.migrate_source_identity: + client = SurrealDBClient() + try: + await client.connect(migrate_source_identity=True) + print(json.dumps({"migration": client.migration_summary})) + return 0 + except Exception as exc: + logger.error("Source identity migration refused or failed", error=str(exc)) + return 1 + finally: + await client.disconnect() pipeline = JournalRAGPipeline() - try: await pipeline.connect() - logger.info("Pipeline ready") - - # Example: Process a transcript file - # transcript_path = Path("path/to/transcript.txt") - # stats = await pipeline.process_transcript_file(transcript_path) - # logger.info("Processing complete", stats=stats) - + stats: dict[str, int] + if args.pending: + stats = dict(await pipeline.process_from_database(base_path=args.base_path)) + else: + stats = { + "total": len(args.files), + "processed": 0, + "failed": 0, + "total_entities": 0, + "total_relationships": 0, + } + for path in args.files: + try: + result = await pipeline.process_transcript_file(path) + stats["processed"] += 1 + stats["total_entities"] += result["entities"] + stats["total_relationships"] += result["relationships"] + except Exception as exc: + stats["failed"] += 1 + logger.error("Transcript incomplete", source=str(path), error=str(exc)) + print(json.dumps(stats)) + return 1 if stats["failed"] else 0 + except Exception as exc: + logger.error("Extraction run failed", error=str(exc)) + return 1 finally: await pipeline.disconnect() +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if args.base_path is not None and not args.pending: + parser.error("--base-path is only valid with --pending") + return asyncio.run(_run(args)) + + if __name__ == "__main__": - asyncio.run(main()) + raise SystemExit(main()) diff --git a/src/journal_utilities/rag/models/entities.py b/src/journal_utilities/rag/models/entities.py index 9069cb4..b95f660 100644 --- a/src/journal_utilities/rag/models/entities.py +++ b/src/journal_utilities/rag/models/entities.py @@ -268,6 +268,7 @@ class Transcript(BaseModel): class Entity(BaseModel): """Simple generic entity for database storage.""" + source: str = Field(..., min_length=1, pattern=r"\S", description="Trusted transcript source") name: str = Field(..., description="Entity name") type: str = Field(..., description="Entity type (concept, researcher, citation, etc.)") description: str = Field(default="", description="Description or definition") @@ -280,6 +281,11 @@ class Entity(BaseModel): class Relationship(BaseModel): """Simple generic relationship for database storage.""" + transcript_source: str = Field( + ..., min_length=1, pattern=r"\S", description="Trusted transcript source" + ) + source_type: str = Field(..., description="Source entity type") + target_type: str = Field(..., description="Target entity type") source: str = Field(..., description="Source entity name") target: str = Field(..., description="Target entity name") relationship_type: str = Field(..., description="Type of relationship") diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 4748f3f..496359b 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -2,9 +2,10 @@ ## Test Suite Overview -The suite is a live snapshot (currently ~475 passing, 3 API-gated skips, snapshot -of `uv run pytest tests/ -q` on 2026-08-02); run -`uv run pytest tests/ -q` for the current count. +Counts and coverage are generated measurements, not maintained prose. See +[`../docs/_generated/gate-arms.md`](../docs/_generated/gate-arms.md) for the +recorded source, runtime, results and limitations. Regenerate only with the +documented gate prerequisites available; historical receipts are not current proof. ## Running Tests @@ -60,6 +61,12 @@ async def test_real_extraction(): # ... ``` +SDK doubles must match the installed SDK's actual return shapes. They cannot +qualify database schema or persistence behavior. The disposable-database +acceptance in `journal_utilities/rag/test_source_identity.py` requires explicit +service admission and checks migration, source isolation, failure recovery and +replay against a real server. A skipped database case is not qualification. + ## Adding Tests 1. Pipeline tests: `tests/journal_utilities/test_*.py` diff --git a/tests/journal_utilities/rag/integration/test_pipeline_integration.py b/tests/journal_utilities/rag/integration/test_pipeline_integration.py index ad88127..224c6ca 100644 --- a/tests/journal_utilities/rag/integration/test_pipeline_integration.py +++ b/tests/journal_utilities/rag/integration/test_pipeline_integration.py @@ -10,7 +10,7 @@ import os from datetime import datetime from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, call import pytest @@ -27,7 +27,7 @@ @pytest.fixture def sample_transcript_file() -> Path: """Get path to sample transcript file.""" - return Path(__file__).parent.parent.parent / "fixtures" / "sample_transcript.txt" + return Path(__file__).parent.parent.parent.parent / "fixtures" / "sample_transcript.txt" @pytest.fixture @@ -48,7 +48,7 @@ def mock_db_client(): """Mock database client to avoid needing SurrealDB.""" mock = AsyncMock() mock.get_or_create_transcript.return_value = "transcript:test-001" - mock.create_entity.return_value = "entity:test-001" + mock.create_entity.side_effect = lambda entity: f"entity:{entity.type}-{entity.name}" mock.create_relationship.return_value = "relates:test-001" mock.update_transcript_status.return_value = None return mock @@ -91,8 +91,11 @@ async def test_process_transcript_with_real_extractor( # Verify database interactions occurred mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] ) # Verify stats from real extraction diff --git a/tests/journal_utilities/rag/test_source_identity.py b/tests/journal_utilities/rag/test_source_identity.py new file mode 100644 index 0000000..f124d20 --- /dev/null +++ b/tests/journal_utilities/rag/test_source_identity.py @@ -0,0 +1,221 @@ +"""Opt-in real SurrealDB schema acceptance. No Cohere requests. + +Requires an existing disposable service, explicit JOURNAL_TEST_SURREALDB_ALLOW_DISPOSABLE=1, +JOURNAL_TEST_SURREALDB_URL, JOURNAL_TEST_SURREALDB_NAMESPACE, USERNAME and PASSWORD. +Creates/removes only a fresh UUID-named database in that namespace. A skip is not proof. +""" + +import os +from datetime import UTC, datetime +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest + +pytest.importorskip("surrealdb") +pytest.importorskip("structlog") +pytest.importorskip("cohere") + +from surrealdb import AsyncSurreal, RecordID + +from journal_utilities.rag.graph.surreal_client import SurrealDBClient +from journal_utilities.rag.main import JournalRAGPipeline +from journal_utilities.rag.models import ( + Concept, + ConceptCategory, + CoreEntities, + Relationship, + Transcript, +) + + +@pytest.fixture +async def disposable_database(): + prefix = "JOURNAL_TEST_SURREALDB_" + required = ("URL", "NAMESPACE", "USERNAME", "PASSWORD") + if os.environ.get(prefix + "ALLOW_DISPOSABLE") != "1" or not all( + os.environ.get(prefix + name) for name in required + ): + pytest.skip( + "Explicit disposable existing SurrealDB service credentials required; no DB proof" + ) + options = {name.lower(): os.environ[prefix + name] for name in required} + options["database"] = "journal_group07_test_" + uuid4().hex + raw = AsyncSurreal(options["url"]) + await raw.connect() + try: + await raw.signin({"username": options["username"], "password": options["password"]}) + await raw.use(options["namespace"], options["database"]) + yield options, raw + finally: + # Generated identifier contains only this literal prefix and UUID hex. + await raw.query(f"REMOVE DATABASE IF EXISTS {options['database']};") + await raw.close() + + +async def test_legacy_migration_source_identity_retry_and_dates(disposable_database, monkeypatch): + options, raw = disposable_database + # Historical schema: global name/type uniqueness, no source field on graph records. + await raw.query(""" + DEFINE TABLE entity SCHEMAFULL; + DEFINE FIELD name ON entity TYPE string; + DEFINE FIELD type ON entity TYPE string; + DEFINE FIELD description ON entity TYPE option; + DEFINE FIELD confidence ON entity TYPE number; + DEFINE FIELD mentions ON entity TYPE number; + DEFINE FIELD context ON entity TYPE array; + DEFINE FIELD metadata ON entity TYPE object; + DEFINE FIELD created_at ON entity TYPE datetime; + DEFINE INDEX entity_name_type ON entity FIELDS name, type UNIQUE; + DEFINE TABLE relates SCHEMAFULL TYPE RELATION; + DEFINE FIELD relationship_type ON relates TYPE string; + DEFINE FIELD description ON relates TYPE option; + DEFINE FIELD confidence ON relates TYPE number; + DEFINE FIELD context ON relates TYPE option; + DEFINE FIELD metadata ON relates TYPE object; + DEFINE FIELD created_at ON relates TYPE datetime; + """) + payload = { + "description": "Legacy", + "confidence": 1.0, + "mentions": 1, + "context": [], + "metadata": {}, + "created_at": datetime(2000, 1, 1, tzinfo=UTC), + } + await raw.create( + RecordID("entity", "legacy_insight"), dict(payload, name="Insight 1", type="key_insight") + ) + await raw.create( + RecordID("entity", "legacy_concept"), dict(payload, name="Shared", type="concept") + ) + await raw.relate( + RecordID("entity", "legacy_concept"), + RecordID("relates", "legacy"), + RecordID("entity", "legacy_insight"), + { + "relationship_type": "related_to", + "description": "Legacy", + "confidence": 1.0, + "context": "", + "metadata": {}, + "created_at": datetime(2000, 1, 1, tzinfo=UTC), + }, + ) + legacy_entities = (await raw.query("SELECT * FROM entity ORDER BY id;"))[0] + legacy_edges = (await raw.query("SELECT * FROM relates ORDER BY id;"))[0] + refused = SurrealDBClient(**options) + with pytest.raises(RuntimeError, match="--migrate-source-identity"): + await refused.connect() + client = SurrealDBClient(**options) + await client.connect(migrate_source_identity=True) + try: + assert client.migration_summary["unknown_source_entities"] == 2 + assert client.migration_summary["unknown_source_relationships"] == 1 + assert (await raw.query("SELECT * FROM entity ORDER BY id;"))[0] == legacy_entities + assert (await raw.query("SELECT * FROM relates ORDER BY id;"))[0] == legacy_edges + info = (await raw.query("INFO FOR TABLE entity;"))[0] + assert "entity_name_type" not in info["indexes"] + assert "entity_source_type_name" in info["indexes"] + core = CoreEntities( + concepts=[ + Concept( + name="Shared", + definition="A", + category=ConceptCategory.THEORETICAL, + related_concepts=["Target"], + ), + Concept( + name="Target", + definition="B", + category=ConceptCategory.THEORETICAL, + related_concepts=["Shared"], + ), + ], + researchers=[], + citations=[], + technical_terms=[], + key_insights=["An insight"], + ) + pipeline = JournalRAGPipeline() + pipeline.db_client = client + pipeline.extractor = MagicMock() + pipeline.extractor.extract_core_entities.return_value = core + a = Transcript(title="A", date=datetime.min, source="a.txt", content="A") + b = Transcript(title="B", date=datetime(2025, 10, 30), source="b.txt", content="B") + await pipeline.process_transcript(a) + relate = client.db.relate + calls = 0 + + def fail_second_edge(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("Injected external edge write failure") + return relate(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(client.db, "relate", fail_second_edge) + with pytest.raises(RuntimeError, match="edge write failure"): + await pipeline.process_transcript(b) + pending = await client.get_transcript_by_source("b.txt") + assert pending["processed"] is False and pending["content"] == "B" + assert pending["source"] == "b.txt" + original_date = pending["date"] + await pipeline.process_transcript(b.model_copy(update={"date": datetime.min})) + assert (await client.get_transcript_by_source("b.txt"))["date"] == original_date + before_replay = await client.query("SELECT * FROM entity;") + edge_ids = {str(row["id"]) for row in await client.query("SELECT * FROM relates;")} + await pipeline.process_transcript(a) + await pipeline.process_transcript(b) + entities = await client.query("SELECT * FROM entity;") + edges = await client.query("SELECT * FROM relates;") + assert {str(row["id"]) for row in entities} == {str(row["id"]) for row in before_replay} + assert {str(row["id"]) for row in edges} == edge_ids + assert len(entities) == 8 and len(edges) == 5 + by_id = {str(row["id"]): row for row in entities} + for edge in edges: + if edge.get("source"): + assert ( + by_id[str(edge["in"])]["source"] + == edge["source"] + == by_id[str(edge["out"])]["source"] + ) + assert ( + sorted( + (row for row in entities if not row.get("source")), key=lambda row: str(row["id"]) + ) + == legacy_entities + ) + assert [row for row in edges if not row.get("source")] == legacy_edges + assert { + row["source"] for row in entities if row["name"] == "Insight 1" and row.get("source") + } == {"a.txt", "b.txt"} + assert (await client.get_transcript_by_source("a.txt"))["processed"] is True + assert (await client.get_transcript_by_source("b.txt"))["processed"] is True + a_entity = await client.get_entity_by_name("Shared", entity_type="concept", source="a.txt") + b_entity = await client.get_entity_by_name("Target", entity_type="concept", source="b.txt") + edge = Relationship( + transcript_source="a.txt", + source="Shared", + source_type="concept", + target="Target", + target_type="concept", + relationship_type="related_to", + ) + with pytest.raises(ValueError, match="trusted source/type/name"): + await client.create_relationship( + edge, source_id=str(a_entity["id"]), target_id=str(b_entity["id"]) + ) + # Explicit migration must refuse contradictory claims without changing data. + await raw.query( + "UPDATE $id MERGE { metadata: { source: 'contradictory.txt' } };", + {"id": a_entity["id"]}, + ) + snapshot = await client.query("SELECT * FROM entity ORDER BY id;") + ambiguous = SurrealDBClient(**options) + with pytest.raises(RuntimeError, match="Conflicting legacy source claims"): + await ambiguous.connect(migrate_source_identity=True) + assert await client.query("SELECT * FROM entity ORDER BY id;") == snapshot + finally: + await client.disconnect() diff --git a/tests/journal_utilities/rag/unit/test_entity_adapter.py b/tests/journal_utilities/rag/unit/test_entity_adapter.py index a32ab1e..a9dbdae 100644 --- a/tests/journal_utilities/rag/unit/test_entity_adapter.py +++ b/tests/journal_utilities/rag/unit/test_entity_adapter.py @@ -2,6 +2,9 @@ Unit tests for EntityAdapter conversion functions. """ +import pytest +from pydantic import ValidationError + from journal_utilities.rag.adapters.entity_adapter import EntityAdapter from journal_utilities.rag.models import Entity from journal_utilities.rag.models.entities import ( @@ -26,7 +29,7 @@ def test_convert_empty_entities(self): concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=[] ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) == 0 @@ -47,7 +50,7 @@ def test_convert_concepts(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 concept_entity = next((e for e in entities if e.name == "Active Inference"), None) @@ -66,7 +69,7 @@ def test_convert_researchers(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 researcher_entity = next((e for e in entities if e.name == "Karl Friston"), None) @@ -90,7 +93,7 @@ def test_convert_citations(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 citation_entity = next((e for e in entities if "Free Energy Principle" in e.name), None) @@ -113,7 +116,7 @@ def test_convert_technical_terms(self): key_insights=[], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 term_entity = next((e for e in entities if e.name == "Variational Free Energy"), None) @@ -130,7 +133,7 @@ def test_convert_key_insights(self): key_insights=["Active Inference unifies perception and action"], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") assert len(entities) >= 1 # The adapter names insights as "Insight 1", "Insight 2", etc. @@ -156,7 +159,7 @@ def test_convert_all_types(self): key_insights=["Insight1"], ) - entities = EntityAdapter.convert_core_entities(core) + entities = EntityAdapter.convert_core_entities(core, source="transcripts/a.txt") # Should have at least 5 entities (one of each type) assert len(entities) >= 5 @@ -178,7 +181,7 @@ def test_extract_relationships_empty(self): concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=[] ) - relationships = EntityAdapter.extract_relationships(core) + relationships = EntityAdapter.extract_relationships(core, source="transcripts/a.txt") assert len(relationships) == 0 @@ -199,7 +202,7 @@ def test_extract_relationships_from_concepts(self): key_insights=[], ) - relationships = EntityAdapter.extract_relationships(core) + relationships = EntityAdapter.extract_relationships(core, source="transcripts/a.txt") # Should have 2 relationships from related_concepts assert len(relationships) >= 2 @@ -211,6 +214,7 @@ class TestEntityModel: def test_entity_creation(self): """Test basic entity creation.""" entity = Entity( + source="transcripts/a.txt", name="Test Entity", type="concept", description="A test", @@ -225,16 +229,15 @@ def test_entity_creation(self): assert entity.mentions == 3 assert len(entity.context) == 2 - def test_entity_default_values(self): - """Test entity has sensible defaults.""" - entity = Entity(name="Minimal", type="concept", confidence=0.8, mentions=1, context=[]) - - assert entity.description == "" - assert entity.metadata == {} + @pytest.mark.parametrize("source", [None, "", " "]) + def test_entity_requires_trusted_source(self, source): + with pytest.raises(ValidationError): + Entity(name="Insight 1", type="key_insight", source=source) def test_entity_with_metadata(self): """Test entity with custom metadata.""" entity = Entity( + source="transcripts/a.txt", name="Test", type="concept", confidence=0.9, @@ -245,3 +248,17 @@ def test_entity_with_metadata(self): assert entity.metadata["source"] == "transcript" assert entity.metadata["importance"] == "high" + + +def test_caller_source_overrides_extraction_metadata(sample_core_entities): + from journal_utilities.rag.models import ExtractedEntities + + extracted = ExtractedEntities(core=sample_core_entities, metadata={"source": "untrusted"}) + entities, relationships = EntityAdapter.convert_extracted_entities( + extracted, source="trusted.txt" + ) + assert {entity.source for entity in entities} == {"trusted.txt"} + assert {relationship.transcript_source for relationship in relationships} == {"trusted.txt"} + assert { + (relationship.source_type, relationship.target_type) for relationship in relationships + } == {("researcher", "citation")} diff --git a/tests/journal_utilities/rag/unit/test_pipeline.py b/tests/journal_utilities/rag/unit/test_pipeline.py index f0186e9..06e586d 100644 --- a/tests/journal_utilities/rag/unit/test_pipeline.py +++ b/tests/journal_utilities/rag/unit/test_pipeline.py @@ -6,7 +6,7 @@ from datetime import datetime from pathlib import Path -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, call import pytest @@ -15,7 +15,7 @@ pytest.importorskip("cohere") pytest.importorskip("surrealdb") -from journal_utilities.rag.main import JournalRAGPipeline +from journal_utilities.rag.main import JournalRAGPipeline, main from journal_utilities.rag.models import CoreEntities, Transcript @@ -43,7 +43,9 @@ def mock_db_client(): """Mock database client.""" mock = AsyncMock() mock.get_or_create_transcript.return_value = "transcript:test-001" - mock.create_entity.return_value = "entity:test-001" + mock.create_entity.side_effect = lambda entity: ( + f"entity:{entity.source}-{entity.type}-{entity.name}" + ) mock.create_relationship.return_value = "relates:test-001" mock.update_transcript_status.return_value = None return mock @@ -83,8 +85,11 @@ async def test_process_transcript_with_file( # Verify database interactions mock_db_client.get_or_create_transcript.assert_called_once() assert mock_db_client.create_entity.call_count > 0 - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] ) # Verify extractor was called @@ -135,7 +140,7 @@ async def test_process_transcript_handles_relationship_errors( mock_db_client, mock_extractor, ) -> None: - """Test that process_transcript handles relationship creation errors gracefully.""" + """A required edge failure is visible, stays pending, and can be retried.""" # Create pipeline with mocked dependencies pipeline = JournalRAGPipeline() pipeline.db_client = mock_db_client @@ -144,14 +149,17 @@ async def test_process_transcript_handles_relationship_errors( # Make relationship creation fail - the sample fixture only generates 1 relationship mock_db_client.create_relationship.side_effect = ValueError("Entity not found") - # Process the transcript - should not raise exception + with pytest.raises(ValueError, match="Entity not found"): + await pipeline.process_transcript(sample_transcript) + assert mock_db_client.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] + mock_db_client.create_relationship.side_effect = None stats = await pipeline.process_transcript(sample_transcript) - - # Should have processed all entities but failed on relationships - assert "entities" in stats - assert "relationships" in stats - # Should have 0 successful relationships since we made it fail - assert stats["relationships"] == 0 + assert stats["relationships"] == 1 + assert mock_db_client.update_transcript_status.await_args == call( + "transcript:test-001", processed=True + ) @pytest.mark.asyncio @@ -171,8 +179,11 @@ async def test_process_transcript_updates_transcript_status( await pipeline.process_transcript(sample_transcript) # Verify transcript was marked as processed - mock_db_client.update_transcript_status.assert_called_once_with( - "transcript:test-001", processed=True + mock_db_client.update_transcript_status.assert_has_awaits( + [ + call("transcript:test-001", processed=False), + call("transcript:test-001", processed=True), + ] ) @@ -234,3 +245,174 @@ async def test_process_transcript_converts_entities_correctly( entity_names = {call[0][0].name for call in entity_calls} assert "Active Inference" in entity_names assert "Karl Friston" in entity_names + + +@pytest.mark.parametrize( + "filename, expected", + [ + ("talk_2025-10-30.txt", datetime(2025, 10, 30)), + ("talk_2025.10.30", datetime(2025, 10, 30)), + ("talk_2025.10.30.txt", datetime(2025, 10, 30)), + ("undated.txt", datetime.min), + ("talk_2025-02-30.txt", datetime.min), + ], +) +async def test_file_and_pending_loaders_share_date_and_source( + tmp_path, filename, expected, mock_db_client, mock_extractor +): + path = tmp_path / filename + path.write_text("Active Inference", encoding="utf-8") + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + await pipeline.process_transcript_file(path) + direct = mock_db_client.get_or_create_transcript.await_args.args[0] + mock_db_client.get_unprocessed_transcript_paths.return_value = [filename] + stats = await pipeline.process_from_database(base_path=tmp_path) + pending = mock_db_client.get_or_create_transcript.await_args.args[0] + assert stats["processed"] == 1 and stats["failed"] == 0 + assert direct.date == pending.date == expected + assert direct.source == str(path) + assert pending.source == filename + + +async def test_required_missing_endpoint_is_not_discarded( + sample_transcript, sample_core_entities, mock_db_client, mock_extractor +): + sample_core_entities.concepts[0].related_concepts = ["Not extracted"] + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + with pytest.raises(ValueError, match="Required relationship endpoint missing"): + await pipeline.process_transcript(sample_transcript) + assert mock_db_client.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] + mock_db_client.create_relationship.assert_not_awaited() + + +async def test_source_and_type_choose_returned_endpoint_ids( + sample_transcript, mock_db_client, mock_extractor +): + from journal_utilities.rag.models import ( + Citation, + CitationType, + Concept, + ConceptCategory, + Researcher, + ResearcherRole, + ) + + mock_extractor.extract_core_entities.return_value = CoreEntities( + concepts=[ + Concept( + name="Same", + definition="Concept", + category=ConceptCategory.THEORETICAL, + related_concepts=["Same"], + ) + ], + researchers=[Researcher(name="Same", role=ResearcherRole.SPEAKER)], + citations=[Citation(title="Same", type=CitationType.PAPER, authors=["Same"])], + technical_terms=[], + key_insights=["One"], + ) + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + for source in ("a.txt", "b.txt"): + await pipeline.process_transcript(sample_transcript.model_copy(update={"source": source})) + edges = mock_db_client.create_relationship.await_args_list + assert len(edges) == 4 + for invocation in edges: + edge = invocation.args[0] + assert invocation.kwargs == { + "source_id": f"entity:{edge.transcript_source}-{edge.source_type}-{edge.source}", + "target_id": f"entity:{edge.transcript_source}-{edge.target_type}-{edge.target}", + } + assert { + invocation.args[0].source for invocation in mock_db_client.create_entity.await_args_list + } == {"a.txt", "b.txt"} + + +@pytest.fixture +def cli_boundaries(monkeypatch, mock_db_client, mock_extractor): + monkeypatch.setattr("journal_utilities.rag.main.SurrealDBClient", lambda: mock_db_client) + monkeypatch.setattr("journal_utilities.rag.main.CohereExtractor", lambda: mock_extractor) + return mock_db_client, mock_extractor + + +def test_cli_requires_work_selection(cli_boundaries): + db, extractor = cli_boundaries + with pytest.raises(SystemExit) as exc: + main([]) + assert exc.value.code == 2 + db.connect.assert_not_awaited() + extractor.extract_core_entities.assert_not_called() + + +def test_cli_files_perform_work_and_report_missing_file(tmp_path, cli_boundaries): + db, extractor = cli_boundaries + path = tmp_path / "2025-10-30.txt" + path.write_text("Active Inference", encoding="utf-8") + assert main(["--file", str(path), str(tmp_path / "missing.txt")]) == 1 + extractor.extract_core_entities.assert_called_once_with("Active Inference") + assert db.update_transcript_status.await_args == call("transcript:test-001", processed=True) + db.disconnect.assert_awaited_once() + + +def test_cli_pending_edge_failure_returns_failure_and_disconnects(tmp_path, cli_boundaries): + db, extractor = cli_boundaries + (tmp_path / "pending.txt").write_text("Pending", encoding="utf-8") + db.get_unprocessed_transcript_paths.return_value = ["pending.txt"] + db.create_relationship.side_effect = RuntimeError("edge write failed") + assert main(["--pending", "--base-path", str(tmp_path)]) == 1 + extractor.extract_core_entities.assert_called_once_with("Pending") + assert db.update_transcript_status.await_args_list == [ + call("transcript:test-001", processed=False) + ] + db.disconnect.assert_awaited_once() + + +def test_cli_empty_pending_does_not_invoke_provider(cli_boundaries): + db, extractor = cli_boundaries + db.get_unprocessed_transcript_paths.return_value = [] + assert main(["--pending"]) == 0 + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +def test_cli_connect_failure_still_disconnects(cli_boundaries): + db, extractor = cli_boundaries + db.connect.side_effect = RuntimeError("offline") + assert main(["--pending"]) == 1 + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +def test_cli_explicit_migration_does_not_extract(cli_boundaries): + db, extractor = cli_boundaries + db.migration_summary = {"unknown_source_entities": 2} + assert main(["--migrate-source-identity"]) == 0 + db.connect.assert_awaited_once_with(migrate_source_identity=True) + extractor.extract_core_entities.assert_not_called() + db.disconnect.assert_awaited_once() + + +async def test_no_required_relationships_can_complete( + sample_transcript, mock_db_client, mock_extractor +): + mock_extractor.extract_core_entities.return_value = CoreEntities( + concepts=[], researchers=[], citations=[], technical_terms=[], key_insights=["An insight"] + ) + pipeline = JournalRAGPipeline() + pipeline.db_client = mock_db_client + pipeline.extractor = mock_extractor + assert await pipeline.process_transcript(sample_transcript) == { + "entities": 1, + "relationships": 0, + } + mock_db_client.create_relationship.assert_not_awaited() + assert mock_db_client.update_transcript_status.await_args == call( + "transcript:test-001", processed=True + ) diff --git a/tests/journal_utilities/rag/unit/test_surreal_client.py b/tests/journal_utilities/rag/unit/test_surreal_client.py index 28cf8df..85ae57e 100644 --- a/tests/journal_utilities/rag/unit/test_surreal_client.py +++ b/tests/journal_utilities/rag/unit/test_surreal_client.py @@ -1,13 +1,10 @@ -""" -Unit tests for SurrealDBClient with mocks. -""" +"""Storage admission/error contracts at the SDK boundary; not real-schema proof.""" -from datetime import UTC, datetime -from unittest.mock import AsyncMock, patch +from datetime import datetime +from unittest.mock import AsyncMock import pytest -# Requires the rag extra; skip cleanly when it is not installed. pytest.importorskip("surrealdb") pytest.importorskip("structlog") @@ -15,304 +12,150 @@ from journal_utilities.rag.models import Entity, Relationship, Transcript -class TestSurrealDBClientInit: - """Tests for SurrealDBClient initialization.""" - - def test_default_initialization(self): - """Test client initializes with default settings.""" - with patch("journal_utilities.rag.graph.surreal_client.settings") as mock_settings: - mock_settings.surrealdb_url = "ws://localhost:8080" - mock_settings.surrealdb_namespace = "test" - mock_settings.surrealdb_database = "test" - mock_settings.surrealdb_username = "root" - mock_settings.surrealdb_password = "root" - - client = SurrealDBClient() - - assert client.url == "ws://localhost:8080" - assert client.namespace == "test" - assert client.database == "test" - - def test_custom_initialization(self): - """Test client with custom parameters.""" - client = SurrealDBClient( - url="ws://custom:9000", - namespace="custom_ns", - database="custom_db", - username="user", - password="pass", - ) - - assert client.url == "ws://custom:9000" - assert client.namespace == "custom_ns" - assert client.database == "custom_db" - - -class TestSurrealDBClientConnection: - """Tests for connection management.""" - - @pytest.fixture - def mock_surreal(self): - """Create a mock Surreal instance.""" - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.close = AsyncMock() - mock.query = AsyncMock(return_value=[]) - mock.create = AsyncMock(return_value=[{"id": "test:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.mark.asyncio - async def test_connect_success(self, mock_surreal): - """Test successful connection.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - - await client.connect() - - mock_surreal.connect.assert_called_once() - mock_surreal.signin.assert_called_once_with({"user": "user", "pass": "pass"}) - mock_surreal.use.assert_called_once_with("ns", "db") - - @pytest.mark.asyncio - async def test_disconnect(self, mock_surreal): - """Test disconnection.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - - await client.connect() - await client.disconnect() - - mock_surreal.close.assert_called_once() - - -class TestSurrealDBClientEntities: - """Tests for entity operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[]]) - mock.create = AsyncMock(return_value=[{"id": "entity:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.fixture - def sample_entity(self): - return Entity( - name="Test Entity", - type="concept", - description="A test entity", - confidence=0.9, - mentions=5, - context=["test context"], - ) - - @pytest.mark.asyncio - async def test_create_new_entity(self, mock_surreal, sample_entity): - """Test creating a new entity.""" - mock_surreal.query = AsyncMock(return_value=[[]]) # No existing - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_entity(sample_entity) - - assert result == "entity:1" - mock_surreal.create.assert_called_once() - - @pytest.mark.asyncio - async def test_update_existing_entity(self, mock_surreal, sample_entity): - """Test updating an existing entity.""" - mock_surreal.query = AsyncMock( - return_value=[ - [ - { - "id": "entity:existing", - "confidence": 0.8, - "context": ["old context"], - "description": "Old description", - } - ] - ] - ) - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_entity(sample_entity) - - assert result == "entity:existing" - mock_surreal.update.assert_called_once() - - -class TestSurrealDBClientTranscripts: - """Tests for transcript operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[]]) - mock.create = AsyncMock(return_value=[{"id": "transcript:1"}]) - mock.update = AsyncMock() - return mock - - @pytest.fixture - def sample_transcript(self): - return Transcript( - title="Test Transcript", - date=datetime.now(UTC), - content="Test content", - source="/path/to/file.txt", - ) - - @pytest.mark.asyncio - async def test_create_transcript(self, mock_surreal, sample_transcript): - """Test creating a transcript.""" - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - result = await client.create_transcript(sample_transcript) - - assert result == "transcript:1" - mock_surreal.create.assert_called_once() - - @pytest.mark.asyncio - async def test_get_unprocessed_transcripts(self, mock_surreal): - """Test getting unprocessed transcript paths.""" - mock_surreal.query = AsyncMock( - return_value=[[{"source": "/path/1.txt"}, {"source": "/path/2.txt"}]] - ) - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - paths = await client.get_unprocessed_transcript_paths() - - assert len(paths) == 2 - assert "/path/1.txt" in paths - assert "/path/2.txt" in paths - - -class TestSurrealDBClientRelationships: - """Tests for relationship operations.""" - - @pytest.fixture - def mock_surreal(self): - mock = AsyncMock() - mock.connect = AsyncMock() - mock.signin = AsyncMock() - mock.use = AsyncMock() - mock.query = AsyncMock(return_value=[[{"id": "entity:1"}]]) - return mock - - @pytest.fixture - def sample_relationship(self): - return Relationship( - source="Entity A", target="Entity B", relationship_type="relates_to", confidence=0.85 - ) - - @pytest.mark.asyncio - async def test_create_relationship_entities_not_found(self, mock_surreal, sample_relationship): - """Test creating relationship when entities not found.""" - mock_surreal.query = AsyncMock(return_value=[[]]) # No entities found - - with patch( - "journal_utilities.rag.graph.surreal_client.AsyncSurreal", return_value=mock_surreal - ): - client = SurrealDBClient( - url="ws://test:8080", - namespace="ns", - database="db", - username="user", - password="pass", - ) - await client.connect() - - with pytest.raises(ValueError, match="Source or target entity not found"): - await client.create_relationship(sample_relationship) - - -class TestSurrealDBClientErrors: - """Tests for error handling.""" - - def test_query_without_connection(self): - """Test that queries fail without connection.""" - client = SurrealDBClient( - url="ws://test:8080", namespace="ns", database="db", username="user", password="pass" - ) - - # db is None until connect() is called - assert client.db is None - - @pytest.mark.asyncio - async def test_create_entity_not_connected(self): - """Test create_entity raises error when not connected.""" - client = SurrealDBClient( - url="ws://test:8080", namespace="ns", database="db", username="user", password="pass" - ) - - entity = Entity(name="Test", type="concept", confidence=0.9, mentions=1, context=["test"]) - - with pytest.raises(RuntimeError, match="Database not connected"): - await client.create_entity(entity) +@pytest.fixture +def connected(): + client = SurrealDBClient() + client.db = AsyncMock() + client.db.query.return_value = [[]] + return client, client.db + + +async def test_ambiguous_entity_does_not_select_first_or_write(connected): + client, db = connected + db.query.return_value = [[{"id": "entity:a"}, {"id": "entity:b"}]] + with pytest.raises(RuntimeError, match="Ambiguous"): + await client.create_entity(Entity(source="a.txt", type="concept", name="Shared")) + db.create.assert_not_awaited() + + +async def test_cross_source_or_wrong_type_endpoints_are_refused(connected): + client, db = connected + db.query.return_value = [ + [{"id": "entity:wrong", "source": "other.txt", "type": "concept", "name": "Shared"}] + ] + edge = Relationship( + transcript_source="a.txt", + source="Shared", + target="Target", + source_type="concept", + target_type="concept", + relationship_type="related_to", + ) + with pytest.raises(ValueError, match="trusted source/type/name"): + await client.create_relationship(edge, source_id="entity:wrong", target_id="entity:target") + db.relate.assert_not_awaited() + + +async def test_transcript_update_does_not_replace_date_or_identity(connected): + client, db = connected + db.query.side_effect = [ + [[{"id": "transcript:old", "source": "a.txt", "date": datetime(2000, 1, 1)}]], + [[{"id": "transcript:old"}]], + ] + transcript = Transcript(title="Updated", date=datetime.min, source="a.txt", content="New text") + assert await client.get_or_create_transcript(transcript) == "transcript:old" + changes = db.query.await_args.args[1]["changes"] + assert changes == { + "title": "Updated", + "content": "New text", + "metadata": {}, + "processed": False, + } + db.create.assert_not_awaited() + + +async def test_disappeared_status_target_is_not_reported_complete(connected): + client, _ = connected + with pytest.raises(RuntimeError, match="disappeared"): + await client.update_transcript_status("transcript:gone", processed=True) + + +async def test_pending_without_source_is_not_silently_skipped(connected): + client, db = connected + db.query.return_value = [[{"source": None}]] + with pytest.raises(RuntimeError, match="no trusted source"): + await client.get_unprocessed_transcript_paths() + + +async def test_legacy_schema_requires_explicit_admission(monkeypatch): + db = AsyncMock() + db.query.side_effect = [ + [{"tables": {"entity": "SCHEMAFULL"}}], + [{"indexes": {"entity_name_type": "name,type UNIQUE"}}], + ] + monkeypatch.setattr("journal_utilities.rag.graph.surreal_client.AsyncSurreal", lambda _: db) + client = SurrealDBClient() + with pytest.raises(RuntimeError, match="--migrate-source-identity"): + await client.connect() + db.close.assert_awaited_once() + assert client.db is None + + +@pytest.mark.parametrize( + "entities,edges", + [ + ( + [ + { + "id": "entity:a", + "name": "Shared", + "type": "concept", + "source": "a.txt", + "metadata": {"source": "b.txt"}, + } + ], + [], + ), + ( + [ + {"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}, + {"id": "entity:b", "name": "Shared", "type": "concept", "source": "a.txt"}, + ], + [], + ), + ( + [{"id": "entity:a", "name": "Shared", "type": "concept", "source": "a.txt"}], + [ + { + "id": "relates:x", + "source": "b.txt", + "in": "entity:a", + "out": "entity:a", + "relationship_type": "related_to", + } + ], + ), + ], +) +async def test_explicit_migration_refuses_ambiguous_attribution(monkeypatch, entities, edges): + db = AsyncMock() + db.query.side_effect = [ + [{"tables": {"entity": "SCHEMAFULL", "relates": "RELATION"}}], + [{"indexes": {"entity_name_type": "name,type UNIQUE"}}], + [entities], + [edges], + ] + monkeypatch.setattr("journal_utilities.rag.graph.surreal_client.AsyncSurreal", lambda _: db) + client = SurrealDBClient() + with pytest.raises(RuntimeError, match="[Aa]mbiguous|Conflicting"): + await client.connect(migrate_source_identity=True) + # No schema/data write was reached after the four evidence reads. + assert db.query.await_count == 4 + db.close.assert_awaited_once() + + +async def test_metadata_only_source_stays_unknown(connected): + client, db = connected + db.query.return_value = [ + [ + { + "id": "entity:old", + "name": "Insight 1", + "type": "key_insight", + "metadata": {"source": "untrusted-claim.txt"}, + } + ] + ] + summary = await client._admit_migration({"entity": "SCHEMAFULL"}) + assert summary["legacy_entities_preserved"] == summary["unknown_source_entities"] == 1 + db.create.assert_not_awaited() diff --git a/tests/journal_utilities/test_app.py b/tests/journal_utilities/test_app.py index d64e90f..1d47490 100644 --- a/tests/journal_utilities/test_app.py +++ b/tests/journal_utilities/test_app.py @@ -12,16 +12,59 @@ # Requires the interface extra; skip cleanly when it is not installed. pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") from fastapi.testclient import TestClient from journal_utilities.interface import chat_engine -from journal_utilities.interface.app import create_app +from journal_utilities.interface.app import ( + MAX_CHAT_BODY_BYTES, + MAX_MESSAGE_LENGTH, + MAX_SESSION_LENGTH, + create_app, +) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def ollama_transport(monkeypatch): + """Run real ChatEngine logic, with only HTTP transport replaced; never sockets.""" + state = {"requests": [], "fail": False} + client_type = httpx.AsyncClient + + def handle(request): + state["requests"].append(request) + if state["fail"]: + raise httpx.ConnectError( + "Connection to http://private-host:11434 failed", request=request + ) + if request.url.path == "/api/tags": + return httpx.Response(200, json={"models": [{"name": chat_engine.OLLAMA_MODEL}]}) + assert request.url.path == "/api/chat" + payload = json.loads(request.content) + if payload["stream"]: + return httpx.Response( + 200, + content=( + json.dumps({"message": {"content": "Bounded answer"}, "done": False}) + + "\n" + + json.dumps({"done": True}) + + "\n" + ), + headers={"content-type": "application/x-ndjson"}, + ) + return httpx.Response(200, json={"message": {"content": "Bounded answer"}}) + + def client(*args, **kwargs): + kwargs["transport"] = httpx.MockTransport(handle) + return client_type(*args, **kwargs) + + monkeypatch.setattr(chat_engine.httpx, "AsyncClient", client) + return state + + @pytest.fixture def sample_data_dir(tmp_path: Path) -> Path: """Create a temporary data directory with realistic sample data.""" @@ -265,8 +308,8 @@ def test_chat_status(self, test_client: TestClient): resp = test_client.get("/api/chat/status") assert resp.status_code == 200 data = resp.json() - # If Ollama is not running, this should still succeed - assert "available" in data or "error" in data + assert data["available"] is True + assert data["current_model"] == chat_engine.OLLAMA_MODEL def test_chat_empty_message_rejected(self, test_client: TestClient): """POST /api/chat rejects empty messages.""" @@ -384,30 +427,18 @@ def test_search_no_results(self, test_client: TestClient): class TestChatEndpointsAdditional: """Additional chat endpoint tests.""" - def test_chat_with_valid_message_returns_response( - self, test_client: TestClient, monkeypatch: pytest.MonkeyPatch - ): - """POST /api/chat with a valid message returns a graceful response. - - With no Ollama reachable the endpoint returns 200 with a sanitized - generic error — it must not leak raw exception detail. The endpoint is - pointed at a dead port so a developer machine running `ollama serve` - still exercises the offline path instead of a live model's answer. - """ - monkeypatch.setattr(chat_engine, "OLLAMA_BASE_URL", "http://127.0.0.1:1") - resp = test_client.post( - "/api/chat", - json={ - "session_id": "test_session", - "message": "What is active inference?", - }, - ) + def test_chat_with_valid_message_returns_response(self, test_client: TestClient): + resp = test_client.post("/api/chat", json={"message": "What is active inference?"}) assert resp.status_code == 200 - body = resp.json() - assert "response" in body - # Internal exception text (host/port/connection) must never be echoed. - assert "Connection" not in body.get("response", "") - assert "http" not in body.get("response", "").lower() + assert resp.json()["response"] == "Bounded answer" + + def test_transport_failure_is_sanitized(self, test_client: TestClient, ollama_transport): + ollama_transport["fail"] = True + resp = test_client.post("/api/chat", json={"message": "What is active inference?"}) + assert resp.status_code == 200 + assert resp.json()["error"] is True + assert "private-host" not in resp.text + assert "http://" not in resp.text def test_chat_rejects_oversized_message(self, test_client: TestClient): """An over-long message is rejected (413), not forwarded to Ollama.""" @@ -416,3 +447,180 @@ def test_chat_rejects_oversized_message(self, test_client: TestClient): json={"session_id": "s", "message": "x" * 40_000}, ) assert resp.status_code == 413 + + +CHAT_ROUTES = ["/api/chat", "/api/chat/stream", "/api/chat/clear"] + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize( + "body,status", + [ + (b"null", 422), + (b"[]", 422), + (b"3", 422), + (b'"text"', 422), + (b"true", 422), + (b"{broken", 400), + (b"\xff", 400), + (b'{"message": null}', 422), + (b'{"message": []}', 422), + (b'{"message": 1}', 422), + (b'{"message": "hello", "session_id": null}', 422), + (b'{"message": "hello", "session_id": {}}', 422), + (b'{"message": "hello", "session_id": 1}', 422), + (b'{"message": "hello", "use_rag": "false"}', 422), + (b'{"message": "hello", "use_rag": 0}', 422), + (b'{"message": "hello", "use_rag": null}', 422), + ], +) +def test_invalid_chat_payloads_are_client_errors( + test_client, ollama_transport, route, body, status +): + response = test_client.post(route, content=body, headers={"content-type": "application/json"}) + assert response.status_code == status + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize("route", ["/api/chat", "/api/chat/stream"]) +def test_exact_field_bounds_allow_escaped_unicode(test_client, route): + payload = { + "message": "😀" * MAX_MESSAGE_LENGTH, + "session_id": "s" * MAX_SESSION_LENGTH, + "use_rag": False, + } + response = test_client.post( + route, content=json.dumps(payload), headers={"content-type": "application/json"} + ) + assert response.status_code == 200 + if route.endswith("/stream"): + events = [ + json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: ") + ] + assert [event["type"] for event in events] == ["context", "token", "done"] + assert events[1]["content"] == "Bounded answer" + else: + assert response.json()["response"] == "Bounded answer" + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize( + "field,length", [("message", MAX_MESSAGE_LENGTH + 1), ("session_id", MAX_SESSION_LENGTH + 1)] +) +def test_field_limit_plus_one_never_reaches_transport( + test_client, ollama_transport, route, field, length +): + payload = {"message": "hello", "session_id": "session", "use_rag": False, field: "x" * length} + assert test_client.post(route, json=payload).status_code == 413 + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize("extra,status", [(0, 200), (1, 413)]) +def test_exact_body_limit(test_client, extra, status): + message = b'{"message":"hello","use_rag":false}' + body = message + b" " * (MAX_CHAT_BODY_BYTES + extra - len(message)) + assert test_client.post("/api/chat", content=body).status_code == status + + +@pytest.mark.parametrize("route", CHAT_ROUTES) +@pytest.mark.parametrize("declared", [None, b"1", str(MAX_CHAT_BODY_BYTES + 1).encode()]) +@pytest.mark.parametrize("single_chunk", [False, True]) +async def test_streamed_body_stops_before_full_accumulation( + sample_data_dir, ollama_transport, route, declared, single_chunk +): + app = create_app(data_dir=sample_data_dir) + chunks = ( + [b"x" * (MAX_CHAT_BODY_BYTES + 1), b"unread"] + if single_chunk + else [b"x" * (MAX_CHAT_BODY_BYTES - 5), b"0123456789", b"unread"] + ) + consumed = 0 + sent = [] + headers = [(b"content-type", b"application/json")] + if declared is not None: + headers.append((b"content-length", declared)) + scope = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.4"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": route, + "raw_path": route.encode(), + "root_path": "", + "query_string": b"", + "headers": headers, + "server": ("test", 80), + "client": ("test", 1234), + } + + async def receive(): + nonlocal consumed + if consumed >= len(chunks) - 1: + pytest.fail("request drained beyond the byte cap") + chunk = chunks[consumed] + consumed += 1 + return {"type": "http.request", "body": chunk, "more_body": True} + + async def send(message): + sent.append(message) + + await app(scope, receive, send) + assert ( + next(message["status"] for message in sent if message["type"] == "http.response.start") + == 413 + ) + assert consumed == ( + 0 if declared == str(MAX_CHAT_BODY_BYTES + 1).encode() else (1 if single_chunk else 2) + ) + assert ollama_transport["requests"] == [] + + +@pytest.mark.parametrize( + "url", + [ + "/api/videos?offset=-1", + "/api/videos?limit=0", + "/api/videos?limit=-1", + "/api/videos?limit=201", + "/api/videos?q=active&offset=-1", + "/api/search?q=active&limit=0", + "/api/search?q=active&limit=-1", + "/api/search?q=active&limit=101", + ], +) +def test_pagination_rejects_invalid_bounds(test_client, url): + assert test_client.get(url).status_code == 422 + + +def test_pagination_upper_bounds_remain_valid(test_client): + videos = test_client.get("/api/videos?offset=0&limit=200") + assert videos.status_code == 200 + assert videos.json()["total"] == 3 + results = test_client.get("/api/search?q=Bayesian&limit=100") + assert results.status_code == 200 + assert results.json()["results"][0]["video_id"] == "vid_bbb22222222" + + +def test_clear_removes_history_without_provider_call(test_client, ollama_transport): + assert ( + test_client.post( + "/api/chat", json={"session_id": "s", "message": "first", "use_rag": False} + ).status_code + == 200 + ) + before = len(ollama_transport["requests"]) + assert test_client.post("/api/chat/clear", json={"session_id": "s"}).json() == { + "status": "cleared" + } + assert len(ollama_transport["requests"]) == before + assert ( + test_client.post( + "/api/chat", json={"session_id": "s", "message": "second", "use_rag": False} + ).status_code + == 200 + ) + payload = json.loads(ollama_transport["requests"][-1].content) + assert [message["content"] for message in payload["messages"] if message["role"] == "user"] == [ + "second" + ] diff --git a/tests/journal_utilities/test_run.py b/tests/journal_utilities/test_run.py index 26d9b41..7e186be 100644 --- a/tests/journal_utilities/test_run.py +++ b/tests/journal_utilities/test_run.py @@ -214,6 +214,24 @@ def test_parser_log_level_override(self): class TestMain: + @pytest.fixture(autouse=True) + def isolated_runner_logging(self, tmp_path, monkeypatch): + import logging + + import run + + root = logging.getLogger() + handlers, level = list(root.handlers), root.level + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + try: + yield + finally: + for handler in list(root.handlers): + if handler not in handlers: + handler.close() + root.handlers[:] = handlers + root.setLevel(level) + def test_config_command_returns_zero(self, sample_config_file): rc = main(["--config", str(sample_config_file), "config"]) assert rc == 0 @@ -302,3 +320,170 @@ def test_default_shows_clickable_url(self, config, capsys, monkeypatch): assert "http://127.0.0.1:9000" in captured.out assert "Open in browser" in captured.out assert rc == 0 + + +@pytest.fixture +def export_corpus(config, tmp_path, monkeypatch): + import run + + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + transcripts = tmp_path / "data" / "output" / "transcripts" + transcripts.mkdir(parents=True) + config.set("export", "markdown", "false") + return config, transcripts, tmp_path / "data" / "export" / "plaintext" + + +def test_export_empty_directory_is_incomplete(export_corpus): + import argparse + + import run + + cfg, _, _ = export_corpus + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + + +@pytest.mark.parametrize("bad_bytes", [b" ", b"\xff\xfe"]) +def test_export_mixed_success_and_failure_is_nonzero(export_corpus, bad_bytes): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "good.txt").write_text("A valid transcript", encoding="utf-8") + (transcripts / "bad.txt").write_bytes(bad_bytes) + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + assert "A valid transcript" in (output / "good.txt").read_text(encoding="utf-8") + assert not (output / "bad.txt").exists() + + +def test_export_read_permission_failure_is_nonzero(export_corpus, monkeypatch): + import argparse + + import run + + cfg, transcripts, _ = export_corpus + blocked = transcripts / "blocked.txt" + blocked.write_text("Cannot read", encoding="utf-8") + read_text = Path.read_text + + def guarded_read(path, *args, **kwargs): + if path == blocked: + raise PermissionError("Access denied") + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", guarded_read) + assert run.cmd_export(cfg, argparse.Namespace()) == 1 + + +def test_export_success_and_existing_outputs_remain_successful(export_corpus): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "good.txt").write_text("A valid transcript", encoding="utf-8") + assert run.cmd_export(cfg, argparse.Namespace()) == 0 + saved = (output / "good.txt").read_bytes() + assert run.cmd_export(cfg, argparse.Namespace()) == 0 + assert (output / "good.txt").read_bytes() == saved + + +@pytest.mark.parametrize("command", ["cmd_full", "cmd_default"]) +def test_required_export_failure_stops_pipeline(export_corpus, monkeypatch, command): + import argparse + + import run + + cfg, transcripts, _ = export_corpus + (transcripts / "empty.txt").write_text("", encoding="utf-8") + monkeypatch.setattr(run, "cmd_download", lambda *_: 0) + monkeypatch.setattr( + run, "cmd_test", lambda *_: pytest.fail("tests started after failed export") + ) + monkeypatch.setattr( + run, "cmd_serve", lambda *_: pytest.fail("server started after failed export") + ) + assert getattr(run, command)(cfg, argparse.Namespace()) == 1 + + +def test_occupied_listener_survives_serve_refusal(config, monkeypatch): + import argparse + import socket + + import run + + uvicorn = pytest.importorskip("uvicorn") + pytest.importorskip("fastapi") + monkeypatch.setattr( + run.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("listener discovery attempted") + ) + monkeypatch.setattr( + run.os, "kill", lambda *_args, **_kwargs: pytest.fail("listener termination attempted") + ) + monkeypatch.setattr( + uvicorn, "run", lambda *_args, **_kwargs: pytest.fail("occupied server started") + ) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(2) + port = listener.getsockname()[1] + config.set("interface", "port", str(port)) + assert run.cmd_serve(config, argparse.Namespace()) == 1 + with socket.create_connection(("127.0.0.1", port), timeout=2) as probe: + accepted, _ = listener.accept() + with accepted: + probe.sendall(b"still alive") + assert accepted.recv(11) == b"still alive" + + +@pytest.mark.parametrize("absolute", [False, True]) +def test_serve_loads_configured_corpus(config, tmp_path, monkeypatch, absolute): + import argparse + import json + + import run + + uvicorn = pytest.importorskip("uvicorn") + pytest.importorskip("fastapi") + from fastapi.testclient import TestClient + + chosen = tmp_path / "chosen" + chosen.mkdir() + (chosen / "channel_videos.json").write_text( + json.dumps( + {"videos": [{"id": "chosen12345", "title": "Chosen corpus", "upload_date": "20250101"}]} + ), + encoding="utf-8", + ) + monkeypatch.setattr(run, "PROJECT_ROOT", tmp_path) + config.set("general", "data_dir", str(chosen) if absolute else "chosen") + config.set("interface", "port", "0") + observed = [] + + def serve_boundary(app, **_kwargs): + with TestClient(app) as client: + response = client.get("/api/videos") + observed.extend(video["id"] for video in response.json()["videos"]) + + monkeypatch.setattr(uvicorn, "run", serve_boundary) + assert run.cmd_serve(config, argparse.Namespace()) == 0 + assert observed == ["chosen12345"] + + +def test_export_writer_failure_is_nonzero(export_corpus, monkeypatch): + import argparse + + import run + + cfg, transcripts, output = export_corpus + (transcripts / "blocked.txt").write_text("Readable input", encoding="utf-8") + write_text = Path.write_text + + def guarded_write(path, *args, **kwargs): + if path == output / "blocked.txt": + raise OSError("Output device unavailable") + return write_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", guarded_write) + assert run.cmd_export(cfg, argparse.Namespace()) == 1