diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index 696d9423f7..3958880b0e 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -229,6 +229,7 @@ const analysisActionLabel = (p: Pack | null, zh: boolean) => p?.continuation?.ca function AnalysisContinuation({pack: p, analysis: a}: {pack: Pack; analysis: Controller}) { const c = p.continuation, zh = a.zh; if (!c) return p.continuation_unavailable ?

{zh ? "基础分析暂不可读；本版仍可独立分析完整范围。" : "The base analysis cannot be read; this version can still assess its complete scope independently."}

: null; + const compared = p.result && c.base_available ? c.compared_pair_count : null; const reassessment = c.can_reuse ? (zh ? "复用原判读" : "Reuse judgments") : `${zh ? (p.result ? "已复核全部" : "完整复核") : "Reassess all"} ${p.episode_count} ${zh ? "片段" : "episodes"}`; return
@@ -236,7 +237,10 @@ function AnalysisContinuation({pack: p, analysis: a}: {pack: Pack; analysis: Con
{zh ? "版本与变化" : "Version and changes"}
{c.can_reuse ? (zh ? "证据内容未变" : "Evidence content unchanged") : (zh ? "接续上一版" : "Continue from previous version")}{reassessment}
-

{p.result && c.base_available ? `${c.verdict_changes.length} ${zh ? "项判断变化" : "changed judgments"}` : (zh ? "查看变化" : "Inspect changes")}

+

{compared == null ? (zh ? "共同判断项数量未知。" : "The shared judgment count is unknown.") + : compared === 0 ? (zh ? "无共同判断项" : "No shared judgments") + : zh ? `共同判断 ${compared} 项 · 变化 ${c.verdict_changes.length} 项` : `${compared} shared judgments · ${c.verdict_changes.length} changed`}

+ {compared != null ?

{zh ? "共同判断项按相同片段和指标计数；数量或标签变化不证明分析质量变化。" : "Shared judgments use the same episode and metric. Counts and label changes do not establish changes in analysis quality."}

: null}
{zh ? "上一版" : "Previous"}
{c.base_title} · {c.base_scope.revision}
{c.base_episode_count} · {c.base_scope.window}
{zh ? "本版" : "Current"}
{c.current_scope.revision}
{p.episode_count} · {c.current_scope.window}
{zh ? "来源变动" : "Source changes"}
{zh ? "新增 / 更正 / 删除" : "Added / revised / removed"}: {c.sources.added.length} / {c.sources.revised.length} / {c.sources.removed.length}
diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index d99cc9ac6f..d69ec10b1d 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -397,6 +397,14 @@ versions remain accessible through the same reader and browser history. Reading, registering or refreshing a successor never starts analysis. A failed, cancelled or concurrently running successor does not replace a valid base result. +`compared_pair_count` is the number of shared episode/metric pairs in two +qualified outputs. It is `null` when either output is unavailable; zero means +there are no shared judgments, not that conclusions stayed unchanged. Changed +judgments are shown with this denominator, separately from added/removed eligible +pairs. Older projections without the count remain unknown in the reader. +These are structural version facts, not matched scientific comparisons or +evidence of analysis quality. + The typed comparison reuses a qualified output only when literal excerpts, source identities/paths/line positions, episode questions, eligibility, objective, selection/window and analyst configuration are equivalent. Capture diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-desktop.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-desktop.png new file mode 100644 index 0000000000..77f5c34958 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-desktop.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-desktop.png new file mode 100644 index 0000000000..2c2f32fdd8 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-phone.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-phone.png new file mode 100644 index 0000000000..584237e682 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-expanded-phone.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-phone.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-phone.png new file mode 100644 index 0000000000..2f831af4ab Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-after-phone.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-desktop.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-desktop.png new file mode 100644 index 0000000000..b74a0b5eb0 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-desktop.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-desktop.png new file mode 100644 index 0000000000..08caecb82b Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-phone.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-phone.png new file mode 100644 index 0000000000..7b515f942c Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-expanded-phone.png differ diff --git a/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-phone.png b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-phone.png new file mode 100644 index 0000000000..2f831af4ab Binary files /dev/null and b/examples/personal-workspace-browser/evidence/research-finding-reader/judgment-comparison-before-phone.png differ diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index d68696d797..e5cfd02c33 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -335,6 +335,7 @@ export const researchHistoryScenario = { let holdCitation = false, failedCitation = false, citationStarted, releaseCitation; let unavailableAnalysis = false, failedAnalysisPack = "a", episodeMetadataLimit = null, comparisonFixture = false; let analysisWrites = 0, analysisReads = 0, localizedFixture = false, continuationFixture = false, equivalentContinuation = false; + let versionComparison = "shared"; let unavailableTask = false; const taskReads = []; try { @@ -371,8 +372,11 @@ export const researchHistoryScenario = { sources: {added: ['follow-up'], revised: [], removed: []}, episodes: {added: [], revised: ['variant-alpha'], removed: []}, can_reuse: false, configuration_changed: false, objective_changed: false, base_available: true, base_episode_count: 4, reassessed_episode_ids: p.episodes.map(e => e.id), new_pair_count: 0, removed_pair_count: 0, + compared_pair_count: versionComparison === "disjoint" ? 0 : p.result.assessments.length, verdict_changes: [{episode_id: 'variant-alpha', metric: 'benefit_test', before: 'yes', after: 'unknown'}], }; + if (!base && ["disjoint", "unchanged"].includes(versionComparison)) p.continuation.verdict_changes = []; + if (!base && versionComparison === "legacy") delete p.continuation.compared_pair_count; } for (const pack of [...readback.catalog.packs, readback.pack]) pack.config = {...pack.config, output_language: language}; if (readback.pack.result) { @@ -1386,6 +1390,7 @@ print(json.dumps([ await version.locator('summary').focus(); await page.keyboard.press('Enter'); await version.getByText(/已复核全部 4 片段/).waitFor(); + await version.getByText(/共同判断 \d+ 项 · 变化 1 项/).waitFor(); assert.match(await version.innerText(), /original-capture.*later-evidence/s); assert.match(await version.innerText(), /variant-alpha.*已做收益检验.*证据不足/); await version.locator('summary').click(); @@ -1400,6 +1405,15 @@ print(json.dumps([ await version.getByText('版本与变化', {exact: true}).waitFor(); await page.reload({waitUntil: 'networkidle'}); await version.getByText('版本与变化', {exact: true}).waitFor(); + for (const [mode, label] of [["unchanged", /共同判断 \d+ 项 · 变化 0 项/], ["disjoint", "无共同判断项"], ["legacy", "共同判断项数量未知。"]]) { + versionComparison = mode; + await page.reload({waitUntil: 'networkidle'}); + await version.getByText('版本与变化', {exact: true}).click(); + await version.getByText(label, {exact: true}).waitFor(); + if (mode !== "unchanged") assert.doesNotMatch(await version.innerText(), /共同判断 \d+ 项 · 变化 0 项|0 项判断变化/); + } + versionComparison = "shared"; + await page.reload({waitUntil: 'networkidle'}); await page.setViewportSize({width: 390, height: 844}); await chineseAnalysis.evaluate(element => element.scrollIntoView({block: 'start'})); assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); @@ -1407,6 +1421,11 @@ print(json.dumps([ await page.evaluate(key => localStorage.setItem(key, 'en'), 'loopx-pw-locale'); await page.reload({waitUntil: 'networkidle'}); await page.getByRole('button', {name: 'Previous version', exact: true}).waitFor(); + versionComparison = "disjoint"; + await page.reload({waitUntil: 'networkidle'}); + await page.getByText('Version and changes', {exact: true}).click(); + await page.getByText('No shared judgments', {exact: true}).waitFor(); + versionComparison = "shared"; equivalentContinuation = true; await page.reload({waitUntil: 'networkidle'}); await page.getByRole('button', {name: 'Reuse analysis', exact: true}).first().waitFor(); diff --git a/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts b/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts index 7a290a8773..d443a46af1 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_continuation.ts @@ -44,14 +44,17 @@ export function compareTrajectoryAssessments(params: Record): T const eligibleBefore = new Set(base.input.episodes.flatMap(e => e.eligible_metrics.map(metric => `${e.id}\0${metric}`))); const newPairs = new Set(current.input.episodes.flatMap(e => e.eligible_metrics.map(metric => `${e.id}\0${metric}`))); const result = params.output === undefined ? null : qualifyTrajectoryAssessment({...params, analysis_key: current.analysis_key}); + const shared = (result?.assessments ?? []).flatMap(after => { + const before = oldPairs.get(pair(after)); + return before ? [{before, after}] : []; + }); return {base_analysis_key: base.analysis_key, base_scope: base.input.scope, current_scope: current.input.scope, sources, episodes, can_reuse, configuration_changed: !same(base.config, current.config), objective_changed: base.input.objective !== current.input.objective, reassessed_episode_ids: can_reuse ? [] : current.input.episodes.map(e => e.id), new_pair_count: [...newPairs].filter(key => !eligibleBefore.has(key)).length, removed_pair_count: [...eligibleBefore].filter(key => !newPairs.has(key)).length, - verdict_changes: (result?.assessments ?? []).flatMap(a => { - const old = oldPairs.get(pair(a)); - return old && old.verdict !== a.verdict ? [{episode_id: a.episode_id, metric: a.metric, before: old.verdict, after: a.verdict}] : []; - })}; + compared_pair_count: oldResult !== null && result !== null ? shared.length : null, + verdict_changes: shared.flatMap(({before, after}) => before.verdict !== after.verdict + ? [{episode_id: after.episode_id, metric: after.metric, before: before.verdict, after: after.verdict}] : [])}; } diff --git a/loopx/control_plane/capabilities/trajectory_assessment_types.ts b/loopx/control_plane/capabilities/trajectory_assessment_types.ts index 6d9e1e28ab..d4aaaa2b1f 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_types.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_types.ts @@ -76,6 +76,8 @@ export interface TrajectoryContinuation { episodes: {added: string[]; revised: string[]; removed: string[]}; can_reuse: boolean; configuration_changed: boolean; objective_changed: boolean; reassessed_episode_ids: string[]; new_pair_count: number; removed_pair_count: number; + /** Shared episode/metric rows in two qualified outputs; null if either is unavailable. */ + compared_pair_count: number | null; verdict_changes: Array<{episode_id: string; metric: TrajectoryMetric; before: AssessmentVerdict; after: AssessmentVerdict}>; } diff --git a/tests/capabilities/test_trajectory_assessment_store.py b/tests/capabilities/test_trajectory_assessment_store.py index f3a28bcbf1..28a8f1f9cb 100644 --- a/tests/capabilities/test_trajectory_assessment_store.py +++ b/tests/capabilities/test_trajectory_assessment_store.py @@ -247,6 +247,7 @@ def test_new_evidence_reassesses_old_judgments_and_complete_denominator(pack, tm assert before["continuation"]["can_reuse"] is False assert before["continuation"]["reassessed_episode_ids"] == ["episode", "follow-up"] assert before["continuation"]["new_pair_count"] == 1 + assert before["continuation"]["compared_pair_count"] is None revised = {"assessments": [dict(episode_id=e, metric="benefit_test", verdict=v, reason="Later evidence corrects the earlier interpretation", citations=[dict(source_id="later", first_line=1, last_line=1)] if v != "unknown" else []) for e, v in [("episode", "no"), ("follow-up", "unknown")]], "limitations": ["Synthetic counterevidence"]} revised["synthesis"] = {"summary": {"text": "The earlier interpretation was refuted by later evidence.", "episode_ids": ["episode"], "citations": [dict(source_id="later", first_line=1, last_line=1)]}, "findings": [], "next_experiments": []} @@ -259,9 +260,35 @@ def test_new_evidence_reassesses_old_judgments_and_complete_denominator(pack, tm benefit = next(row for row in view["result"]["metrics"] if row["metric"] == "benefit_test") assert (benefit["yes"], benefit["no"], benefit["unknown"], benefit["eligible"]) == (0, 1, 1, 2) assert view["continuation"]["verdict_changes"] == [dict(episode_id="episode", metric="benefit_test", before="yes", after="no")] + assert view["continuation"]["compared_pair_count"] == 1 assert inspect_pack(**scope, pack_id=registered["pack_id"])["result"] == previous +def test_disjoint_research_units_have_zero_comparisons_without_overwriting_prior_result(pack, tmp_path): + scope, operation, registered, path, sources, provider = pack + execute_pack(**operation, codex=provider) + previous = inspect_pack(**scope)["result"] + old_files = {p: p.read_bytes() for p in scope["runtime_root"].glob("benchmark/trajectory-assessments/*/jobs/*/*") if p.is_file()} + raw = json.loads(next(p for p in old_files if p.name == "model-output.json").read_text()) + data = json.loads(path.read_text()) + data["scope"]["revision"] = "different-research-unit" + data["episodes"][0]["id"] = "new-question" + raw["assessments"][0]["episode_id"] = "new-question" + raw["synthesis"]["summary"]["episode_ids"] = ["new-question"] + path.write_text(json.dumps(data)) + successor = register_pack(**scope, input_path=path, source_root=sources, title="Different research unit", base_pack_id=registered["pack_id"]) + assert inspect_pack(**scope)["continuation"]["compared_pair_count"] is None + executable = tmp_path / "disjoint-output-fixture" + executable.write_text(f"#!{sys.executable}\nimport sys\nfrom pathlib import Path\nsys.stdin.read()\nPath(sys.argv[sys.argv.index('-o')+1]).write_text({json.dumps(raw)!r})\n") + executable.chmod(0o700) + execute_pack(**scope, pack_id=successor["pack_id"], analysis_key=successor["analysis_key"], codex=str(executable)) + delta = inspect_pack(**scope)["continuation"] + assert (delta["compared_pair_count"], delta["new_pair_count"], delta["removed_pair_count"]) == (0, 1, 1) + assert delta["verdict_changes"] == [] + assert inspect_pack(**scope, pack_id=registered["pack_id"])["result"] == previous + assert all(p.read_bytes() == content for p, content in old_files.items()) + + def test_continuation_does_not_cross_owner_run_or_revoked_source(pack, tmp_path): scope, operation, registered, path, sources, provider = pack execute_pack(**operation, codex=provider) @@ -282,6 +309,7 @@ def test_continuation_does_not_cross_owner_run_or_revoked_source(pack, tmp_path) view = inspect_pack(**scope) assert view["availability"] == "available" assert not view["continuation"]["base_available"] and not view["continuation"]["can_reuse"] + assert view["continuation"]["compared_pair_count"] is None assert view["continuation"]["reassessed_episode_ids"] == ["episode"] result = execute_pack(**scope, pack_id=successor["pack_id"], analysis_key=successor["analysis_key"], codex=provider) assert result["status"] == "succeeded" and not result["reused"] diff --git a/tests/control_plane_ts/trajectory_assessment.test.ts b/tests/control_plane_ts/trajectory_assessment.test.ts index 1341bd238e..6778e400f1 100644 --- a/tests/control_plane_ts/trajectory_assessment.test.ts +++ b/tests/control_plane_ts/trajectory_assessment.test.ts @@ -368,6 +368,43 @@ test('later evidence can overturn an earlier judgment; full new denominators are assert.throws(() => compareTrajectoryAssessments({...current, output: output(), previous: {...original, output: output()}}), /missing eligible/); }); +test('comparison denominator distinguishes unchanged judgments, disjoint units and unavailable results', () => { + const current = prepared(), previous = {...prepared(), output: output()}; + const unchanged = compareTrajectoryAssessments({...current, previous, output: output()}); + assert.equal(unchanged.compared_pair_count, 3); + assert.deepEqual(unchanged.verdict_changes, []); + for (const args of [ + {...current, previous}, + {...current, previous: prepared(), output: output()}, + {...current, previous: prepared()}, + ]) assert.equal(compareTrajectoryAssessments(args).compared_pair_count, null, 'Missing qualified output is unknown, not zero comparisons'); + + const next = input(), renamed = output(); + next.scope.revision = 'different-research-units'; + for (const episode of next.episodes) episode.id = `new-${episode.id}`; + for (const assessment of renamed.assessments) assessment.episode_id = `new-${assessment.episode_id}`; + const disjoint = compareTrajectoryAssessments({input: next, config, previous, output: renamed}); + assert.equal(disjoint.compared_pair_count, 0); + assert.equal(disjoint.new_pair_count, 3); + assert.equal(disjoint.removed_pair_count, 3); + assert.deepEqual(disjoint.verdict_changes, [], 'No shared rows is not evidence of unchanged conclusions'); +}); + +test('comparison denominator counts shared episode and metric pairs, not total new judgments', () => { + const next = input(), revised = output(); + next.episodes[0]!.eligible_metrics = ['information_gain']; + next.episodes.push({id: 'e3', question: 'Does the new intervention help?', source_ids: ['s'], eligible_metrics: ['information_gain']}); + next.selection.population_episodes = 3; + revised.assessments.splice(1, 1); + revised.assessments[0]!.verdict = 'no'; + revised.assessments.push({episode_id: 'e3', metric: 'information_gain', verdict: 'yes', reason: 'New evidence', citations: [{source_id: 's', first_line: 10, last_line: 10}]}); + const delta = compareTrajectoryAssessments({input: next, config, previous: {...prepared(), output: output()}, output: revised}); + assert.equal(delta.compared_pair_count, 2); + assert.equal(delta.new_pair_count, 1); + assert.equal(delta.removed_pair_count, 1); + assert.deepEqual(delta.verdict_changes, [{episode_id: 'e1', metric: 'information_gain', before: 'yes', after: 'no'}]); +}); + test('corrections, removals, scope and model changes require full reassessment and identity cannot cross runs', () => { const previous = {...prepared(), output: output()}; for (const edit of [(x: TrajectoryInput) => {x.sources[0]!.text = 'corrected negative evidence';},