Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 24 additions & 23 deletions src/quant_platform_kit/strategy_lifecycle/ai_reviewer.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,13 +179,6 @@ def llm_enhanced_review(
claude = _parse_reviewer_result(proposal, results, _PRIMARY_LLM)
gpt = _parse_reviewer_result(proposal, results, _SECONDARY_LLM)

# Claude confident → early return
if claude and claude.verdict in ("approve", "reject"):
return AiReviewVerdict(proposal=proposal, verdict=claude.verdict,
overall_score=claude.overall_score, dimensions=base.dimensions,
summary=f"[{_PRIMARY_LLM}] {claude.summary}",
requires_human=claude.verdict == "approve")

# L4: Codex VPS execution verification
codex = None
if client.config.verifier is not None:
Expand All @@ -208,14 +201,10 @@ def _resolve_multi_consensus(
"""Confidence-driven consensus resolution.

Decision logic (ordered):
1. No AI available → escalate
2. Unanimous approve + avg confidence ≥ 0.85 → candidate-ready (human decision required)
3. Unanimous approve + avg confidence ≥ 0.60 → candidate-ready (human decision required)
4. Unanimous reject + avg confidence ≥ 0.85 → reject
5. Codex VERIFIED + LLMs approve + avg confidence ≥ 0.70 → approve
6. Codex MISMATCH → reject
7. Single LLM approve + confidence ≥ 0.85 → approve
8. Otherwise → escalate
Promotion rule: both independent LLM reviewers must complete and approve.
Codex is an optional additional execution verifier, never a substitute for
either reviewer. A missing reviewer or any disagreement is fail-closed for
candidate readiness.
"""
verdicts: list[tuple[str, AiReviewVerdict]] = []
for l, v in [(_PRIMARY_LLM, claude), (_SECONDARY_LLM, gpt), (_CODEX_VPS, codex)]:
Expand All @@ -226,6 +215,26 @@ def _resolve_multi_consensus(
dimensions=base.dimensions, summary="No AI available. " + base.summary,
requires_human=True, confidence=0.0)

missing_independent_reviewers = [
label
for label, verdict in [(_PRIMARY_LLM, claude), (_SECONDARY_LLM, gpt)]
if verdict is None
]
if missing_independent_reviewers:
return AiReviewVerdict(
proposal=proposal,
verdict="escalate",
overall_score=base.overall_score,
dimensions=base.dimensions,
summary=(
"[DUAL_REVIEW_INCOMPLETE] missing independent reviewer(s): "
+ ", ".join(missing_independent_reviewers)
),
requires_human=True,
confidence=0.0,
recommended_action="escalate",
)

apps = [l for l, v in verdicts if v.verdict == "approve"]
rejs = [l for l, v in verdicts if v.verdict == "reject"]
cx_ok = codex and codex.verdict == "approve"
Expand Down Expand Up @@ -275,14 +284,6 @@ def _resolve_multi_consensus(
summary=f"[{_CODEX_VPS} verified] {', '.join(apps)} approve (conf={avg_conf:.0%}).",
requires_human=True, confidence=avg_conf, recommended_action="candidate_ready")

# Single LLM with high confidence
if len(verdicts) == 1 and not codex:
sl, sv = verdicts[0]
if sv.verdict == "approve" and sv.confidence >= 0.85:
return AiReviewVerdict(proposal=proposal, verdict="approve", overall_score=sv.overall_score,
dimensions=base.dimensions, summary=f"[Single: {sl}] high confidence={sv.confidence:.0%}",
requires_human=True, confidence=sv.confidence, recommended_action="candidate_ready")

# Disagreement → escalate
detail = "; ".join(f"{l}={v.verdict}(c={v.confidence:.0%})" for l, v in verdicts)
return AiReviewVerdict(proposal=proposal, verdict="escalate", overall_score=base.overall_score,
Expand Down
48 changes: 48 additions & 0 deletions tests/test_lifecycle_reviewer.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
OptimizationProposal,
)
from quant_platform_kit.strategy_lifecycle.ai_reviewer import (
AiReviewVerdict,
_resolve_multi_consensus,
review_proposal,
llm_enhanced_review,
)
Expand Down Expand Up @@ -46,6 +48,52 @@ def _make_proposal(

class AiReviewerTests(unittest.TestCase):

def test_candidate_readiness_requires_both_independent_reviewers(self) -> None:
proposal = _make_proposal()
base = review_proposal(proposal)
primary = AiReviewVerdict(
proposal=proposal,
verdict="approve",
overall_score=0.9,
dimensions=(),
summary="primary approves",
requires_human=True,
confidence=0.95,
)

result = _resolve_multi_consensus(proposal, base, primary, None, None)

self.assertEqual(result.verdict, "escalate")
self.assertEqual(result.recommended_action, "escalate")
self.assertIn("DUAL_REVIEW_INCOMPLETE", result.summary)

def test_two_independent_approvals_can_become_candidate_ready(self) -> None:
proposal = _make_proposal()
base = review_proposal(proposal)
primary = AiReviewVerdict(
proposal=proposal,
verdict="approve",
overall_score=0.9,
dimensions=(),
summary="primary approves",
requires_human=True,
confidence=0.95,
)
secondary = AiReviewVerdict(
proposal=proposal,
verdict="approve",
overall_score=0.8,
dimensions=(),
summary="secondary approves",
requires_human=True,
confidence=0.9,
)

result = _resolve_multi_consensus(proposal, base, primary, secondary, None)

self.assertEqual(result.verdict, "approve")
self.assertEqual(result.recommended_action, "candidate_ready")

# ── Approve cases ─────────────────────────────────────────────

def test_good_proposal_approves(self) -> None:
Expand Down