diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 1526c309d..24fbad1a9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -16,7 +16,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport @@ -86,11 +86,12 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, -) -> list[str] | None: +) -> AgenticEvalOutcome | list[str] | None: """Call the appropriate evaluate_agentic_* function for the item's test_kind. - Returns whatever that function returns -- only alert_skill/metric_skill/conversation - currently return their reasoning_steps; the rest still return None (unchanged). + Returns whatever that function returns -- alert_skill/metric_skill/conversation return + an AgenticEvalOutcome; the rest still return None + (unchanged). """ kind = item.test_kind eo = item.expected_output @@ -232,16 +233,26 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - reasoning_steps = _dispatch_agentic( + outcome = _dispatch_agentic( item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id ) + if isinstance(outcome, AgenticEvalOutcome): + reasoning_steps = outcome.reasoning_steps + conversation_id = outcome.conversation_id + response_id = outcome.response_id + else: + reasoning_steps, conversation_id, response_id = outcome, None, None item_report.pass_at_k = True item_report.runs = k item_report.reasoning_steps = reasoning_steps or [] + item_report.conversation_id = conversation_id + item_report.response_id = response_id except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] + item_report.conversation_id = getattr(exc, "conversation_id", None) + item_report.response_id = getattr(exc, "response_id", None) print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 532abced8..e21737371 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -14,7 +14,7 @@ from gooddata_eval.core.agentic._catalog import CatalogMetricAlert from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -303,6 +303,7 @@ class AlertRunResult: eval: AlertEvaluation actual_alert_arguments: dict reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -452,6 +453,7 @@ def _run_once(conv_id: str) -> AlertRunResult: actual_args: dict = {} tool_called = False reasoning_steps: list[str] = [] + response_id: str | None = None # conversation_history stores prior turns for GPT-4o context. # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply. conversation_history: list = [] @@ -460,6 +462,7 @@ def _run_once(conv_id: str) -> AlertRunResult: for _iteration in range(max_iterations): chat_result = client.send_message(conv_id, current_question) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id @@ -496,6 +499,7 @@ def _run_once(conv_id: str) -> AlertRunResult: eval=ev, actual_alert_arguments=actual_args, reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if alert_id_to_delete: @@ -547,6 +551,8 @@ class AlertSkillAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_alert_skill( @@ -566,12 +572,15 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> AgenticEvalOutcome: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. - Returns the best run's reasoning_steps on success; on failure the same list is attached - to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception - idiom in `ChatClient.ask()`) so callers can retrieve it either way. + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -655,5 +664,11 @@ def evaluate_agentic_alert_skill( f"Actual args: {best.actual_alert_arguments}" ) exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 51fd1d4e9..95821bb43 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -15,7 +15,7 @@ from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent from gooddata_eval.core.scoring import ( check_filters, check_viz_type, @@ -266,6 +266,7 @@ class ConversationResult: conversation_success: bool total_clarification_turns: int reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None def run_agentic_conversation( @@ -298,6 +299,7 @@ def run_agentic_conversation( # the end — a later turn may $ref a metric an earlier turn created. created_metric_ids: list[str] = [] reasoning_steps: list[str] = [] + response_id: str | None = None try: if initial_conversation_id is not None: @@ -335,6 +337,7 @@ def run_agentic_conversation( final_result = chat_result all_tool_calls.extend(chat_result.tool_call_events or []) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id if _check_output_present(resolved_turn, chat_result): break @@ -399,6 +402,7 @@ def run_agentic_conversation( conversation_success=conversation_success, total_clarification_turns=total_clarification_turns, reasoning_steps=reasoning_steps, + response_id=response_id, ) @@ -407,6 +411,8 @@ class ConversationAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_conversation( @@ -424,12 +430,14 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> AgenticEvalOutcome: """Run conversation evaluation, log to Langfuse, and raise on failure. - Returns the conversation's reasoning_steps on success; on failure the same list is - attached to the raised exception as ``.reasoning_steps`` (mirrors the - `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it + Returns the conversation's outcome (reasoning_steps, conversation_id, response_id) as + an AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. """ from datetime import datetime as _dt # noqa: PLC0415 @@ -514,5 +522,11 @@ def evaluate_agentic_conversation( f"Failed turns: {[t.turn_id for t in failed_turns]}" ) exc.reasoning_steps = result.reasoning_steps + exc.conversation_id = result.conversation_id + exc.response_id = result.response_id raise exc - return result.reasoning_steps + return AgenticEvalOutcome( + reasoning_steps=result.reasoning_steps, + conversation_id=result.conversation_id, + response_id=result.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2db890209..a699ebe63 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -12,7 +12,7 @@ from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import ToolCallEvent +from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent try: from openai import OpenAI as _OpenAI @@ -130,6 +130,7 @@ class MetricRunResult: maql_correct: bool total_turns: float reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None @dataclass @@ -207,12 +208,14 @@ def _execute_single_metric_run( turns = 0 current_question = question reasoning_steps: list[str] = [] + response_id: str | None = None try: for _iteration in range(max_iterations): turns += 1 chat_result = client.send_message(conversation_id, current_question) reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: metric_result = candidate @@ -240,6 +243,7 @@ def _execute_single_metric_run( maql_correct=maql_correct, total_turns=float(turns), reasoning_steps=reasoning_steps, + response_id=response_id, ) finally: if metric_id_to_delete: @@ -311,6 +315,8 @@ class MetricSkillAssertionError(AssertionError): __tracebackhide__ = True reasoning_steps: list[str] + conversation_id: str + response_id: str | None def evaluate_agentic_metric_skill( @@ -330,12 +336,15 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, -) -> list[str]: +) -> AgenticEvalOutcome: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. - Returns the best run's reasoning_steps on success; on failure the same list is attached - to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception - idiom in `ChatClient.ask()`) so callers can retrieve it either way. + Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an + AgenticEvalOutcome on success; on failure the same three values are attached to the + raised exception as + ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the + `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them + either way. """ from datetime import datetime as _dt # noqa: PLC0415 from datetime import timezone as _tz # noqa: PLC0415 @@ -408,5 +417,11 @@ def evaluate_agentic_metric_skill( f"Actual MAQL: {best.actual_maql}." ) exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id raise exc - return summary.best.reasoning_steps + return AgenticEvalOutcome( + reasoning_steps=summary.best.reasoning_steps, + conversation_id=summary.best.conversation_id, + response_id=summary.best.response_id, + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 40f7ede96..0c44cc114 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -107,6 +107,14 @@ class ChatResult(BaseModel): turn_wall_clock_sec: float | None = None +class AgenticEvalOutcome(BaseModel): + """Reasoning trace and trace-lookup IDs returned by an evaluate_agentic_* call on success.""" + + reasoning_steps: list[str] = Field(default_factory=list) + conversation_id: str | None = None + response_id: str | None = None + + class SummaryInput(BaseModel): """Structured input for the `dashboard_summary` test kind. diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 140c62fd4..acc0b5669 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -565,7 +565,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), ): - reasoning = evaluate_agentic_alert_skill( + outcome = evaluate_agentic_alert_skill( host="http://host", token="tok", workspace_id="ws1", @@ -575,7 +575,9 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): max_iterations=1, ) - assert reasoning == ["thinking about it"] + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -604,3 +606,5 @@ def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_f max_iterations=1, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index cc4ab0791..a6368683b 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -532,6 +532,7 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): chat_result.created_visualizations = [MagicMock()] chat_result.tool_call_events = [tc] chat_result.reasoning_steps = ["thinking about it"] + chat_result.response_id = "resp-1" mock_client.send_message.return_value = chat_result fixture = ConversationFixture( @@ -547,13 +548,15 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass(): ], ) with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client): - reasoning = evaluate_agentic_conversation( + outcome = evaluate_agentic_conversation( host="http://host", token="tok", workspace_id="ws1", fixture=fixture, ) - assert reasoning == ["thinking about it"] + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id == "resp-1" def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail(): @@ -568,6 +571,7 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ chat_result.tool_call_events = [tc] chat_result.alert_proposals = [] chat_result.reasoning_steps = ["confused thinking"] + chat_result.response_id = "resp-2" mock_client.send_message.return_value = chat_result fixture = ConversationFixture( @@ -594,3 +598,5 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_ max_clarification_turns=0, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id == "resp-2" diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index ccc24299c..684436ee1 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -381,7 +381,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): } ) with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): - reasoning = evaluate_agentic_metric_skill( + outcome = evaluate_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", workspace_id="ws1", @@ -390,7 +390,9 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): k=1, max_iterations=1, ) - assert reasoning == ["thinking about it"] + assert outcome.reasoning_steps == ["thinking about it"] + assert outcome.conversation_id == "conv-1" + assert outcome.response_id is None def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): @@ -417,3 +419,5 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_ max_iterations=1, ) assert exc_info.value.reasoning_steps == ["confused thinking"] + assert exc_info.value.conversation_id == "conv-1" + assert exc_info.value.response_id is None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index fda604542..627afebd9 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -5,7 +5,7 @@ import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError -from gooddata_eval.core.models import DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): @@ -108,7 +108,9 @@ def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem: def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): with patch( "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", - return_value=["it created the alert"], + return_value=AgenticEvalOutcome( + reasoning_steps=["it created the alert"], conversation_id="conv-1", response_id="resp-1" + ), ): report = run_agentic_items( [_item()], @@ -119,11 +121,15 @@ def test_run_agentic_items_surfaces_reasoning_steps_on_pass(): ) assert report.items[0].pass_at_k is True assert report.items[0].reasoning_steps == ["it created the alert"] + assert report.items[0].conversation_id == "conv-1" + assert report.items[0].response_id == "resp-1" def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): exc = AlertSkillAssertionError("nope") exc.reasoning_steps = ["it got confused"] + exc.conversation_id = "conv-2" + exc.response_id = "resp-2" with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): report = run_agentic_items( [_item()], @@ -134,6 +140,8 @@ def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail(): ) assert report.items[0].pass_at_k is False assert report.items[0].reasoning_steps == ["it got confused"] + assert report.items[0].conversation_id == "conv-2" + assert report.items[0].response_id == "resp-2" def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none(): @@ -149,6 +157,8 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_ run_ts="2026-01-01", ) assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds(): @@ -163,3 +173,5 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds ) assert report.items[0].pass_at_k is True assert report.items[0].reasoning_steps == [] + assert report.items[0].conversation_id is None + assert report.items[0].response_id is None