feat(gooddata-eval): record the turns and steps each agentic run took

The effort comparison in gdc-nas reads these. A pass/fail bit cannot separate
two reasoning efforts, while the step count moves with the effort, and the turn
count tells a wrong answer from a run max_iterations cut short.

alert_skill and kda_skill gained the counters; metric_skill and conversation
already had them and only needed the write. conversation also logs
clarification_turns, since its total includes the turns its fixture asks for.

Author: myhoai

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -477,6 +477,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    total_turns: float = 0.0
+    total_steps: float = 0.0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -676,9 +678,13 @@ def _run_once(conv_id: str) -> AlertRunResult:
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
+            turns = 0
+            steps = 0.0
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                turns += 1
+                steps += float(chat_result.reasoning_step_count)
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
@@ -728,6 +734,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                total_turns=float(turns),
+                total_steps=steps,
                 reasoning_steps=reasoning_steps,
                 response_id=response_id,
                 tool_call_events=all_tool_call_events,
@@ -851,6 +859,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -336,6 +336,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    total_steps: float = 0.0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -365,6 +366,7 @@ def run_agentic_conversation(
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
     total_clarification_turns = 0
+    total_steps = 0.0
     conversation_id: str = ""
     owns_conversation = False
     # Metrics created during this conversation, deleted after it completes so they do
@@ -434,6 +436,7 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
+                total_steps += float(chat_result.reasoning_step_count)
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
                     chat_result,
                     turn_offset=turn_offset,
@@ -522,6 +525,7 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        total_steps=total_steps,
         reasoning_steps=reasoning_steps,
         response_id=response_id,
         tool_call_events=conversation_tool_call_events,
@@ -611,6 +615,22 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     value=float(result.full_skill_coverage),
                     data_type="BOOLEAN",
                 )
+                # One turn per fixture turn, plus every simulated-user round the agent triggered.
+                # The clarification count alone hides how much of the conversation the fixture
+                # asked for, so the comparison needs the total.
+                ctx.score(
+                    tid,
+                    name="turns",
+                    value=float(len(result.turn_results) + result.total_clarification_turns),
+                    data_type="NUMERIC",
+                )
+                ctx.score(tid, name="steps", value=result.total_steps, data_type="NUMERIC")
+                ctx.score(
+                    tid,
+                    name="clarification_turns",
+                    value=float(result.total_clarification_turns),
+                    data_type="NUMERIC",
+                )
                 for tr in result.turn_results:
                     ctx.score(
                         tid,

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -185,6 +185,8 @@ class KdaRunResult:
     # Wall-clock time of the turn that called create (None if create never happened) --
     # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once.
     turn_wall_clock_sec: float | None = None
+    total_turns: float = 0.0
+    total_steps: float = 0.0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -276,6 +278,9 @@ def _accumulate(result: ChatResult) -> None:
             all_tool_call_events.extend(result.tool_call_events or [])
             all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
+        turns = 0
+        steps = 0.0
+
         for iteration in range(max_iterations):
             try:
                 chat_result = client.send_message(conv_id, current_question)
@@ -291,6 +296,8 @@ def _accumulate(result: ChatResult) -> None:
                         turn_wall_clock_sec = partial.turn_wall_clock_sec
                 turn_completed = False
                 break
+            turns += 1
+            steps += float(chat_result.reasoning_step_count)
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
@@ -330,6 +337,8 @@ def _accumulate(result: ChatResult) -> None:
             actual_create_args=create_args,
             actual_execute_result=execute_result,
             turn_wall_clock_sec=turn_wall_clock_sec,
+            total_turns=float(turns),
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -442,6 +451,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
                     ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     if turn_wall_clock_sec is not None:
                         # combo_report.py reads this score directly -- no trace re-resolution needed.
                         ctx.score(

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -163,6 +163,7 @@ class MetricRunResult:
     actual_maql: str
     maql_correct: bool
     total_turns: float
+    total_steps: float = 0.0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -253,6 +254,7 @@ def _execute_single_metric_run(
     metric_result: dict | None = None
     created_metric_ids: list[str] = []
     turns = 0
+    steps = 0.0
     current_question = question
     reasoning_steps: list[str] = []
     response_id: str | None = None
@@ -280,6 +282,7 @@ def _execute_single_metric_run(
             )
             all_tool_call_events.extend(chat_result.tool_call_events or [])
             all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+            steps += float(chat_result.reasoning_step_count)
             for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []):
                 if metric_id not in created_metric_ids:
                     created_metric_ids.append(metric_id)
@@ -331,6 +334,7 @@ def _execute_single_metric_run(
             actual_maql=actual_maql,
             maql_correct=maql_correct,
             total_turns=float(turns),
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -466,6 +470,8 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 ) as tid:
                     ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN")
                     ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

packages/gooddata-eval/tests/test_agentic_conversation.py

@@ -1025,3 +1025,49 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
         ],
         "latency_breakdown": [],
     }
+
+
+def test_run_agentic_conversation_sums_the_reasoning_steps_of_every_turn():
+    """QA-29110: the effort comparison reads `steps`. A clarification round is part of the
+    work the effort setting changes, so its steps count with the rest."""
+    proposal_turn = ChatResult.model_validate(
+        {
+            "text_response": None,
+            "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
+            "reasoningStepCount": 2,
+            "toolCallEvents": [
+                {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
+                {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
+            ],
+        }
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "text_response": "Alert created.",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'}
+            ],
+        }
+    )
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [proposal_turn, created_turn]
+
+    with (
+        patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"),
+        patch(
+            "gooddata_eval.core.agentic.conversation._get_sim_user_response",
+            return_value="Yes, please create it.",
+        ),
+    ):
+        result = run_agentic_conversation(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            fixture=_alert_turn_fixture(),
+        )
+
+    assert result.total_steps == 5.0
+    assert result.total_clarification_turns == 1

packages/gooddata-eval/tests/test_agentic_kda_skill.py

@@ -1195,3 +1195,53 @@ def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partia
 
     assert exc_info.value.reasoning_steps == ["analyzing before cutoff"]
     assert exc_info.value.response_id == "resp-3"
+
+
+def test_run_agentic_kda_skill_counts_the_turns_and_reasoning_steps_it_used():
+    """QA-29110: the effort comparison reads these. A binary pass/fail cannot separate two
+    efforts on a nightly's sample, while the reasoning-step count moves with the effort."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    # Turn 1 asks for clarification, turn 2 runs the analysis: 2 turns, 1 step each.
+    mock_client.send_message.side_effect = [
+        _no_kda_chat_result("Which metric did you mean?"),
+        _kda_chat_result(success=True),
+    ]
+
+    with (
+        patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", return_value="Revenue"),
+    ):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="What drove the change?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=2,
+        )
+
+    assert summary.best.total_turns == 2.0
+    assert summary.best.total_steps == 2.0
+
+
+def test_run_agentic_kda_skill_reports_no_turns_when_the_first_send_fails():
+    """A run that never got a reply must not report a turn it did not take."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = RuntimeError("stream died")
+
+    with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="What drove the change?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=1,
+        )
+
+    assert summary.best.total_turns == 0.0
+    assert summary.best.total_steps == 0.0