Skip to content

Commit 100a966

Browse files
committed
1.3: severity-tracked validation hints
Phase 1.3 — when a turn includes high-severity mutations (td_create_node, td_delete_node, td_exec_python, etc.) without a follow-up validator (td_get_errors / td_audit_project / td_validate_recipe / patch_validate), the runtime emits ``EV_HINT`` so the chat UI can render a soft nudge below the assistant's reply. Soft signal — never blocks the conversation, never fires on read-only or medium-only turns. Reviewer asked for a forced validation continuation; that was too heavyweight (interrupts every minor mutation, breaks the agent's flow on legitimate "I'll validate after the next step" plans). A severity-aware hint is the right balance — informational, dismissable, visible enough to catch dropped validations. td_component/tdpilot_api_runtime.py: - New constants: EV_HINT, _TOOL_SEVERITY (12 high + 3 medium tools classified), _VALIDATOR_TOOLS (4 entries that satisfy a high mutation), _tool_severity(name) helper. - AgentRuntime gains _turn_tool_calls list (cleared at start_turn, written via on_tool_result callback). _maybe_emit_validation_hint runs at on_turn_done — checks for high-severity tools without a validator, emits one EV_HINT event listing the unique offenders. - Failed tool calls (is_error=True) don't count toward the high tally — the model already saw the error and didn't actually mutate state. td_component/tdpilot_api_extension.py: - _handle_event grew an EV_HINT branch that writes a "hint"-role line into both the textTable transcript and the HTML chat. td_component/tdpilot_api_chat.html: - .msg.hint CSS class — yellow-ish, dim, italic. Distinct from .msg.error (red, urgent) since the turn was technically successful. - appendMessage's role-to-class mapping covers "hint". Tests (tests/test_tdpilot_api_runtime.py): 8 new cases. - test_severity_lookup_classifies_known_tools_correctly - test_validation_hint_fires_on_high_severity_without_validator - test_validation_hint_suppressed_when_validator_called - test_validation_hint_suppressed_for_low_severity_only_turns - test_validation_hint_suppressed_for_medium_severity_only - test_validation_hint_ignores_failed_tool_calls - test_validation_hint_lists_all_high_severity_tools (dedup + sort verified) - test_validation_hint_cleared_between_turns (per-turn ledger reset behaviour) Pytest 1000 passing (up from 992). Lints + format clean. Three td_component sources changed — standalone tdpilot_API.tox needs rebuilding before the chat UI picks up the hint rendering.
1 parent 4213d6c commit 100a966

4 files changed

Lines changed: 326 additions & 3 deletions

File tree

td_component/tdpilot_api_chat.html

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,16 @@
168168
color: var(--error-fg);
169169
}
170170
.msg.error .role { color: var(--error-fg); }
171+
/* Phase 1.3 — soft validation nudge. Distinct from "error": the
172+
turn was successful, this is a non-blocking hint to consider
173+
calling td_get_errors. Yellow-ish, dim, italic. */
174+
.msg.hint {
175+
border-left-color: #d9a900;
176+
color: #d9a900;
177+
font-style: italic;
178+
opacity: 0.85;
179+
}
180+
.msg.hint .role { color: #d9a900; }
171181

172182
/* Welcome screen: ASCII block logo + minimal instructions. */
173183
.welcome {
@@ -401,7 +411,8 @@
401411
role === 'assistant' ? 'assistant' :
402412
role === 'tool_call' ? 'tool_call' :
403413
role === 'tool_result' ? 'tool_result' :
404-
role === 'error' ? 'error' : 'tool_call'
414+
role === 'error' ? 'error' :
415+
role === 'hint' ? 'hint' : 'tool_call'
405416
);
406417
const r = document.createElement('div');
407418
r.className = 'role';

td_component/tdpilot_api_extension.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,7 @@ def _handle_event(self, kind: str, payload: Any) -> None:
368368
from tdpilot_api_runtime import ( # type: ignore[import-not-found]
369369
EV_DONE,
370370
EV_ERROR,
371+
EV_HINT,
371372
EV_MODEL,
372373
EV_STATE,
373374
EV_SUB_DONE,
@@ -403,6 +404,17 @@ def _handle_event(self, kind: str, payload: Any) -> None:
403404
self._set_status("error")
404405
self._html_status("error")
405406
self._play_done_sound("error")
407+
elif kind == EV_HINT:
408+
# Phase 1.3 — soft validation nudge. Rendered as a "hint"
409+
# role; the chat HTML / textTable show it dimmer than the
410+
# main reply so the user sees it but the agent's text
411+
# remains primary. Never blocks.
412+
if isinstance(payload, dict):
413+
msg = payload.get("message", "")
414+
else:
415+
msg = str(payload)
416+
self._append_transcript("hint", msg)
417+
self._html_append("hint", msg)
406418
elif kind == EV_STATE:
407419
self._set_status(str(payload))
408420
self._html_status(str(payload))

td_component/tdpilot_api_runtime.py

Lines changed: 116 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,11 @@ def cancel_pending(self) -> None:
151151
EV_ERROR = "error" # payload: str (redacted message)
152152
EV_STATE = "state" # payload: str ("idle"|"calling"|"thinking")
153153
EV_MODEL = "model" # payload: {"tier": str, "model": str} — Sprint 4.3
154+
# Phase 1.3 — severity-tracked validation hint. Emitted at turn end
155+
# when high-severity mutations went out without a follow-up validator
156+
# call. Soft signal; chat UI renders as a subtle nudge below the
157+
# final assistant text.
158+
EV_HINT = "hint" # payload: {"kind": str, "message": str, "tools": list[str]}
154159
# Sprint 4.1 — subagent events. Forwarded by SubagentManager into the
155160
# parent runtime's event queue so the chat UI can display sub-task
156161
# progress under collapsible [worker:<id>] sections.
@@ -161,6 +166,62 @@ def cancel_pending(self) -> None:
161166
)
162167

163168

169+
# Phase 1.3 — mutation-severity classifier. The runtime tracks tool
170+
# calls per turn; if a turn included one or more HIGH-severity
171+
# mutations without a follow-up validator call, EV_HINT fires at
172+
# turn end. Soft signal — never blocks the conversation.
173+
#
174+
# Severity rationale:
175+
# - high: changes that can leave a network in a broken state if the
176+
# model's mental model diverges from the TD reality. exec_python
177+
# in particular can do anything; create_node + delete_node +
178+
# wire/unwire mutate topology.
179+
# - medium: parameter changes that may or may not have downstream
180+
# effects depending on the operator. Currently we don't emit hints
181+
# for medium — the noise/signal ratio is too low.
182+
# - low: reads. Inspections.
183+
#
184+
# Validators that satisfy a high-severity mutation:
185+
# td_get_errors - canonical post-mutation check.
186+
# td_audit_project - whole-project sanity sweep.
187+
# td_validate_recipe - asserts recipe consistency.
188+
_TOOL_SEVERITY: dict[str, str] = {
189+
"td_create_node": "high",
190+
"td_delete_node": "high",
191+
"td_disconnect": "high",
192+
"td_connect_nodes": "high",
193+
"td_exec_python": "high",
194+
"td_set_content": "high",
195+
"td_copy_node": "high",
196+
"td_rename_node": "high",
197+
"td_create_macro": "high",
198+
"patch_begin": "high",
199+
"patch_commit": "high",
200+
"recipe_replay": "high",
201+
"td_set_params": "medium",
202+
"td_pulse_param": "medium",
203+
"td_custom_parameters": "medium",
204+
}
205+
206+
_VALIDATOR_TOOLS: frozenset[str] = frozenset(
207+
(
208+
"td_get_errors",
209+
"td_audit_project",
210+
"td_validate_recipe",
211+
"patch_validate",
212+
)
213+
)
214+
215+
216+
def _tool_severity(name: str) -> str:
217+
"""Return ``"high" | "medium" | "low"`` for a tool name. Unknown
218+
tools default to ``"low"`` (read-only assumption — they don't
219+
contribute to the validation-hint signal). Severity is data, not
220+
policy: callers decide what to do with it.
221+
"""
222+
return _TOOL_SEVERITY.get(name, "low")
223+
224+
164225
SYSTEM_PROMPT_BASE = (
165226
"You are TDPilot API, an AI assistant operating inside TouchDesigner. "
166227
"You have direct access to the TD network through tools.\n\n"
@@ -439,6 +500,14 @@ def __init__(
439500
self._dynamic_context_snapshot: list[dict] = []
440501
self._refresh_dynamic_context()
441502

503+
# Phase 1.3 — per-turn validation tracking. Worker thread fills
504+
# ``_turn_tool_calls`` via the on_tool_result callback; cook
505+
# thread reads it at turn end (in the on_turn_done handler) to
506+
# decide whether to emit EV_HINT. The list is cleared on every
507+
# ``start_turn``. Race is benign: both reads and writes happen
508+
# in producer/consumer order around the worker's lifecycle.
509+
self._turn_tool_calls: list[str] = []
510+
442511
self._agent: Agent | None = None
443512
self._build_agent()
444513

@@ -478,10 +547,12 @@ def _build_agent(self) -> None:
478547
flash_model=cfg.get("flash_model", "deepseek-v4-flash"),
479548
on_text=lambda s: self._push(EV_TEXT, s),
480549
on_tool_call=lambda n, a: self._push(EV_TOOL_CALL, {"name": n, "args": a}),
481-
on_tool_result=lambda n, r, e: self._push(
482-
EV_TOOL_RESULT, {"name": n, "result": r, "is_error": e}
550+
on_tool_result=lambda n, r, e: (
551+
self._record_tool_call(n, e),
552+
self._push(EV_TOOL_RESULT, {"name": n, "result": r, "is_error": e}),
483553
),
484554
on_turn_done=lambda s: (
555+
self._maybe_emit_validation_hint(),
485556
self._push(EV_DONE, s),
486557
self._push(EV_STATE, "idle"),
487558
),
@@ -508,6 +579,46 @@ def reload_config(self) -> None:
508579
# Phase 0.1 — dynamic context refresh (cook-thread only)
509580
# ------------------------------------------------------------------
510581

582+
# ------------------------------------------------------------------
583+
# Phase 1.3 — severity-tracked validation hints
584+
# ------------------------------------------------------------------
585+
586+
def _record_tool_call(self, name: str, is_error: bool) -> None:
587+
"""Hook called from the worker thread (on_tool_result) for every
588+
tool the agent invokes. Failed calls don't count — the model
589+
already saw the error and hasn't actually mutated state.
590+
"""
591+
if not is_error:
592+
self._turn_tool_calls.append(name)
593+
594+
def _maybe_emit_validation_hint(self) -> None:
595+
"""Inspect the just-finished turn's tool-call list. If any
596+
high-severity mutation went out without a follow-up validator
597+
call, emit ``EV_HINT`` so the chat UI can render a soft nudge.
598+
Never blocks the conversation; never fires on low/medium-only
599+
turns.
600+
"""
601+
calls = list(self._turn_tool_calls)
602+
high_severity = [name for name in calls if _tool_severity(name) == "high"]
603+
if not high_severity:
604+
return
605+
if any(name in _VALIDATOR_TOOLS for name in calls):
606+
return
607+
unique = sorted({name for name in high_severity})
608+
self._push(
609+
EV_HINT,
610+
{
611+
"kind": "missing_validation",
612+
"tools": unique,
613+
"message": (
614+
"You modified the network ("
615+
+ ", ".join(unique)
616+
+ ") without validating. Consider calling td_get_errors "
617+
"or td_audit_project to confirm the result is healthy."
618+
),
619+
},
620+
)
621+
511622
def _refresh_dynamic_context(self) -> None:
512623
"""Snapshot the per-turn volatile context on the cook thread.
513624
@@ -544,6 +655,9 @@ def start_turn(self, user_text: str) -> bool:
544655
# call would re-trigger TD's THREAD CONFLICT detector when the
545656
# bundled-knowledge enumerator hits parent().op('kb').
546657
self._refresh_dynamic_context()
658+
# Phase 1.3 — clear the per-turn tool-call ledger so the
659+
# validation-hint check at turn end only considers THIS turn.
660+
self._turn_tool_calls = []
547661
self._agent.add_user_message(user_text)
548662
self._push(EV_STATE, "thinking")
549663
self._worker = threading.Thread(target=self._run_safe, name="tdpilot_api_agent", daemon=True)

0 commit comments

Comments
 (0)