|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +from dataclasses import asdict, dataclass, field |
| 4 | +from datetime import datetime, timezone |
| 5 | +from typing import TYPE_CHECKING, Any, Literal |
| 6 | + |
| 7 | +from corral.backend.schema import ToolCall, ToolCallStatus |
| 8 | + |
| 9 | +if TYPE_CHECKING: |
| 10 | + from corral.backend.task import TaskDefinition |
| 11 | + |
| 12 | +TrialStatus = Literal[ |
| 13 | + "created", |
| 14 | + "running", |
| 15 | + "submitted", |
| 16 | + "surrendered", |
| 17 | + "scored", |
| 18 | + "finalized", |
| 19 | +] |
| 20 | + |
| 21 | + |
| 22 | +def utcnow() -> datetime: |
| 23 | + return datetime.now(tz=timezone.utc) |
| 24 | + |
| 25 | + |
| 26 | +def get_path(data: Any, path: str) -> Any: |
| 27 | + """Resolve a dotted path (e.g. ``"answer.smiles"``) into nested data.""" |
| 28 | + current = data |
| 29 | + for part in path.split("."): |
| 30 | + current = current[part] |
| 31 | + return current |
| 32 | + |
| 33 | + |
| 34 | +@dataclass |
| 35 | +class TaskRunState: |
| 36 | + """Runtime outcome of one task: its output, score, and feedback.""" |
| 37 | + |
| 38 | + task_id: str |
| 39 | + output: dict[str, Any] = field(default_factory=dict) |
| 40 | + score: float | None = None |
| 41 | + feedback: str | None = None |
| 42 | + |
| 43 | + |
| 44 | +@dataclass |
| 45 | +class CorralState: |
| 46 | + """Single source of truth for one Corral runtime session. |
| 47 | +
|
| 48 | + This is the only mutable runtime state object. Environments, routers, |
| 49 | + tools, and scoring code should read/write through this class. |
| 50 | + Task definitions remain immutable configuration; this class stores runtime data. |
| 51 | + """ |
| 52 | + |
| 53 | + # Identity |
| 54 | + task_id: str |
| 55 | + task_prompt: str | list[dict[str, Any]] |
| 56 | + trial_id: str = "0" |
| 57 | + trial_counter: int = -1 |
| 58 | + run_id: str | None = None |
| 59 | + |
| 60 | + # Optional task-group identity |
| 61 | + task_group_id: str | None = None |
| 62 | + |
| 63 | + # Trial lifecycle |
| 64 | + status: TrialStatus = "created" |
| 65 | + started_at: datetime = field(default_factory=utcnow) |
| 66 | + ended_at: datetime | None = None |
| 67 | + |
| 68 | + # Agent/environment interaction trace |
| 69 | + messages: list[dict[str, Any]] = field(default_factory=list) |
| 70 | + tool_calls: list[ToolCall] = field(default_factory=list) |
| 71 | + |
| 72 | + # Submission/scoring |
| 73 | + submitted_answer: str | None = None |
| 74 | + score: float | None = None |
| 75 | + feedback: str | None = None |
| 76 | + surrendered: bool = False |
| 77 | + is_attempted: bool = False |
| 78 | + |
| 79 | + # Per-task runtime outcomes; the dict object is shared between linked |
| 80 | + # environments so chained tasks see their dependencies' outputs |
| 81 | + task_runs: dict[str, TaskRunState] = field(default_factory=dict) |
| 82 | + |
| 83 | + # Runtime resources |
| 84 | + workspace: str | None = None |
| 85 | + artifacts: dict[str, Any] = field(default_factory=dict) |
| 86 | + hidden_args: dict[str, Any] = field(default_factory=dict) |
| 87 | + |
| 88 | + # Archived trial snapshots |
| 89 | + trials: dict[str, dict[str, Any]] = field(default_factory=dict) |
| 90 | + |
| 91 | + def start_new_trial( |
| 92 | + self, |
| 93 | + task_prompt: str | list[dict[str, Any]], |
| 94 | + workspace: str | None = None, |
| 95 | + ) -> str: |
| 96 | + """Archive current completed trial, then start a clean trial.""" |
| 97 | + if self.status in {"submitted", "surrendered", "scored", "finalized"}: |
| 98 | + self.finalize() |
| 99 | + self.trials[self.trial_id] = self.snapshot(include_trials=False) |
| 100 | + |
| 101 | + self.trial_counter += 1 |
| 102 | + self.trial_id = str(self.trial_counter) |
| 103 | + self.task_prompt = task_prompt |
| 104 | + self.workspace = workspace |
| 105 | + |
| 106 | + self.status = "running" |
| 107 | + self.started_at = utcnow() |
| 108 | + self.ended_at = None |
| 109 | + |
| 110 | + self.messages.clear() |
| 111 | + self.tool_calls.clear() |
| 112 | + self.submitted_answer = None |
| 113 | + self.score = None |
| 114 | + self.feedback = None |
| 115 | + self.surrendered = False |
| 116 | + self.is_attempted = False |
| 117 | + |
| 118 | + return self.trial_id |
| 119 | + |
| 120 | + def record_message(self, role: str, content: str, **metadata: Any) -> None: |
| 121 | + self.messages.append( |
| 122 | + { |
| 123 | + "role": role, |
| 124 | + "content": content, |
| 125 | + "timestamp": utcnow().isoformat(), |
| 126 | + **metadata, |
| 127 | + } |
| 128 | + ) |
| 129 | + |
| 130 | + def record_tool_call(self, call: ToolCall) -> None: |
| 131 | + self.tool_calls.append(call) |
| 132 | + |
| 133 | + def submit(self, answer: str) -> None: |
| 134 | + self.submitted_answer = answer |
| 135 | + self.is_attempted = True |
| 136 | + self.status = "submitted" |
| 137 | + self.ended_at = utcnow() |
| 138 | + |
| 139 | + def surrender(self) -> float: |
| 140 | + self.surrendered = True |
| 141 | + self.is_attempted = True |
| 142 | + self.status = "surrendered" |
| 143 | + self.ended_at = utcnow() |
| 144 | + return 0.0 if self.score is None else self.score |
| 145 | + |
| 146 | + def set_score(self, score: float, feedback: str | None = None) -> None: |
| 147 | + self.score = score |
| 148 | + self.feedback = feedback |
| 149 | + self.status = "scored" |
| 150 | + |
| 151 | + def store_task_output( |
| 152 | + self, |
| 153 | + task_id: str, |
| 154 | + answer: Any, |
| 155 | + score: float | None = None, |
| 156 | + feedback: str | None = None, |
| 157 | + ) -> None: |
| 158 | + """Record a task's output so dependent tasks can consume it.""" |
| 159 | + self.task_runs[task_id] = TaskRunState( |
| 160 | + task_id=task_id, |
| 161 | + output={"answer": answer}, |
| 162 | + score=score, |
| 163 | + feedback=feedback, |
| 164 | + ) |
| 165 | + |
| 166 | + def get_output(self, task_id: str, key: str = "answer") -> Any: |
| 167 | + """Read one field of a task's output; `key` may be a dotted path.""" |
| 168 | + if task_id not in self.task_runs: |
| 169 | + raise KeyError(f"Task {task_id!r} has no stored output") |
| 170 | + try: |
| 171 | + return get_path(self.task_runs[task_id].output, key) |
| 172 | + except (KeyError, TypeError) as e: |
| 173 | + raise KeyError(f"Task {task_id!r} has no output key {key!r}") from e |
| 174 | + |
| 175 | + def is_completed(self, task_id: str) -> bool: |
| 176 | + return task_id in self.task_runs |
| 177 | + |
| 178 | + def dependencies_satisfied(self, task: TaskDefinition) -> bool: |
| 179 | + return all(self.is_completed(dep_id) for dep_id in task.dependencies()) |
| 180 | + |
| 181 | + def resolve_inputs(self, task: TaskDefinition) -> dict[str, Any]: |
| 182 | + """Combine a task's initial input with its resolved dependency outputs.""" |
| 183 | + resolved = dict(task.initial_input) |
| 184 | + |
| 185 | + for input_name, ref in task.input_map.items(): |
| 186 | + if not self.is_completed(ref.task_id): |
| 187 | + raise RuntimeError( |
| 188 | + f"Task {task.name!r} is not ready: dependency " |
| 189 | + f"{ref.task_id!r} has not completed." |
| 190 | + ) |
| 191 | + resolved[input_name] = self.get_output(ref.task_id, ref.key) |
| 192 | + |
| 193 | + return resolved |
| 194 | + |
| 195 | + def finalize(self) -> None: |
| 196 | + if self.ended_at is None: |
| 197 | + self.ended_at = utcnow() |
| 198 | + self.status = "finalized" |
| 199 | + |
| 200 | + def duration_seconds(self) -> float | None: |
| 201 | + if self.ended_at is None: |
| 202 | + return None |
| 203 | + return (self.ended_at - self.started_at).total_seconds() |
| 204 | + |
| 205 | + def tool_statistics(self) -> dict[str, Any]: |
| 206 | + return { |
| 207 | + "total_calls": len(self.tool_calls), |
| 208 | + "successful_calls": len( |
| 209 | + [t for t in self.tool_calls if t.status == ToolCallStatus.SUCCESS] |
| 210 | + ), |
| 211 | + "failed_calls": len( |
| 212 | + [t for t in self.tool_calls if t.status != ToolCallStatus.SUCCESS] |
| 213 | + ), |
| 214 | + "tools_used": sorted({t.tool_name for t in self.tool_calls}), |
| 215 | + "error_types": { |
| 216 | + status.value: len([t for t in self.tool_calls if t.status == status]) |
| 217 | + for status in ToolCallStatus |
| 218 | + if status != ToolCallStatus.SUCCESS |
| 219 | + }, |
| 220 | + "tool_calls": [ |
| 221 | + { |
| 222 | + "tool_name": call.tool_name, |
| 223 | + "arguments": call.arguments, |
| 224 | + "result": call.result, |
| 225 | + "status": call.status.value, |
| 226 | + "error_message": call.error_message, |
| 227 | + "duration": call.duration, |
| 228 | + "timestamp": ( |
| 229 | + call.timestamp.isoformat() if call.timestamp else None |
| 230 | + ), |
| 231 | + } |
| 232 | + for call in self.tool_calls |
| 233 | + ], |
| 234 | + } |
| 235 | + |
| 236 | + def snapshot(self, include_trials: bool = True) -> dict[str, Any]: |
| 237 | + data = asdict(self) |
| 238 | + # Hidden args can hold scoring secrets and arbitrary objects; never expose them. |
| 239 | + data.pop("hidden_args", None) |
| 240 | + data["started_at"] = self.started_at.isoformat() |
| 241 | + data["ended_at"] = self.ended_at.isoformat() if self.ended_at else None |
| 242 | + data["duration"] = self.duration_seconds() |
| 243 | + data["tool_statistics"] = self.tool_statistics() |
| 244 | + # ToolCall records are already serialized inside tool_statistics |
| 245 | + data.pop("tool_calls", None) |
| 246 | + if not include_trials: |
| 247 | + data.pop("trials", None) |
| 248 | + return data |
0 commit comments