feat(analytics): track recoverable task outcomes (#6411) (#6473)

(cherry picked from commit e6aeb38)

Co-authored-by: open-design-crew[bot] <299007234+open-design-crew[bot]@users.noreply.github.com>
Co-authored-by: bone3deep1962-collab <bone3deep1962@gmail.com>

Author: 3 people

apps/daemon/src/db.ts

@@ -216,6 +216,7 @@ function migrate(db: SqliteDb): void {
       pre_turn_file_names_json TEXT,
       session_mode TEXT,
       run_context_json TEXT,
+      task_analytics_json TEXT,
       applied_plugin_snapshot_json TEXT,
       telemetry_finalized_at INTEGER,
       started_at INTEGER,
@@ -417,6 +418,9 @@ function migrate(db: SqliteDb): void {
   if (!messageCols.some((c: DbRow) => c.name === 'run_context_json')) {
     db.exec(`ALTER TABLE messages ADD COLUMN run_context_json TEXT`);
   }
+  if (!messageCols.some((c: DbRow) => c.name === 'task_analytics_json')) {
+    db.exec(`ALTER TABLE messages ADD COLUMN task_analytics_json TEXT`);
+  }
   if (!messageCols.some((c: DbRow) => c.name === 'applied_plugin_snapshot_json')) {
     db.exec(`ALTER TABLE messages ADD COLUMN applied_plugin_snapshot_json TEXT`);
   }
@@ -2512,6 +2516,7 @@ export function listMessages(db: SqliteDb, conversationId: string) {
               pre_turn_file_names_json AS preTurnFileNamesJson,
               session_mode AS sessionMode,
               run_context_json AS runContextJson,
+              task_analytics_json AS taskAnalyticsJson,
               applied_plugin_snapshot_json AS appliedPluginSnapshotJson,
               created_at AS createdAt, started_at AS startedAt, ended_at AS endedAt,
               position
@@ -2565,7 +2570,8 @@ export function upsertMessage(db: SqliteDb, conversationId: string, m: DbRow) {
               events_json = ?, attachments_json = ?, comment_attachments_json = ?,
               produced_files_json = ?, trace_object_files_json = ?, feedback_json = ?,
               pre_turn_file_names_json = ?,
-              session_mode = ?, run_context_json = ?, applied_plugin_snapshot_json = ?,
+              session_mode = ?, run_context_json = ?, task_analytics_json = ?,
+              applied_plugin_snapshot_json = ?,
               telemetry_finalized_at = CASE
                 WHEN ? THEN COALESCE(telemetry_finalized_at, ?)
                 ELSE telemetry_finalized_at
@@ -2590,6 +2596,7 @@ export function upsertMessage(db: SqliteDb, conversationId: string, m: DbRow) {
       m.preTurnFileNames ? JSON.stringify(m.preTurnFileNames) : null,
       normalizeMessageSessionModeForStorage(m.sessionMode),
       m.runContext ? JSON.stringify(m.runContext) : null,
+      m.taskAnalytics ? JSON.stringify(m.taskAnalytics) : null,
       m.appliedPluginSnapshot ? JSON.stringify(m.appliedPluginSnapshot) : null,
       m.telemetryFinalized === true ? 1 : 0,
       now,
@@ -2611,17 +2618,18 @@ export function upsertMessage(db: SqliteDb, conversationId: string, m: DbRow) {
     // run_id, run_status, result_delivery_state, last_run_event_id, events_json, attachments_json,
     // comment_attachments_json, produced_files_json, trace_object_files_json,
     // feedback_json, pre_turn_file_names_json, session_mode, run_context_json,
-    // applied_plugin_snapshot_json, telemetry_finalized_at, started_at,
-    // ended_at, position, created_at.
+    // task_analytics_json, applied_plugin_snapshot_json,
+    // telemetry_finalized_at, started_at, ended_at, position, created_at.
     db.prepare(
       `INSERT INTO messages
          (id, conversation_id, role, content, agent_id, agent_name,
           run_id, run_status, result_delivery_state, last_run_event_id, events_json,
           attachments_json, comment_attachments_json, produced_files_json,
           trace_object_files_json, feedback_json, pre_turn_file_names_json,
-          session_mode, run_context_json, applied_plugin_snapshot_json,
+          session_mode, run_context_json, task_analytics_json,
+          applied_plugin_snapshot_json,
           telemetry_finalized_at, started_at, ended_at, position, created_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
     ).run(
       m.id,
       conversationId,
@@ -2642,6 +2650,7 @@ export function upsertMessage(db: SqliteDb, conversationId: string, m: DbRow) {
       m.preTurnFileNames ? JSON.stringify(m.preTurnFileNames) : null,
       normalizeMessageSessionModeForStorage(m.sessionMode),
       m.runContext ? JSON.stringify(m.runContext) : null,
+      m.taskAnalytics ? JSON.stringify(m.taskAnalytics) : null,
       m.appliedPluginSnapshot ? JSON.stringify(m.appliedPluginSnapshot) : null,
       m.telemetryFinalized === true ? now : null,
       m.startedAt ?? null,
@@ -2670,6 +2679,7 @@ export function upsertMessage(db: SqliteDb, conversationId: string, m: DbRow) {
               pre_turn_file_names_json AS preTurnFileNamesJson,
               session_mode AS sessionMode,
               run_context_json AS runContextJson,
+              task_analytics_json AS taskAnalyticsJson,
               applied_plugin_snapshot_json AS appliedPluginSnapshotJson,
               created_at AS createdAt, started_at AS startedAt, ended_at AS endedAt,
               position
@@ -3675,6 +3685,7 @@ function normalizeMessage(row: DbRow) {
     preTurnFileNames: parseJsonOrUndef(row.preTurnFileNamesJson),
     sessionMode: normalizeMessageSessionMode(row.sessionMode),
     runContext: parseJsonOrUndef(row.runContextJson),
+    taskAnalytics: parseJsonOrUndef(row.taskAnalyticsJson),
     appliedPluginSnapshot: parseJsonOrUndef(row.appliedPluginSnapshotJson),
     createdAt: row.createdAt ?? undefined,
     startedAt: row.startedAt ?? undefined,

apps/daemon/src/routes/runs.ts

@@ -15,12 +15,16 @@ import {
   type RunResultPackageResponse,
 } from '@open-design/contracts';
 import {
+  buildRunCreatedV4Aliases,
+  buildRunFinishedV4Aliases,
   deriveConfigureGlobals,
   modelIdForTracking,
   sessionModeToTracking,
   type TrackingDesignSystemSource,
   type TrackingDesignSystemKind,
   type TrackingDesignSystemEditSurface,
+  type RunTaskLineageProps,
+  type TrackingRunRecoveryActionType,
 } from '@open-design/contracts/analytics';
 import type { OdNativeEvent } from '@open-design/agui-adapter';
 import { newInsertId, readAnalyticsContext } from '../analytics.js';
@@ -87,7 +91,10 @@ import {
 } from '../run-analytics-observability.js';
 import {
   diffRunArtifacts,
+  primaryArtifactChangeForRun,
   snapshotProjectArtifacts,
+  supportingAssetFilesChangedForRun,
+  type RunArtifactDiff,
   type RunArtifactBaseline,
 } from '../run-artifact-fs.js';
 import {
@@ -342,6 +349,7 @@ interface ChatRun {
     artifactsModified?: number;
     designSystemCreated: boolean;
     previewModuleCount: number;
+    diff?: RunArtifactDiff;
   };
   designSystemId?: string | null;
   designSystemRequestedId?: string | null;
@@ -1875,6 +1883,46 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
       const hintProjectTurnIndex = typeof analyticsHints.projectTurnIndex === 'number'
         ? analyticsHints.projectTurnIndex
         : undefined;
+      const taskExecutionId = typeof analyticsHints.taskExecutionId === 'string'
+        && analyticsHints.taskExecutionId.length > 0
+        ? analyticsHints.taskExecutionId
+        : run.clientRequestId ?? run.id;
+      const initialRunId = typeof analyticsHints.initialRunId === 'string'
+        && analyticsHints.initialRunId.length > 0
+        ? analyticsHints.initialRunId
+        : run.id;
+      const taskRunIndex = typeof analyticsHints.taskRunIndex === 'number'
+        && Number.isInteger(analyticsHints.taskRunIndex)
+        && analyticsHints.taskRunIndex >= 0
+        ? analyticsHints.taskRunIndex
+        : 0;
+      const recoveryActionTypes: ReadonlySet<TrackingRunRecoveryActionType> = new Set([
+        'manual_retry',
+        'resume_run',
+        'authorize_and_retry',
+        'switch_model_retry',
+        'switch_runtime_retry',
+        'question_answer',
+      ]);
+      const recoveryActionType = typeof analyticsHints.recoveryActionType === 'string'
+        && recoveryActionTypes.has(
+          analyticsHints.recoveryActionType as TrackingRunRecoveryActionType,
+        )
+        ? analyticsHints.recoveryActionType as TrackingRunRecoveryActionType
+        : undefined;
+      const taskLineage: RunTaskLineageProps = {
+        task_execution_id: taskExecutionId,
+        initial_run_id: initialRunId,
+        task_run_index: taskRunIndex,
+        ...(typeof analyticsHints.sourceRunId === 'string' && analyticsHints.sourceRunId.length > 0
+          ? { source_run_id: analyticsHints.sourceRunId }
+          : {}),
+        ...(recoveryActionType ? { recovery_action_type: recoveryActionType } : {}),
+        ...(typeof analyticsHints.recoveryActionInstanceId === 'string'
+          && analyticsHints.recoveryActionInstanceId.length > 0
+          ? { recovery_action_instance_id: analyticsHints.recoveryActionInstanceId }
+          : {}),
+      };
       const conversationTurnIndex = run.conversationId
         ? conversationTurnIndexForRun(db, run.conversationId, run.id)
         : null;
@@ -2088,6 +2136,7 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
             }
           : {}),
       };
+      Object.assign(baseProps, buildRunCreatedV4Aliases(baseProps, taskLineage));
       design.runs.setAnalyticsRecovery?.(run, {
         context: analyticsContext,
         properties: baseProps,
@@ -2170,7 +2219,9 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
         // in the rollout `last_token_usage`, read here best-effort.
         const firstCallUsage = await (async (): Promise<{
           first_call_input_tokens?: number;
+          first_call_input_tokens_effective?: number;
           first_call_cache_read_input_tokens?: number;
+          first_call_cache_creation_input_tokens?: number;
           first_call_cache_hit_ratio?: number;
         } | null> => {
           if (run.agentId === 'codex') {
@@ -2187,20 +2238,39 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
                   'codex',
                 ),
               ).CODEX_HOME;
-              return await readCodexRolloutFirstCall({ codexHome, sessionId });
+              const codexUsage = await readCodexRolloutFirstCall({ codexHome, sessionId });
+              return codexUsage
+                ? {
+                    ...codexUsage,
+                    first_call_input_tokens_effective:
+                      codexUsage.first_call_input_tokens,
+                  }
+                : null;
             } catch {
               return null;
             }
           }
           if (usageAnalytics.first_call_input_tokens === undefined) return null;
           return {
             first_call_input_tokens: usageAnalytics.first_call_input_tokens,
+            ...(usageAnalytics.first_call_input_tokens_effective !== undefined
+              ? {
+                  first_call_input_tokens_effective:
+                    usageAnalytics.first_call_input_tokens_effective,
+                }
+              : {}),
             ...(usageAnalytics.first_call_cache_read_input_tokens !== undefined
               ? {
                   first_call_cache_read_input_tokens:
                     usageAnalytics.first_call_cache_read_input_tokens,
                 }
               : {}),
+            ...(usageAnalytics.first_call_cache_creation_input_tokens !== undefined
+              ? {
+                  first_call_cache_creation_input_tokens:
+                    usageAnalytics.first_call_cache_creation_input_tokens,
+                }
+              : {}),
             ...(usageAnalytics.first_call_cache_hit_ratio !== undefined
               ? { first_call_cache_hit_ratio: usageAnalytics.first_call_cache_hit_ratio }
               : {}),
@@ -2225,13 +2295,15 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
         let artifactsModified: number | undefined;
         let designSystemCreated: boolean;
         let previewModuleCount: number;
+        let artifactDiff: RunArtifactDiff | undefined;
         const artifactOutcome = run.artifactOutcome;
         if (artifactOutcome) {
           artifactCount = artifactOutcome.artifactCount;
           artifactsCreated = artifactOutcome.artifactsCreated;
           artifactsModified = artifactOutcome.artifactsModified;
           designSystemCreated = artifactOutcome.designSystemCreated;
           previewModuleCount = artifactOutcome.previewModuleCount;
+          artifactDiff = artifactOutcome.diff;
         } else {
           const artifactBaseline = runArtifactBaselines.take(run.id);
           if (artifactBaseline && !artifactBaseline.contended) {
@@ -2245,6 +2317,7 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
               diff = null;
             }
             if (diff) {
+              artifactDiff = diff;
               artifactCount = diff.touched;
               artifactsCreated = diff.created;
               artifactsModified = diff.modified;
@@ -2309,7 +2382,23 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
             insertId: `${runInsertId}-${retryEvent.event}-${index}`,
           });
         }
-        const finishedProperties = {
+        const clarificationRequested = runAskedUserQuestion(run.events);
+        const interactionMode = typeof reqBody.sessionMode === 'string'
+          ? sessionModeToTracking(reqBody.sessionMode)
+          : undefined;
+        const primaryArtifactChange = artifactDiff
+          ? primaryArtifactChangeForRun({
+              diff: artifactDiff,
+              projectKind: runProjectKind,
+              hadExistingArtifacts: hintHasExistingArtifact === true,
+              ...(interactionMode ? { interactionMode } : {}),
+              clarificationRequested,
+            })
+          : undefined;
+        const supportingAssetFilesChanged = artifactDiff
+          ? supportingAssetFilesChangedForRun(artifactDiff, runProjectKind)
+          : undefined;
+        const finishedProperties: Record<string, unknown> = {
             ...baseProps,
             design_system_id: run.designSystemId ?? undefined,
             design_system_digest: run.designSystemDigest ?? undefined,
@@ -2345,7 +2434,7 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
               : {}),
             ...(artifactsCreated !== undefined ? { artifacts_created: artifactsCreated } : {}),
             ...(artifactsModified !== undefined ? { artifacts_modified: artifactsModified } : {}),
-            asked_user_question: runAskedUserQuestion(run.events),
+            asked_user_question: clarificationRequested,
             retry_attempt_count: run.retryAttemptCount ?? 0,
             retry_final_result: run.retryFinalResult ?? 'not_attempted',
             ...(agentCliVersion
@@ -2444,6 +2533,55 @@ export function registerRunRoutes(app: Express, ctx: RegisterRunRoutesDeps) {
             tool_name_count: toolAnalytics.tool_name_count,
             tool_names: toolAnalytics.tool_names_csv,
           };
+        Object.assign(
+          finishedProperties,
+          buildRunFinishedV4Aliases(finishedProperties, taskLineage, {
+            inputAccountingMode: usageAnalytics.input_accounting_mode,
+            ...(firstCallUsage
+              ? {
+                  firstModelCall: {
+                    ...(firstCallUsage.first_call_input_tokens !== undefined
+                      ? { provider_input_tokens: firstCallUsage.first_call_input_tokens }
+                      : {}),
+                    ...(firstCallUsage.first_call_input_tokens_effective !== undefined
+                      ? { effective_input_tokens: firstCallUsage.first_call_input_tokens_effective }
+                      : {}),
+                    ...(firstCallUsage.first_call_cache_read_input_tokens !== undefined
+                      ? { cache_read_tokens: firstCallUsage.first_call_cache_read_input_tokens }
+                      : {}),
+                    ...(firstCallUsage.first_call_cache_creation_input_tokens !== undefined
+                      ? { cache_write_tokens: firstCallUsage.first_call_cache_creation_input_tokens }
+                      : {}),
+                  },
+                }
+              : {}),
+            ...(primaryArtifactChange
+              ? { primaryArtifactChange }
+              : {}),
+            ...(artifactDiff
+              ? {
+                  artifactFiles: {
+                    changed_file_count: artifactDiff.contentTouched,
+                    created_file_count: artifactDiff.contentCreated,
+                    modified_file_count: artifactDiff.contentModified,
+                    ...(supportingAssetFilesChanged !== undefined
+                      ? {
+                          supporting_asset_files_changed_count:
+                            supportingAssetFilesChanged,
+                        }
+                      : {}),
+                  },
+                }
+              : {}),
+            ...(isDesignSystemRun
+              ? {
+                  designSystemChangeType: designSystemCreated
+                    ? hintHasExistingArtifact === true ? 'modified' : 'created'
+                    : 'none',
+                }
+              : {}),
+          }),
+        );
         // Refresh local recovery snapshot so crash recovery matches PostHog
         // `run_finished` (usage/timing/tools), not only run_created baseProps.
         // Keep the base insertId here: reconcileDurableRunTerminals appends