feat: Implement runtime observability metrics and dashboard specifications

Author: mnkiefer

actions/setup/js/generate_observability_summary.cjs

@@ -1,133 +1,19 @@
 // @ts-check
 /// <reference types="@actions/github-script" />
 
-const fs = require("fs");
 const { main: exportCopilotOtelTraces } = require("./export_copilot_otel_traces.cjs");
-
-const AW_INFO_PATH = "/tmp/gh-aw/aw_info.json";
-const AGENT_OUTPUT_PATH = "/tmp/gh-aw/agent_output.json";
-const gatewayEventPaths = ["/tmp/gh-aw/mcp-logs/gateway.jsonl", "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"];
-
-function readJSONIfExists(path) {
-  if (!fs.existsSync(path)) {
-    return null;
-  }
-
-  try {
-    return JSON.parse(fs.readFileSync(path, "utf8"));
-  } catch {
-    return null;
-  }
-}
-
-function countBlockedRequests() {
-  let total = 0;
-
-  for (const path of gatewayEventPaths) {
-    if (!fs.existsSync(path)) {
-      continue;
-    }
-
-    const lines = fs.readFileSync(path, "utf8").split("\n");
-    for (const raw of lines) {
-      const line = raw.trim();
-      if (!line) continue;
-      try {
-        const entry = JSON.parse(line);
-        if (entry && entry.type === "DIFC_FILTERED") total++;
-      } catch {
-        // skip malformed lines
-      }
-    }
-  }
-
-  return total;
-}
-
-function uniqueCreatedItemTypes(items) {
-  const types = new Set();
-
-  for (const item of items) {
-    if (item && typeof item.type === "string" && item.type.trim() !== "") {
-      types.add(item.type);
-    }
-  }
-
-  return [...types].sort();
-}
-
-function collectObservabilityData() {
-  const awInfo = readJSONIfExists(AW_INFO_PATH) || {};
-  const agentOutput = readJSONIfExists(AGENT_OUTPUT_PATH) || { items: [], errors: [] };
-  const items = Array.isArray(agentOutput.items) ? agentOutput.items : [];
-  const errors = Array.isArray(agentOutput.errors) ? agentOutput.errors : [];
-  // Prefer GITHUB_AW_OTEL_TRACE_ID (written to GITHUB_ENV by action_setup_otlp.cjs)
-  // so the summary always shows the trace ID that is actually present in the OTLP backend.
-  // Fall back to context.otel_trace_id for cross-workflow traces propagated from a parent.
-  // Do NOT fall back to workflow_call_id — it is not a valid OTLP trace ID.
-  const traceId = process.env.GITHUB_AW_OTEL_TRACE_ID || (awInfo.context ? awInfo.context.otel_trace_id || "" : "");
-
-  return {
-    workflowName: awInfo.workflow_name || "",
-    engineId: awInfo.engine_id || "",
-    traceId,
-    staged: awInfo.staged === true,
-    firewallEnabled: awInfo.firewall_enabled === true,
-    createdItemCount: items.length,
-    createdItemTypes: uniqueCreatedItemTypes(items),
-    outputErrorCount: errors.length,
-    blockedRequests: countBlockedRequests(),
-  };
-}
-
-function buildObservabilitySummary(data) {
-  const posture = data.createdItemCount > 0 ? "write-capable" : "read-only";
-  const lines = [];
-
-  lines.push("<details>");
-  lines.push("<summary>Observability</summary>");
-  lines.push("");
-
-  if (data.workflowName) {
-    lines.push(`- **workflow**: ${data.workflowName}`);
-  }
-  if (data.engineId) {
-    lines.push(`- **engine**: ${data.engineId}`);
-  }
-  if (data.traceId) {
-    lines.push(`- **trace id**: ${data.traceId}`);
-  }
-
-  lines.push(`- **posture**: ${posture}`);
-  lines.push(`- **created items**: ${data.createdItemCount}`);
-  lines.push(`- **blocked requests**: ${data.blockedRequests}`);
-  lines.push(`- **agent output errors**: ${data.outputErrorCount}`);
-  lines.push(`- **firewall enabled**: ${data.firewallEnabled}`);
-  lines.push(`- **staged**: ${data.staged}`);
-
-  if (data.createdItemTypes.length > 0) {
-    lines.push("- **item types**:");
-    for (const itemType of data.createdItemTypes) {
-      lines.push(`  - ${itemType}`);
-    }
-  }
-
-  lines.push("");
-  lines.push("</details>");
-
-  return lines.join("\n") + "\n";
-}
+const { buildObservabilitySummary, collectRuntimeObservabilityData } = require("./runtime_observability.cjs");
 
 async function main(core) {
   await exportCopilotOtelTraces(core);
-  const data = collectObservabilityData();
+  const data = collectRuntimeObservabilityData();
   const markdown = buildObservabilitySummary(data);
   await core.summary.addRaw(markdown).write();
   core.info("Generated observability summary in step summary");
 }
 
 module.exports = {
   buildObservabilitySummary,
-  collectObservabilityData,
+  collectObservabilityData: collectRuntimeObservabilityData,
   main,
 };

actions/setup/js/generate_observability_summary.test.cjs

@@ -21,7 +21,7 @@ describe("generate_observability_summary.cjs", () => {
   });
 
   afterEach(() => {
-    for (const path of ["/tmp/gh-aw/aw_info.json", "/tmp/gh-aw/agent_output.json", "/tmp/gh-aw/mcp-logs/gateway.jsonl", "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"]) {
+    for (const path of ["/tmp/gh-aw/aw_info.json", "/tmp/gh-aw/agent_output.json", "/tmp/gh-aw/agent_usage.json", "/tmp/gh-aw/agent-stdio.log", "/tmp/gh-aw/mcp-logs/gateway.jsonl", "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"]) {
       if (fs.existsSync(path)) {
         fs.unlinkSync(path);
       }
@@ -46,6 +46,8 @@ describe("generate_observability_summary.cjs", () => {
         errors: ["validation failed"],
       })
     );
+    fs.writeFileSync("/tmp/gh-aw/agent_usage.json", JSON.stringify({ input_tokens: 1000, output_tokens: 200, cache_read_tokens: 1000, cache_write_tokens: 50, effective_tokens: 500 }));
+    fs.writeFileSync("/tmp/gh-aw/agent-stdio.log", '[WARN] first warning\n{"type":"result","num_turns":7,"total_cost_usd":1.75}\n');
     fs.writeFileSync("/tmp/gh-aw/mcp-logs/gateway.jsonl", [JSON.stringify({ type: "DIFC_FILTERED" }), JSON.stringify({ type: "REQUEST" })].join("\n"));
 
     await module.main(mockCore);
@@ -58,6 +60,13 @@ describe("generate_observability_summary.cjs", () => {
     expect(summary).toContain("- **trace id**: a3f2c8d1e4b7091f6a5c2e3d8f401b72");
     expect(summary).not.toContain("12345678901-1");
     expect(summary).toContain("- **posture**: write-capable");
+    expect(summary).toContain("- **runtime status**: error");
+    expect(summary).toContain("- **total tokens**: 500");
+    expect(summary).toContain("- **estimated cost usd**: 1.75");
+    expect(summary).toContain("- **turns**: 7");
+    expect(summary).toContain("- **cache efficiency**: 50%");
+    expect(summary).toContain("- **runtime risk score**: 31");
+    expect(summary).toContain("- **optimization score**: 31");
     expect(summary).toContain("- **created items**: 2");
     expect(summary).toContain("- **blocked requests**: 1");
     expect(summary).toContain("- **agent output errors**: 1");