feat: aggregate AI credits from aggregated usage JSONL files in conclusion post-step

actions/setup/js/send_otlp_span.cjs

@@ -1396,6 +1396,20 @@ const OTLP_EXPORT_ERROR_DETAILS_PATH = "/tmp/gh-aw/otlp-export-errors.jsonl";
  */
 const FAILURE_CATEGORIES_PATH = "/tmp/gh-aw/failure_categories.json";
 
+/**
+ * Path to the per-agent AI usage JSONL file written to the temp folder.
+ * Read by the conclusion job post-step before /tmp/gh-aw/ is deleted.
+ * @type {string}
+ */
+const AGENTS_USAGE_JSONL_PATH = "/tmp/gh-aw/agent_usage.jsonl";
+
+/**
+ * Path to the detection job AI usage JSONL file written to the temp folder.
+ * Read by the conclusion job post-step before /tmp/gh-aw/ is deleted.
+ * @type {string}
+ */
+const DETECTION_USAGE_JSONL_PATH = "/tmp/gh-aw/detection_usage.jsonl";
+
 /**
  * Path to the agent stdio log file.
  * @type {string}
@@ -1705,6 +1719,45 @@ function normalizeRuntimeTokenUsage(rawUsage) {
   return Object.keys(normalized).length > 0 ? normalized : undefined;
 }
 
+/**
+ * Parse a JSONL usage file and return the total AI credits consumed.
+ *
+ * Each line is expected to be a JSON object containing an `ai_credits` or
+ * `aiCredits` field.  Lines that are missing, malformed, or lack a valid
+ * non-negative numeric AI-credits field are silently skipped.
+ *
+ * Used by the conclusion job post-step to read AI usage data from
+ * `/tmp/gh-aw/agent_usage.jsonl` and `/tmp/gh-aw/detection_usage.jsonl`
+ * before the temp folder is deleted.
+ *
+ * @param {string} filePath - Absolute path to the JSONL usage file
+ * @returns {number} Total AI credits summed across all valid entries (0 when the
+ *   file is absent, empty, or contains no entries with a valid AI-credits field)
+ */
+function parseAICreditsFromUsageJsonl(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, "utf8");
+    if (!content.trim()) return 0;
+    let total = 0;
+    for (const line of content.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const entry = JSON.parse(trimmed);
+        if (!entry || typeof entry !== "object") continue;
+        const raw = "ai_credits" in entry ? entry.ai_credits : "aiCredits" in entry ? entry.aiCredits : undefined;
+        const parsed = normalizeNonNegativeNumber(raw);
+        if (typeof parsed === "number") total += parsed;
+      } catch {
+        // ignore malformed lines
+      }
+    }
+    return total;
+  } catch {
+    return 0;
+  }
+}
+
 /**
  * Read steering-event count from the first available API proxy event-log JSONL.
  *
@@ -2081,6 +2134,21 @@ async function sendJobConclusionSpan(spanName, options = {}) {
   if (typeof aiCredits === "number") {
     attributes.push(buildAttr("gh-aw.aic", aiCredits));
   }
+  // For the conclusion job, aggregate AI credits from per-job usage JSONL files
+  // that reside in /tmp/gh-aw/ and must be read before the temp folder is deleted
+  // in the post-step.  These files are written by the agent and detection jobs
+  // and carry the actual AI credit consumption for each job in the workflow run.
+  // The conclusion job is not in jobEmitsOwnTokenUsage, so aiCredits is undefined
+  // above and no gh-aw.aic attribute has been pushed yet — these files are the
+  // sole source of AIC for conclusion spans.
+  if (jobName === "conclusion" && typeof aiCredits !== "number") {
+    const agentsAIC = parseAICreditsFromUsageJsonl(AGENTS_USAGE_JSONL_PATH);
+    const detectionAIC = parseAICreditsFromUsageJsonl(DETECTION_USAGE_JSONL_PATH);
+    const usageFileAIC = agentsAIC + detectionAIC;
+    if (usageFileAIC > 0) {
+      attributes.push(buildAttr("gh-aw.aic", usageFileAIC));
+    }
+  }
   if (typeof runtimeMetrics.turns === "number") {
     attributes.push(buildAttr("gh-aw.turns", runtimeMetrics.turns));
   }
@@ -2398,6 +2466,9 @@ module.exports = {
   resolveEngineId,
   GITHUB_RATE_LIMITS_JSONL_PATH,
   FAILURE_CATEGORIES_PATH,
+  AGENTS_USAGE_JSONL_PATH,
+  DETECTION_USAGE_JSONL_PATH,
+  parseAICreditsFromUsageJsonl,
   sendJobSetupSpan,
   sendJobConclusionSpan,
   OTEL_JSONL_PATH,

actions/setup/js/send_otlp_span.test.cjs

@@ -39,6 +39,9 @@ const {
   parseOTLPCustomAttributes,
   buildCustomOTLPAttributes,
   FAILURE_CATEGORIES_PATH,
+  AGENTS_USAGE_JSONL_PATH,
+  DETECTION_USAGE_JSONL_PATH,
+  parseAICreditsFromUsageJsonl,
 } = await import("./send_otlp_span.cjs");
 
 const { readExperimentAssignments, EXPERIMENT_ASSIGNMENTS_PATH } = await import("./experiment_helpers.cjs");
@@ -6756,3 +6759,232 @@ describe("sendJobConclusionSpan does not emit OTLP metrics", () => {
     expect(traceCalls.length).toBeGreaterThan(0);
   });
 });
+
+// ---------------------------------------------------------------------------
+// parseAICreditsFromUsageJsonl
+// ---------------------------------------------------------------------------
+
+describe("parseAICreditsFromUsageJsonl", () => {
+  let readFileSpy;
+
+  beforeEach(() => {
+    readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => {
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+  });
+
+  afterEach(() => {
+    readFileSpy.mockRestore();
+  });
+
+  it("returns 0 when the file does not exist", () => {
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0);
+  });
+
+  it("returns 0 for an empty file", () => {
+    readFileSpy.mockImplementation(() => "");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0);
+  });
+
+  it("returns 0 for a file with only whitespace", () => {
+    readFileSpy.mockImplementation(() => "   \n  \n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0);
+  });
+
+  it("sums ai_credits from a single-entry file", () => {
+    readFileSpy.mockImplementation(() => JSON.stringify({ ai_credits: 1.5, model: "gpt-4o" }) + "\n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(1.5);
+  });
+
+  it("sums ai_credits across multiple entries", () => {
+    const lines = [JSON.stringify({ ai_credits: 1.0, model: "gpt-4o" }), JSON.stringify({ ai_credits: 0.5, model: "claude-3-5-sonnet" }), JSON.stringify({ ai_credits: 0.25 })].join("\n");
+    readFileSpy.mockImplementation(() => lines);
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(1.75);
+  });
+
+  it("supports camelCase aiCredits field", () => {
+    readFileSpy.mockImplementation(() => JSON.stringify({ aiCredits: 2.0 }) + "\n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(2.0);
+  });
+
+  it("prefers snake_case ai_credits over camelCase aiCredits when both are present", () => {
+    readFileSpy.mockImplementation(() => JSON.stringify({ ai_credits: 1.0, aiCredits: 9.0 }) + "\n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(1.0);
+  });
+
+  it("skips entries without an ai_credits or aiCredits field", () => {
+    const lines = [JSON.stringify({ model: "gpt-4o", input_tokens: 100 }), JSON.stringify({ ai_credits: 0.75 })].join("\n");
+    readFileSpy.mockImplementation(() => lines);
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0.75);
+  });
+
+  it("skips malformed JSON lines without throwing", () => {
+    const lines = ["{not valid json}", JSON.stringify({ ai_credits: 1.0 }), "also bad"].join("\n");
+    readFileSpy.mockImplementation(() => lines);
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(1.0);
+  });
+
+  it("skips entries with negative ai_credits", () => {
+    const lines = [JSON.stringify({ ai_credits: -0.5 }), JSON.stringify({ ai_credits: 1.0 })].join("\n");
+    readFileSpy.mockImplementation(() => lines);
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(1.0);
+  });
+
+  it("skips entries where ai_credits is a non-numeric string", () => {
+    const lines = [JSON.stringify({ ai_credits: "invalid" }), JSON.stringify({ ai_credits: 0.5 })].join("\n");
+    readFileSpy.mockImplementation(() => lines);
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0.5);
+  });
+
+  it("accepts ai_credits expressed as a numeric string", () => {
+    readFileSpy.mockImplementation(() => JSON.stringify({ ai_credits: "1.234" }) + "\n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBeCloseTo(1.234);
+  });
+
+  it("accepts ai_credits of zero", () => {
+    readFileSpy.mockImplementation(() => JSON.stringify({ ai_credits: 0 }) + "\n");
+    expect(parseAICreditsFromUsageJsonl("/tmp/gh-aw/agent_usage.jsonl")).toBe(0);
+  });
+
+  it("uses the AGENTS_USAGE_JSONL_PATH constant for the agents file", () => {
+    expect(AGENTS_USAGE_JSONL_PATH).toBe("/tmp/gh-aw/agent_usage.jsonl");
+  });
+
+  it("uses the DETECTION_USAGE_JSONL_PATH constant for the detection file", () => {
+    expect(DETECTION_USAGE_JSONL_PATH).toBe("/tmp/gh-aw/detection_usage.jsonl");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// sendJobConclusionSpan — conclusion job usage file aggregation
+// ---------------------------------------------------------------------------
+
+describe("sendJobConclusionSpan conclusion job AI credits from usage files", () => {
+  /** @type {Record<string, string | undefined>} */
+  const savedEnv = {};
+  const envKeys = ["GH_AW_OTLP_ENDPOINTS", "INPUT_JOB_NAME", "GH_AW_AIC", "GH_AW_AGENT_CONCLUSION", "GITHUB_RUN_ID", "GITHUB_ACTOR", "GITHUB_REPOSITORY"];
+  let mkdirSpy, appendSpy, readFileSpy;
+
+  beforeEach(() => {
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }));
+    for (const k of envKeys) {
+      savedEnv[k] = process.env[k];
+      delete process.env[k];
+    }
+    mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => {});
+    appendSpy = vi.spyOn(fs, "appendFileSync").mockImplementation(() => {});
+    readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => {
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+    process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
+    process.env.INPUT_JOB_NAME = "conclusion";
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    for (const k of envKeys) {
+      if (savedEnv[k] !== undefined) {
+        process.env[k] = savedEnv[k];
+      } else {
+        delete process.env[k];
+      }
+    }
+    mkdirSpy.mockRestore();
+    appendSpy.mockRestore();
+    readFileSpy.mockRestore();
+  });
+
+  /**
+   * Extract attributes from the first span in the first fetch call body.
+   * @param {import("vitest").MockInstance} mockFetch
+   * @returns {Record<string, unknown>}
+   */
+  function getSpanAttrMap(mockFetch) {
+    const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+    const span = body.resourceSpans[0].scopeSpans[0].spans[0];
+    return Object.fromEntries(span.attributes.map(a => [a.key, a.value.doubleValue ?? a.value.intValue ?? a.value.stringValue ?? a.value.boolValue]));
+  }
+
+  it("emits gh-aw.aic from agent_usage.jsonl when only that file is present", async () => {
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === AGENTS_USAGE_JSONL_PATH) return JSON.stringify({ ai_credits: 1.5 }) + "\n";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeCloseTo(1.5);
+  });
+
+  it("emits gh-aw.aic from detection_usage.jsonl when only that file is present", async () => {
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === DETECTION_USAGE_JSONL_PATH) return JSON.stringify({ ai_credits: 0.75 }) + "\n";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeCloseTo(0.75);
+  });
+
+  it("emits gh-aw.aic as the sum of both usage files when both are present", async () => {
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === AGENTS_USAGE_JSONL_PATH) return JSON.stringify({ ai_credits: 1.0 }) + "\n" + JSON.stringify({ ai_credits: 0.5 }) + "\n";
+      if (filePath === DETECTION_USAGE_JSONL_PATH) return JSON.stringify({ ai_credits: 0.25 }) + "\n";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeCloseTo(1.75);
+  });
+
+  it("does not emit gh-aw.aic when both usage files are absent", async () => {
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeUndefined();
+  });
+
+  it("does not emit gh-aw.aic when both usage files are empty", async () => {
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === AGENTS_USAGE_JSONL_PATH || filePath === DETECTION_USAGE_JSONL_PATH) return "";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeUndefined();
+  });
+
+  it("does not emit gh-aw.aic from usage files for non-conclusion jobs", async () => {
+    process.env.INPUT_JOB_NAME = "safe_outputs";
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === AGENTS_USAGE_JSONL_PATH) return JSON.stringify({ ai_credits: 5.0 }) + "\n";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.safe-outputs.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeUndefined();
+  });
+
+  it("reads usage files from the fixed /tmp/gh-aw/ paths regardless of GH_AW_AGENT_OUTPUT", async () => {
+    process.env.GH_AW_AGENT_OUTPUT = "/custom/path/output.json";
+    readFileSpy.mockImplementation(filePath => {
+      if (filePath === "/tmp/gh-aw/agent_usage.jsonl") return JSON.stringify({ ai_credits: 2.0 }) + "\n";
+      throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+    });
+
+    await sendJobConclusionSpan("gh-aw.conclusion.conclusion", { startMs: 1_700_000_000_000 });
+
+    const attrs = getSpanAttrMap(fetch);
+    expect(attrs["gh-aw.aic"]).toBeCloseTo(2.0);
+    delete process.env.GH_AW_AGENT_OUTPUT;
+  });
+});

pkg/constants/job_constants.go

@@ -112,6 +112,16 @@ const SafeOutputsFilename = "safeoutputs.jsonl"
 // consume structured token data without parsing the step summary or GITHUB_OUTPUT.
 const TokenUsageFilename = "agent_usage.json"
 
+// AgentUsageJsonlFilename is the filename of the per-run agent AI credit usage JSONL file written to /tmp/gh-aw/.
+// Each line is a JSON object with an ai_credits (or aiCredits) field tracking credit consumption.
+// This file is read by the conclusion job post-step to aggregate AIC before the temp folder is removed.
+const AgentUsageJsonlFilename = "agent_usage.jsonl"
+
+// DetectionUsageJsonlFilename is the filename of the per-run detection job AI credit usage JSONL file written to /tmp/gh-aw/.
+// Each line is a JSON object with an ai_credits (or aiCredits) field tracking credit consumption.
+// This file is read by the conclusion job post-step to aggregate AIC before the temp folder is removed.
+const DetectionUsageJsonlFilename = "detection_usage.jsonl"
+
 // GithubRateLimitsFilename is the filename of the GitHub API rate-limit log written to /tmp/gh-aw/.
 // Each line is a JSON object recording the x-ratelimit-* headers (or rate-limit API snapshot)
 // captured during github.rest API calls, enabling post-run analysis of rate-limit consumption.

pkg/workflow/notify_comment.go

@@ -660,12 +660,12 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string)
 		"        run: |\n",
 		"          mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection\n",
 		"          echo \"Usage artifact source file status:\"\n",
-		"          for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do\n",
+		fmt.Sprintf("          for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/%s /tmp/gh-aw/%s /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do\n", constants.AgentUsageJsonlFilename, constants.DetectionUsageJsonlFilename),
 		"            [ -f \"$file\" ] && echo \"FOUND: $file\" || echo \"MISSING: $file\"\n",
 		"          done\n",
 		"          [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true\n",
-		"          [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true\n",
-		"          [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true\n",
+		fmt.Sprintf("          [ -f /tmp/gh-aw/%[1]s ] && cp /tmp/gh-aw/%[1]s /tmp/gh-aw/usage/%[1]s || true\n", constants.AgentUsageJsonlFilename),
+		fmt.Sprintf("          [ -f /tmp/gh-aw/%[1]s ] && cp /tmp/gh-aw/%[1]s /tmp/gh-aw/usage/%[1]s || true\n", constants.DetectionUsageJsonlFilename),
 		"          [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n",
 		"          [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n",
 		"          [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n",
@@ -683,8 +683,8 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string)
 		fmt.Sprintf("          name: %s\n", usageArtifactName),
 		"          path: |\n",
 		"            /tmp/gh-aw/usage/aw-info.jsonl\n",
-		"            /tmp/gh-aw/usage/agent_usage.jsonl\n",
-		"            /tmp/gh-aw/usage/detection_usage.jsonl\n",
+		"            /tmp/gh-aw/usage/" + constants.AgentUsageJsonlFilename + "\n",
+		"            /tmp/gh-aw/usage/" + constants.DetectionUsageJsonlFilename + "\n",
 		"            /tmp/gh-aw/usage/agent/token_usage.jsonl\n",
 		"            /tmp/gh-aw/usage/detection/token_usage.jsonl\n",
 		"          if-no-files-found: ignore\n",