Skip to content

Commit 77f950f

Browse files
authored
Add AI credit cap observability attributes to OTLP conclusion spans (#38550)
1 parent 99c0150 commit 77f950f

7 files changed

Lines changed: 126 additions & 22 deletions

File tree

actions/setup/js/ai_credits_context.cjs

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -277,37 +277,40 @@ function parseAuditLogCombined(auditJsonlPathOverride) {
277277
}
278278

279279
/**
280+
* @param {{ logProvenance?: boolean }} [options]
280281
* @returns {{ aiCredits: string, maxAICredits: string, aiCreditsRateLimitError: boolean, maxAICreditsExceeded: boolean }}
281282
*/
282-
function resolveAICreditsFailureState() {
283+
function resolveAICreditsFailureState({ logProvenance = true } = {}) {
283284
const stdioSignals = parseAICreditsExceededFromAgentStdio();
284285
const { aiCredits: auditAICredits, maxAICredits: auditMaxAICredits, rateLimitError: auditRateLimitError, maxAICreditsExceeded: auditMaxAICreditsExceeded } = parseAuditLogCombined();
285286
const envAICredits = parsePositiveNumberString(process.env.GH_AW_AIC);
286287
const envMaxAICredits = parsePositiveNumberString(process.env.GH_AW_MAX_AI_CREDITS);
287288

288289
// Log provenance so failing issues can be diagnosed when credit data is missing.
289-
if (auditAICredits) {
290-
console.log(`[ai-credits] aiCredits source=audit_log value=${auditAICredits}`);
291-
} else if (stdioSignals.aiCredits) {
292-
console.log(`[ai-credits] aiCredits source=agent_stdio value=${stdioSignals.aiCredits}`);
293-
} else if (envAICredits) {
294-
console.log(`[ai-credits] aiCredits source=env(GH_AW_AIC) value=${envAICredits}`);
295-
} else {
296-
console.log(`[ai-credits] aiCredits source=none GH_AW_AIC=${process.env.GH_AW_AIC || "(unset)"}`);
297-
}
290+
if (logProvenance) {
291+
if (auditAICredits) {
292+
console.log(`[ai-credits] aiCredits source=audit_log value=${auditAICredits}`);
293+
} else if (stdioSignals.aiCredits) {
294+
console.log(`[ai-credits] aiCredits source=agent_stdio value=${stdioSignals.aiCredits}`);
295+
} else if (envAICredits) {
296+
console.log(`[ai-credits] aiCredits source=env(GH_AW_AIC) value=${envAICredits}`);
297+
} else {
298+
console.log(`[ai-credits] aiCredits source=none GH_AW_AIC=${process.env.GH_AW_AIC || "(unset)"}`);
299+
}
298300

299-
if (auditMaxAICredits) {
300-
console.log(`[ai-credits] maxAICredits source=audit_log value=${auditMaxAICredits}`);
301-
} else if (stdioSignals.maxAICredits) {
302-
console.log(`[ai-credits] maxAICredits source=agent_stdio value=${stdioSignals.maxAICredits}`);
303-
} else if (envMaxAICredits) {
304-
console.log(`[ai-credits] maxAICredits source=env(GH_AW_MAX_AI_CREDITS) value=${envMaxAICredits}`);
305-
} else {
306-
console.log(`[ai-credits] maxAICredits source=none GH_AW_MAX_AI_CREDITS=${process.env.GH_AW_MAX_AI_CREDITS || "(unset)"}`);
307-
}
301+
if (auditMaxAICredits) {
302+
console.log(`[ai-credits] maxAICredits source=audit_log value=${auditMaxAICredits}`);
303+
} else if (stdioSignals.maxAICredits) {
304+
console.log(`[ai-credits] maxAICredits source=agent_stdio value=${stdioSignals.maxAICredits}`);
305+
} else if (envMaxAICredits) {
306+
console.log(`[ai-credits] maxAICredits source=env(GH_AW_MAX_AI_CREDITS) value=${envMaxAICredits}`);
307+
} else {
308+
console.log(`[ai-credits] maxAICredits source=none GH_AW_MAX_AI_CREDITS=${process.env.GH_AW_MAX_AI_CREDITS || "(unset)"}`);
309+
}
308310

309-
const rawRateLimitSignalSource = auditRateLimitError ? "audit_log" : stdioSignals.rateLimitError ? "agent_stdio" : process.env.GH_AW_AI_CREDITS_RATE_LIMIT_ERROR === "true" ? "env(GH_AW_AI_CREDITS_RATE_LIMIT_ERROR)" : "none";
310-
console.log(`[ai-credits] rateLimitSignal source=${rawRateLimitSignalSource}`);
311+
const rawRateLimitSignalSource = auditRateLimitError ? "audit_log" : stdioSignals.rateLimitError ? "agent_stdio" : process.env.GH_AW_AI_CREDITS_RATE_LIMIT_ERROR === "true" ? "env(GH_AW_AI_CREDITS_RATE_LIMIT_ERROR)" : "none";
312+
console.log(`[ai-credits] rateLimitSignal source=${rawRateLimitSignalSource}`);
313+
}
311314

312315
const aiCredits = auditAICredits || stdioSignals.aiCredits || envAICredits || "";
313316
const maxAICredits = auditMaxAICredits || stdioSignals.maxAICredits || envMaxAICredits || "";

actions/setup/js/handle_agent_failure.cjs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ const COPILOT_SESSION_STATE_DIR = path.join(os.tmpdir(), "gh-aw", "sandbox", "ag
3535
// - Copilot/CAPI "CAPIError: 429" and utility-model quota text
3636
// - retry wrapper text that includes the canonical "Failed to get response..." phrase
3737
const ENGINE_RATE_LIMIT_429_RE =
38-
/(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|rate_limit_(?:error|exceeded)|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i;
38+
/(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|\brate_limit_(?:error|exceeded)\b|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i;
3939

4040
/**
4141
* Parse action failure issue expiration from environment.

actions/setup/js/handle_agent_failure.test.cjs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2963,13 +2963,15 @@ describe("handle_agent_failure", () => {
29632963
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-test-engine-fail-guard-"));
29642964
stdioLogPath = path.join(tmpDir, "agent-stdio.log");
29652965
process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "agent_output.json");
2966+
process.env.GH_AW_OTEL_JSONL_PATH = path.join(tmpDir, "otel.jsonl");
29662967
process.env.RUNNER_TEMP = tmpDir;
29672968
({ buildEngineFailureContext } = require("./handle_agent_failure.cjs"));
29682969
});
29692970

29702971
afterEach(() => {
29712972
delete process.env.GH_AW_AGENT_OUTPUT;
29722973
delete process.env.GH_AW_ENGINE_ID;
2974+
delete process.env.GH_AW_OTEL_JSONL_PATH;
29732975
delete process.env.RUNNER_TEMP;
29742976
if (fs.existsSync(tmpDir)) {
29752977
fs.rmSync(tmpDir, { recursive: true, force: true });

actions/setup/js/send_otlp_span.cjs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const { buildWorkflowRunUrl } = require("./workflow_metadata_helpers.cjs");
1111
const { readExperimentAssignments, EXPERIMENT_ASSIGNMENTS_PATH } = require("./experiment_helpers.cjs");
1212
const { parseJsonlContent } = require("./jsonl_helpers.cjs");
1313
const { countSteeringEventsInApiProxyJsonl } = require("./steering_helpers.cjs");
14+
const { resolveAICreditsFailureState } = require("./ai_credits_context.cjs");
1415

1516
/**
1617
* send_otlp_span.cjs
@@ -2359,6 +2360,18 @@ async function sendJobConclusionSpan(spanName, options = {}) {
23592360
attributes.push(...usageAttrs);
23602361
}
23612362

2363+
const { maxAICredits, aiCreditsRateLimitError, maxAICreditsExceeded } = resolveAICreditsFailureState({ logProvenance: false });
2364+
const maxAICreditsValue = normalizeNonNegativeNumber(maxAICredits);
2365+
if (typeof maxAICreditsValue === "number") {
2366+
attributes.push(buildAttr("gh-aw.max_ai_credits", maxAICreditsValue));
2367+
}
2368+
if (typeof maxAICreditsExceeded === "boolean") {
2369+
attributes.push(buildAttr("gh-aw.max_ai_credits_exceeded", maxAICreditsExceeded));
2370+
}
2371+
if (typeof aiCreditsRateLimitError === "boolean") {
2372+
attributes.push(buildAttr("gh-aw.ai_credits_rate_limit_error", aiCreditsRateLimitError));
2373+
}
2374+
23622375
const payload = buildOTLPPayload({
23632376
traceId,
23642377
spanId: conclusionSpanId,

actions/setup/js/send_otlp_span.test.cjs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2523,6 +2523,8 @@ describe("sendJobConclusionSpan", () => {
25232523
"OTEL_SERVICE_NAME",
25242524
"GH_AW_EFFECTIVE_TOKENS",
25252525
"GH_AW_AIC",
2526+
"GH_AW_MAX_AI_CREDITS",
2527+
"GH_AW_AI_CREDITS_RATE_LIMIT_ERROR",
25262528
"GH_AW_INFO_VERSION",
25272529
"GH_AW_INFO_CLI_VERSION",
25282530
"GITHUB_AW_OTEL_TRACE_ID",
@@ -2645,6 +2647,7 @@ describe("sendJobConclusionSpan", () => {
26452647
process.env.INPUT_JOB_NAME = "agent";
26462648
process.env.GITHUB_AW_OTEL_TRACE_ID = "f".repeat(32);
26472649
process.env.GITHUB_AW_OTEL_PARENT_SPAN_ID = "abcdef1234567890";
2650+
process.env.GH_AW_MAX_AI_CREDITS = "1000";
26482651

26492652
const startMs = 1_700_000_000_000;
26502653
const endMs = 1_700_000_005_000;
@@ -2677,6 +2680,14 @@ describe("sendJobConclusionSpan", () => {
26772680
expect(conclusionSpan.parentSpanId).toBe("abcdef1234567890");
26782681
expect(agentSpan.attributes).toContainEqual({ key: "gh-aw.output.item_count", value: { intValue: 2 } });
26792682
expect(conclusionSpan.attributes).toContainEqual({ key: "gh-aw.output.item_count", value: { intValue: 2 } });
2683+
const agentKeys = agentSpan.attributes.map(a => a.key);
2684+
const conclusionKeys = conclusionSpan.attributes.map(a => a.key);
2685+
expect(agentKeys).not.toContain("gh-aw.max_ai_credits");
2686+
expect(agentKeys).not.toContain("gh-aw.max_ai_credits_exceeded");
2687+
expect(agentKeys).not.toContain("gh-aw.ai_credits_rate_limit_error");
2688+
expect(conclusionKeys).toContain("gh-aw.max_ai_credits");
2689+
expect(conclusionKeys).toContain("gh-aw.max_ai_credits_exceeded");
2690+
expect(conclusionKeys).toContain("gh-aw.ai_credits_rate_limit_error");
26802691
});
26812692

26822693
it("uses agent_cli_start_ms.txt as agent span start time when file is present", async () => {
@@ -3578,6 +3589,75 @@ describe("sendJobConclusionSpan", () => {
35783589
expect(aicAttr.value.doubleValue).toBe(0.125);
35793590
});
35803591

3592+
it("emits gh-aw.max_ai_credits as a numeric conclusion-span attribute when available", async () => {
3593+
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
3594+
vi.stubGlobal("fetch", mockFetch);
3595+
3596+
process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
3597+
process.env.GH_AW_MAX_AI_CREDITS = "1000.5";
3598+
3599+
await sendJobConclusionSpan("gh-aw.job.conclusion");
3600+
3601+
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
3602+
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
3603+
const maxAICreditsAttr = span.attributes.find(a => a.key === "gh-aw.max_ai_credits");
3604+
expect(maxAICreditsAttr).toBeDefined();
3605+
expect(maxAICreditsAttr.value.doubleValue).toBe(1000.5);
3606+
});
3607+
3608+
it("emits AI credits boolean cap/rate-limit attributes on conclusion spans when detected", async () => {
3609+
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
3610+
vi.stubGlobal("fetch", mockFetch);
3611+
3612+
process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
3613+
3614+
const stdioContent = Buffer.from("CAPIError: 429 Maximum AI credits exceeded (1002.381900 / 1000).", "utf8");
3615+
const stdioLogPath = "/tmp/gh-aw/agent-stdio.log";
3616+
const MOCK_FD = 42;
3617+
const existsSpy = vi.spyOn(fs, "existsSync").mockImplementation(p => p === stdioLogPath);
3618+
const statSpy = vi.spyOn(fs, "statSync").mockImplementation(p => {
3619+
if (p === stdioLogPath) return /** @type {fs.Stats} */ { size: stdioContent.length };
3620+
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
3621+
});
3622+
const openSpy = vi.spyOn(fs, "openSync").mockReturnValue(/** @type {number} */ MOCK_FD);
3623+
const readSpy = vi.spyOn(fs, "readSync").mockImplementation((_fd, buf) => {
3624+
stdioContent.copy(/** @type {Buffer} */ buf);
3625+
return stdioContent.length;
3626+
});
3627+
const closeSpy = vi.spyOn(fs, "closeSync").mockImplementation(() => {});
3628+
3629+
try {
3630+
await sendJobConclusionSpan("gh-aw.job.conclusion");
3631+
} finally {
3632+
existsSpy.mockRestore();
3633+
statSpy.mockRestore();
3634+
openSpy.mockRestore();
3635+
readSpy.mockRestore();
3636+
closeSpy.mockRestore();
3637+
}
3638+
3639+
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
3640+
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
3641+
const attrs = Object.fromEntries(span.attributes.map(a => [a.key, a.value.boolValue ?? a.value.doubleValue ?? a.value.intValue ?? a.value.stringValue]));
3642+
expect(attrs["gh-aw.max_ai_credits_exceeded"]).toBe(true);
3643+
expect(attrs["gh-aw.ai_credits_rate_limit_error"]).toBe(true);
3644+
});
3645+
3646+
it("does not emit gh-aw.max_ai_credits when max AI credits is missing or invalid", async () => {
3647+
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
3648+
vi.stubGlobal("fetch", mockFetch);
3649+
3650+
process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]);
3651+
process.env.GH_AW_MAX_AI_CREDITS = "not-a-number";
3652+
3653+
await sendJobConclusionSpan("gh-aw.job.conclusion");
3654+
3655+
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
3656+
const span = body.resourceSpans[0].scopeSpans[0].spans[0];
3657+
const keys = span.attributes.map(a => a.key);
3658+
expect(keys).not.toContain("gh-aw.max_ai_credits");
3659+
});
3660+
35813661
it("emits dashboard metrics and aliases on the conclusion span", async () => {
35823662
const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
35833663
vi.stubGlobal("fetch", mockFetch);

docs/src/content/docs/reference/open-telemetry.mdx

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ These attributes appear on built-in workflow setup, agent, and conclusion spans
8888
<tr><td><code>gh-aw.action_minutes</code></td><td>Elapsed runtime converted to minutes.</td></tr>
8989
<tr><td><code>gh-aw.tracker.id</code></td><td>Tracker identifier when present.</td></tr>
9090
<tr><td><code>gh-aw.aic</code></td><td>AI credits consumed for the run when available.</td></tr>
91+
<tr><td><code>gh-aw.max_ai_credits</code></td><td>Configured max AI credits budget for the run when available.</td></tr>
92+
<tr><td><code>gh-aw.max_ai_credits_exceeded</code></td><td>Whether the run exceeded the max AI credits budget.</td></tr>
93+
<tr><td><code>gh-aw.ai_credits_rate_limit_error</code></td><td>Whether an AI-credits rate-limit or budget-exhaustion signal was detected.</td></tr>
9194
<tr><td><code>gh-aw.turns</code></td><td>Total agent turns recorded for the run.</td></tr>
9295
<tr><td><code>gh-aw.agent.conclusion</code></td><td>Normalized agent conclusion.</td></tr>
9396
<tr><td><code>gh-aw.detection.conclusion</code></td><td>Detection subsystem conclusion when present.</td></tr>

specs/otel-observability-spec.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,9 @@ This section defines the attributes each span type MUST or MAY carry.
455455
| `gh-aw.trigger.*` | string | Trigger context (same fields as setup span) |
456456
| `gh-aw.frontmatter.*` | string | Frontmatter metadata (same fields as setup span) |
457457
| `gh-aw.aic` | double | AI credits consumed (AIC); always emitted as a numeric attribute on agent and detection conclusion spans (0 when no usage data is available, so Sentry EAP and Tempo index the field as numeric from first emission). |
458+
| `gh-aw.max_ai_credits` | double | Configured max AI credits budget for the run when a valid numeric value is available. |
459+
| `gh-aw.max_ai_credits_exceeded` | boolean | True when a max-AI-credits hard-limit exceedance signal is detected for the run. |
460+
| `gh-aw.ai_credits_rate_limit_error` | boolean | True when an AI-credits-related rate-limit or budget-exhaustion signal is detected. |
458461
| `gh-aw.turns` | int | Number of agent turns |
459462
| `gh-aw.agent.conclusion` | string | Agent job outcome |
460463
| `gh-aw.detection.conclusion` | string | Threat detection outcome |

0 commit comments

Comments
 (0)