Skip to content

Commit b0e8a3d

Browse files
authored
Merge pull request #333 from Blazity/feat/run-analysis-report
feat: add durable run analysis reports
2 parents 4df76a4 + 6dcbdf3 commit b0e8a3d

39 files changed

Lines changed: 10673 additions & 19 deletions

.github/workflows/ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ jobs:
3535
ci:
3636
runs-on: ubuntu-latest
3737
# `pnpm run test` alone measured 16:52, 17:52, 17:55 and 18:58 across four
38-
# branches on one day, so a 20 minute job left one to three minutes for
38+
# branches on one day, so a 35 minute job left too little headroom for
3939
# everything else and cancelled mid-step whenever a runner was slow. A
4040
# cancelled check reads as neither red nor green, which is worse than a
4141
# failure: it is easy to merge past. The suite's cost is dominated by
4242
# per-file database setup rather than test count, so this ceiling has to
4343
# cover that until the database fixture is shared.
44-
timeout-minutes: 35
44+
timeout-minutes: 40
4545
steps:
4646
- uses: actions/checkout@v4
4747
- uses: pnpm/action-setup@v4

apps/dashboard/app/(cockpit)/cockpit-shell.test.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ function makeDetail(status: Run["status"]): RunDetailResponse {
168168
deploymentId: null,
169169
},
170170
steps: [],
171+
analysisReport: null,
171172
clarification: null,
172173
};
173174
}
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import assert from "node:assert/strict";
2+
import test from "node:test";
3+
import React from "react";
4+
import { renderToStaticMarkup } from "react-dom/server";
5+
import type { RunAnalysisReport } from "@shared/contracts";
6+
import { RunAnalysisReportCard } from "./run-analysis-report";
7+
8+
(globalThis as typeof globalThis & { React: typeof React }).React = React;
9+
10+
const base: RunAnalysisReport = {
11+
version: 1,
12+
runId: "run-report",
13+
sourceResearchRunId: "run-report",
14+
researchRevision: 1,
15+
stage: "published",
16+
researchCompletedAt: "2026-08-20T00:00:00.000Z",
17+
repositories: [{ provider: "github", repoPath: "acme/api", defaultBranch: "main", researchBranch: "arthur/AWT-1", researchBaseSha: "abcdef123456", access: "write", rationale: "ticket" }],
18+
expansionRounds: 1,
19+
repositoryRequests: [],
20+
writeRepositories: [{ provider: "github", repoPath: "acme/api", rationale: "ticket" }],
21+
evidenceStatus: "captured",
22+
evidence: ["github:acme/api src/index.ts: checked"],
23+
planMarkdown: "# Plan\n\nImplement the change.",
24+
noChangeNeeded: false,
25+
resolutionEvidence: [],
26+
publication: { prs: [{ provider: "github", repoPath: "acme/api", id: 1, url: "https://github.com/acme/api/pull/1" }], changeSummary: "Implemented the change." },
27+
usage: {
28+
research: {
29+
capturedAt: "2026-08-20T00:00:00.000Z",
30+
costUsd: 1.2,
31+
costKnown: false,
32+
tokensInput: null,
33+
tokensCached: null,
34+
tokensOutput: null,
35+
phases: {
36+
research: {
37+
costUsd: 1.2,
38+
tokens: { input: 100, cachedInput: 25, output: 50 },
39+
durationMs: 1200,
40+
numTurns: 2,
41+
model: "gpt-5.6",
42+
},
43+
},
44+
},
45+
publication: null,
46+
final: null,
47+
},
48+
jira: {
49+
research: { state: "posted", attemptedAt: "2026-08-20T00:00:00.000Z", commentUrl: "https://jira.example/comment/1", error: null },
50+
pullRequest: { state: "failed", attemptedAt: "2026-08-20T00:00:00.000Z", commentUrl: null, error: "Jira unavailable" },
51+
},
52+
sanitization: { redactions: { token: 1 }, truncated: false, originalBytes: 1, storedBytes: 1, unavailable: false, unavailableReason: null },
53+
};
54+
55+
test("renders the complete report with accessible disclosures and delivery state", () => {
56+
const html = renderToStaticMarkup(<RunAnalysisReportCard report={base} runStatus="success" currentRunId="run-report" />);
57+
assert.match(html, /Analysis report/);
58+
assert.match(html, /PR\/MR published/);
59+
assert.match(html, /aria-expanded="true"/);
60+
assert.match(html, /Some report content was redacted/);
61+
assert.match(html, /Automatic retries exhausted/);
62+
assert.match(html, /\$1\.20\+/);
63+
assert.match(html, /gpt-5\.6/);
64+
assert.match(html, /100 in \/ 25 cached \/ 50 out/);
65+
assert.match(html, /1200ms \/ 2 turns/);
66+
assert.match(html, /https:\/\/jira\.example\/comment\/1/);
67+
assert.match(html, /base: main/);
68+
assert.doesNotMatch(html, /overflow-x-hidden/); // code blocks retain a bounded local scroller
69+
});
70+
71+
test("distinguishes missing evidence and hides runs without a report", () => {
72+
const notRetained = renderToStaticMarkup(<RunAnalysisReportCard report={{ ...base, stage: "research_complete", evidenceStatus: "not_retained", evidence: [], publication: null }} runStatus="success" currentRunId="run-report" />);
73+
assert.match(notRetained, /Source evidence was not retained/);
74+
const missing = renderToStaticMarkup(<RunAnalysisReportCard report={null} runStatus="failed" currentRunId="legacy" />);
75+
assert.equal(missing, "");
76+
const active = renderToStaticMarkup(<RunAnalysisReportCard report={null} runStatus="running" currentRunId="live" />);
77+
assert.equal(active, "");
78+
});
Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
"use client";
2+
3+
import React from "react";
4+
import Link from "next/link";
5+
import type {
6+
RunAnalysisPhaseUsage,
7+
RunAnalysisReport,
8+
RunAnalysisUsageSnapshot,
9+
RunStatus,
10+
} from "@shared/contracts";
11+
import { CkCard, CkChip } from "@/components/ui";
12+
import { PromptPreview } from "@/components/cockpit/prompt-library/prompt-preview";
13+
import { runHref } from "@/lib/run-href";
14+
15+
function stageLabel(stage: RunAnalysisReport["stage"]): string {
16+
if (stage === "published") return "PR/MR published";
17+
if (stage === "no_change") return "No change needed";
18+
return "Research complete";
19+
}
20+
21+
function stageTone(stage: RunAnalysisReport["stage"]): "success" | "warn" | "mariner" {
22+
return stage === "published" ? "success" : stage === "no_change" ? "warn" : "mariner";
23+
}
24+
25+
function costLabel(snapshot: RunAnalysisUsageSnapshot | null): string {
26+
if (!snapshot) return "Not reached";
27+
return `$${snapshot.costUsd.toFixed(2)}${snapshot.costKnown ? "" : "+"}`;
28+
}
29+
30+
function phaseCostLabel(phase: RunAnalysisPhaseUsage): string {
31+
return phase.costUsd === null ? "Unknown" : `$${phase.costUsd.toFixed(2)}`;
32+
}
33+
34+
function tokenLabel(tokens: RunAnalysisPhaseUsage["tokens"]): string {
35+
if (!tokens) return "Unknown";
36+
return `${tokens.input} in / ${tokens.cachedInput} cached / ${tokens.output} out`;
37+
}
38+
39+
function deliveryLabel(delivery: RunAnalysisReport["jira"]["research"]): string {
40+
if (delivery.state === "posted") return "Posted";
41+
if (delivery.state === "failed") return delivery.error ? `Failed: ${delivery.error}` : "Failed";
42+
if (delivery.state === "pending") return "Pending";
43+
return "Not applicable";
44+
}
45+
46+
function DeliveryStatus({
47+
label,
48+
delivery,
49+
}: {
50+
label: string;
51+
delivery: RunAnalysisReport["jira"]["research"];
52+
}) {
53+
return (
54+
<span>
55+
{label}: {delivery.commentUrl ? (
56+
<a
57+
href={delivery.commentUrl}
58+
target="_blank"
59+
rel="noreferrer"
60+
className="text-mariner underline-offset-2 hover:underline"
61+
>
62+
{deliveryLabel(delivery)}
63+
</a>
64+
) : deliveryLabel(delivery)}
65+
</span>
66+
);
67+
}
68+
69+
function Disclosure({ title, children, defaultOpen = false }: { title: string; children: React.ReactNode; defaultOpen?: boolean }) {
70+
const [open, setOpen] = React.useState(defaultOpen);
71+
return (
72+
<div className="border-t border-neutral-200 pt-3">
73+
<button
74+
type="button"
75+
aria-expanded={open}
76+
onClick={() => setOpen((value) => !value)}
77+
className="flex w-full items-center justify-between gap-3 border-0 bg-transparent p-0 text-left font-mono text-[11px] uppercase tracking-[0.04em] text-neutral-800"
78+
>
79+
<span>{title}</span>
80+
<span aria-hidden="true">{open ? "−" : "+"}</span>
81+
</button>
82+
{open && <div className="mt-3 min-w-0">{children}</div>}
83+
</div>
84+
);
85+
}
86+
87+
function UsageTable({ report }: { report: RunAnalysisReport }) {
88+
const rows: Array<[string, RunAnalysisUsageSnapshot | null]> = [
89+
["Research", report.usage.research],
90+
["Publication", report.usage.publication],
91+
["Final", report.usage.final],
92+
];
93+
return (
94+
<div className="overflow-x-auto">
95+
<table className="w-full min-w-[720px] border-collapse font-mono text-[11px]">
96+
<thead><tr className="text-left text-neutral-500"><th className="pb-2 pr-3 font-normal">Snapshot / phase</th><th className="pb-2 pr-3 font-normal">Cost</th><th className="pb-2 pr-3 font-normal">Tokens</th><th className="pb-2 pr-3 font-normal">Model</th><th className="pb-2 font-normal">Duration / turns</th></tr></thead>
97+
<tbody>
98+
{rows.map(([name, snapshot]) => (
99+
<React.Fragment key={name}>
100+
<tr className="border-t border-neutral-200 align-top">
101+
<td className="py-2 pr-3 font-medium text-neutral-900">{name}</td>
102+
<td className="py-2 pr-3 text-neutral-800">{costLabel(snapshot)}</td>
103+
<td className="py-2 pr-3 text-neutral-700">{snapshot?.tokensInput === null || !snapshot ? "Unknown" : `${snapshot.tokensInput} in / ${snapshot.tokensCached ?? 0} cached / ${snapshot.tokensOutput ?? 0} out`}</td>
104+
<td className="py-2 pr-3 text-neutral-500">{snapshot ? `Captured ${new Date(snapshot.capturedAt).toLocaleString()}` : "—"}</td>
105+
<td className="py-2 text-neutral-500"></td>
106+
</tr>
107+
{snapshot ? Object.entries(snapshot.phases).map(([phaseName, phase]) => (
108+
<tr key={`${name}:${phaseName}`} className="border-t border-neutral-100 align-top">
109+
<td className="py-2 pr-3 pl-3 text-neutral-700">{phaseName}</td>
110+
<td className="py-2 pr-3 text-neutral-700">{phaseCostLabel(phase)}</td>
111+
<td className="py-2 pr-3 text-neutral-700">{tokenLabel(phase.tokens)}</td>
112+
<td className="py-2 pr-3 text-neutral-700">{phase.model ?? "Unknown"}</td>
113+
<td className="py-2 text-neutral-700">{phase.durationMs}ms / {phase.numTurns} turns</td>
114+
</tr>
115+
)) : null}
116+
</React.Fragment>
117+
))}
118+
</tbody>
119+
</table>
120+
</div>
121+
);
122+
}
123+
124+
export function RunAnalysisReportCard({ report, runStatus: _runStatus, currentRunId }: { report: RunAnalysisReport | null; runStatus: RunStatus; currentRunId: string }) {
125+
if (!report) return null;
126+
const finalOrPublication = report.usage.final ?? report.usage.publication ?? report.usage.research;
127+
const sourceDiffers = report.sourceResearchRunId !== currentRunId;
128+
return (
129+
<CkCard eyebrow="Run analysis" title="Analysis report" action={<CkChip tone={stageTone(report.stage)}>{stageLabel(report.stage)}</CkChip>}>
130+
<div className="flex min-w-0 flex-col gap-4 font-body text-[13px]">
131+
<div className="flex flex-wrap items-center gap-x-4 gap-y-1 text-neutral-600">
132+
<span>Captured {new Date(report.researchCompletedAt).toLocaleString()}</span>
133+
{sourceDiffers && <Link href={runHref({ id: report.sourceResearchRunId, ticket: "" })} className="text-mariner underline-offset-2 hover:underline">Source research run: {report.sourceResearchRunId}</Link>}
134+
</div>
135+
<div className="grid grid-cols-2 gap-2 md:grid-cols-4">
136+
<Metric label="Repositories inspected" value={report.repositories.length} />
137+
<Metric label="Evidence items" value={report.evidenceStatus === "not_retained" ? "—" : report.evidence.length} />
138+
<Metric label="Expansion rounds" value={report.expansionRounds} />
139+
<Metric label="Current cost" value={costLabel(finalOrPublication)} />
140+
</div>
141+
{report.sanitization.truncated || Object.keys(report.sanitization.redactions).length > 0 ? (
142+
<div role="status" className="rounded-[3px] border border-[#F0D9A8] bg-[#FFF9E8] px-3 py-2 text-[12px] text-[#6E5200]">Some report content was redacted or truncated for safety.</div>
143+
) : null}
144+
{report.evidenceStatus === "not_retained" && <div className="rounded-[3px] border border-neutral-200 bg-app-bg px-3 py-2 text-[12px] text-neutral-700">Source evidence was not retained.</div>}
145+
<div className="min-w-0 overflow-hidden">
146+
<h3 className="m-0 mb-2 font-mono text-[10px] uppercase tracking-[0.05em] text-neutral-500">Repositories analyzed</h3>
147+
<div className="flex min-w-0 flex-col gap-2">
148+
{report.repositories.map((repo) => <div key={`${repo.provider}:${repo.repoPath}`} className="grid min-w-0 gap-1 rounded-[3px] border border-neutral-200 bg-off-white p-2 md:grid-cols-[1.3fr_.5fr_1fr_1.4fr] md:items-start"><span className="break-all font-mono text-[11px] text-neutral-900">{repo.provider}:{repo.repoPath}</span><span className="font-mono text-[10px] uppercase text-neutral-600">{repo.access}</span><span className="break-all font-mono text-[10px] text-neutral-600">{repo.researchBranch}@{repo.researchBaseSha ? repo.researchBaseSha.slice(0, 8) : "unknown SHA"}<span className="block text-neutral-500">base: {repo.defaultBranch}</span></span><span className="break-words text-[12px] text-neutral-700">{repo.rationale}</span></div>)}
149+
</div>
150+
</div>
151+
<Disclosure title="What was checked" defaultOpen={report.evidenceStatus === "captured"}>
152+
{report.evidenceStatus === "not_retained" ? <p className="m-0 text-neutral-600">Source evidence was not retained.</p> : report.evidence.length > 0 ? <ol className="m-0 flex list-decimal flex-col gap-1 pl-5 text-neutral-800">{report.evidence.map((item, index) => <li key={index} className="break-words">{item}</li>)}</ol> : <p className="m-0 text-neutral-600">No evidence items were captured.</p>}
153+
</Disclosure>
154+
<Disclosure title="Decisions">
155+
<div className="flex flex-col gap-2 text-neutral-800">
156+
<p className="m-0">Expansion rounds: {report.expansionRounds}</p>
157+
<DecisionList title="Requests" items={report.repositoryRequests} />
158+
<DecisionList title="Write repositories" items={report.writeRepositories} />
159+
{report.resolutionEvidence.length > 0 && <p className="m-0 break-words">Resolution evidence: {report.resolutionEvidence.join(" · ")}</p>}
160+
</div>
161+
</Disclosure>
162+
<Disclosure title="Implementation plan" defaultOpen>
163+
<div className="min-w-0 overflow-hidden"><PromptPreview body={report.planMarkdown || "No implementation plan was retained."} /></div>
164+
</Disclosure>
165+
<div className="border-t border-neutral-200 pt-3"><h3 className="m-0 mb-2 font-mono text-[10px] uppercase tracking-[0.05em] text-neutral-500">Usage</h3><UsageTable report={report} /></div>
166+
{report.publication && <div className="border-t border-neutral-200 pt-3"><h3 className="m-0 mb-2 font-mono text-[10px] uppercase tracking-[0.05em] text-neutral-500">Published</h3><div className="flex flex-col gap-2"><div className="flex flex-wrap gap-2">{report.publication.prs.map((pr) => <a key={`${pr.provider}:${pr.repoPath}:${pr.id}`} href={pr.url} target="_blank" rel="noreferrer" className="max-w-full break-all text-mariner underline-offset-2 hover:underline">{pr.provider}:{pr.repoPath} #{pr.id}</a>)}</div><p className="m-0 whitespace-pre-wrap break-words text-neutral-800">{report.publication.changeSummary}</p></div></div>}
167+
<div className="border-t border-neutral-200 pt-3"><h3 className="m-0 mb-2 font-mono text-[10px] uppercase tracking-[0.05em] text-neutral-500">Jira delivery</h3><div className="grid gap-1 font-mono text-[11px] text-neutral-700 md:grid-cols-2"><DeliveryStatus label="Research" delivery={report.jira.research} /><DeliveryStatus label="PR/MR" delivery={report.jira.pullRequest} /></div>{(report.jira.research.state === "failed" || report.jira.pullRequest.state === "failed") && <p className="m-0 mt-2 text-[12px] text-fail-fg">Automatic retries exhausted; code delivery was not blocked.</p>}</div>
168+
</div>
169+
</CkCard>
170+
);
171+
}
172+
173+
function Metric({ label, value }: { label: string; value: React.ReactNode }) {
174+
return <div className="min-w-0 rounded-[3px] border border-neutral-200 bg-off-white px-2.5 py-2"><div className="font-mono text-[9px] uppercase tracking-[0.04em] text-neutral-500">{label}</div><div className="mt-1 break-words font-display text-lg text-neutral-900">{value}</div></div>;
175+
}
176+
177+
function DecisionList({
178+
title,
179+
items,
180+
}: {
181+
title: string;
182+
items: RunAnalysisReport["repositoryRequests"];
183+
}) {
184+
if (items.length === 0) return <p className="m-0">{title}: none</p>;
185+
return (
186+
<div>
187+
<p className="m-0">{title}:</p>
188+
<ul className="m-0 mt-1 flex list-disc flex-col gap-1 pl-5">
189+
{items.map((item) => (
190+
<li key={`${item.provider}:${item.repoPath}`} className="break-words">
191+
<span className="font-mono">{item.provider}:{item.repoPath}</span>{item.rationale}
192+
</li>
193+
))}
194+
</ul>
195+
</div>
196+
);
197+
}

apps/dashboard/components/cockpit/screens/trace-replay.test.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ const detail: RunDetailResponse = {
5252
error: { message: "Review failed" },
5353
},
5454
],
55+
analysisReport: null,
5556
clarification: null,
5657
};
5758

apps/dashboard/components/cockpit/screens/trace.tsx

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { runModelLabel } from "@/lib/run-model";
1717
import { runPullRequests } from "@/lib/run-prs";
1818
import { hasActiveRun, useRunRefresh } from "@/lib/use-run-refresh";
1919
import { RunRefreshControl } from "@/components/cockpit/run-refresh-control";
20+
import { RunAnalysisReportCard } from "./run-analysis-report";
2021
import { SPAN_KIND_COLOR } from "@/lib/theme";
2122
import { pullRequestRef, pullRequestRepoLabels } from "@shared/contracts";
2223
import type { Span, SpanKind, SpanStatus } from "@/lib/types";
@@ -483,6 +484,12 @@ export function TraceDetail({
483484
/>
484485
)}
485486

487+
<RunAnalysisReportCard
488+
report={shownData.analysisReport ?? null}
489+
runStatus={run.status}
490+
currentRunId={run.id}
491+
/>
492+
486493
{hasReplay || replayPending ? (
487494
<WorkflowReplay
488495
runId={runId}
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import assert from "node:assert/strict";
2+
import test from "node:test";
3+
import { runDetailFallback } from "./fallbacks";
4+
5+
test("run detail fallback carries a nullable analysis report", () => {
6+
assert.equal(runDetailFallback("now").analysisReport, null);
7+
});

apps/dashboard/lib/api/fallbacks.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ export function recentRunsFallback(now: string): RunsResponse {
3737
}
3838

3939
export function runDetailFallback(now: string): RunDetailResponse {
40-
return { generatedAt: now, available: false, run: null, steps: [] };
40+
return { generatedAt: now, available: false, run: null, steps: [], analysisReport: null };
4141
}
4242

4343
export function runReplayFallback(): WorkflowRunReplayResponse {

apps/shared/contracts/api.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import type {
2424
WebhookAuthScheme,
2525
} from "./domain.js";
2626
import type { PromptSlotDefinition } from "./prompt-slots.js";
27+
import type { RunAnalysisReport } from "./run-analysis.js";
2728

2829
export interface ErrorEnvelope {
2930
error: { code: string; message: string; details?: unknown };
@@ -161,6 +162,7 @@ export interface RunDetailResponse {
161162
available: boolean;
162163
run: RunDetail | null;
163164
steps: RunStep[];
165+
analysisReport: RunAnalysisReport | null;
164166
clarification?: ClarificationRequest | null;
165167
}
166168

apps/shared/contracts/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ export * from "./default-agent-prompt-references.js";
1212
export * from "./workflow-value-compatibility.js";
1313
export * from "./review-result.js";
1414
export * from "./repository-scripts.js";
15+
export * from "./run-analysis.js";

0 commit comments

Comments
 (0)