Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 40 additions & 27 deletions apps/mission-control/e2e/live-feed.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ const OPS_CONFIG = {
},
safety: {
fail_safe_on_config_error: true,
incident_high_burst_threshold: 5,
incident_high_burst_window_ms: 60_000,
incident_auto_cooldown_ms: 10 * 60_000,
incident_health_degraded_trigger_ms: 30_000,
incident_healthy_exit_ms: 5 * 60_000,
recovery_retention_window_ms: 30 * 60_000,
recovery_log_max_bytes: 50 * 1024 * 1024,
mark_read_undo_window_ms: 5 * 60_000,
Expand Down Expand Up @@ -199,7 +194,7 @@ test.describe("mission-control live feed + incident automation @p2", () => {
await expect(page.getByText("recoverable-event-two")).toBeVisible();
});

test("incident auto-trigger honors cooldown but re-enters on critical", async ({ page, request }) => {
test("event volume and severity copy never flip incident mode; composed facts do", async ({ page, request }) => {
await completeQuickstartLocalOnboarding(page, {
beforeGoto: async (nextPage) => {
await nextPage.addInitScript(
Expand All @@ -217,45 +212,63 @@ test.describe("mission-control live feed + incident automation @p2", () => {

await expect(page.locator(".mc-topbar")).not.toHaveClass(/mc-topbar-incident/);

// An event burst and a critical-severity notice are copy and volume,
// not authoritative facts. Neither may raise the fire alarm.
await emitWsBurst(request, {
count: 5,
count: 8,
event_type: "system.alert",
entity: "system",
payload: {
domain: "system",
severity: "high",
summary: "high-burst",
summary: "high-burst-not-a-fact",
},
});

await expect(page.locator(".mc-topbar")).toHaveClass(/mc-topbar-incident/);

await page.locator(".mc-incident-toggle").click();
await expect(page.locator(".mc-topbar")).not.toHaveClass(/mc-topbar-incident/);

await emitWsBurst(request, {
count: 5,
await emitWsEvent(request, {
event_type: "system.alert",
entity: "system",
payload: {
domain: "system",
severity: "high",
summary: "high-burst-cooldown",
severity: "critical",
summary: "critical-copy-not-a-fact",
},
});

await page.getByTestId("live-feed-toggle").click();
await expect(page.getByText("critical-copy-not-a-fact")).toBeVisible();
await expect(page.locator(".mc-topbar")).not.toHaveClass(/mc-topbar-incident/);
await expect(page.locator('[data-testid="incident-band"]')).toHaveCount(0);

await emitWsEvent(request, {
event_type: "system.alert",
entity: "system",
payload: {
domain: "system",
severity: "critical",
summary: "critical-bypass-cooldown",
// A real authoritative fact — an open core circuit breaker — still
// raises incident mode through the composed posture.
const opsResponse = await request.post(`${GATEWAY_URL}/api/v1/e2e/ops-state`, {
headers: { Authorization: `Bearer ${TEST_TOKEN}` },
data: {
circuit_breakers: [
{
scope: "provider",
target_id: "openai-live-feed-fact",
state: "open",
consecutive_failures: 3,
cooldown_until: Date.now() + 5 * 60_000,
last_error_code: "timeout_live_feed",
updated_at: Date.now(),
},
],
},
});
expect(opsResponse.ok()).toBeTruthy();
await emitWsEvent(request, {
event_type: "job.updated",
entity: "job",
payload: { job_id: "job-heartbeat" },
});

await expect(page.locator(".mc-topbar")).toHaveClass(/mc-topbar-incident/);
await expect(page.locator(".mc-topbar")).toHaveClass(/mc-topbar-incident/, {
timeout: 20_000,
});
await expect(page.locator('[data-testid="incident-band"]')).toBeVisible();
await expect(page.locator('[data-testid="incident-band"]')).toContainText(
"openai-live-feed-fact",
);
});
});
56 changes: 56 additions & 0 deletions apps/mission-control/e2e/mockGateway.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1091,6 +1091,7 @@ function resetExecassState() {
globalSequence: 1000,
outbox: [],
intakeCount: 0,
receiptIntegrityQuarantined: false,
stopAll: {
engaged: false,
drain_state: "disengaged",
Expand Down Expand Up @@ -3629,6 +3630,16 @@ async function routeRequest(req, res) {
}

if (req.method === "GET" && requestUrl.pathname === "/api/v1/execass/summary") {
if (execassState.receiptIntegrityQuarantined) {
execassApiError(
res,
500,
"execass.v1.receipt_integrity_quarantined",
"The authoritative summary could not be rendered safely.",
true,
);
return;
}
sendJson(res, 200, buildExecassSummary());
return;
}
Expand Down Expand Up @@ -6198,6 +6209,51 @@ async function routeRequest(req, res) {
return;
}

if (
req.method === "POST" &&
requestUrl.pathname === "/api/v1/e2e/execass-event"
) {
// Inject one durable ExecAss stream event through the same broadcast
// path production events use (sequence, dedupe identity, resume gating).
const payload = await readJson(req);
const eventName =
typeof payload.event_name === "string" ? payload.event_name : "";
if (!eventName.startsWith("execass.v1.")) {
sendJson(res, 400, { error: "event_name must be an execass.v1.* event" });
return;
}
if (eventName === "execass.v1.receipt.integrity_failed") {
execassState.receiptIntegrityQuarantined = true;
}
broadcastExecassEvent(eventName, {
summary:
typeof payload.summary === "string" && payload.summary.trim()
? payload.summary.trim()
: "E2E injected event",
delegation_id:
typeof payload.delegation_id === "string"
? payload.delegation_id
: null,
receipt_ref:
typeof payload.receipt_ref === "string" ? payload.receipt_ref : null,
});
sendJson(res, 200, { event_name: eventName });
return;
}

if (
req.method === "POST" &&
requestUrl.pathname === "/api/v1/e2e/execass-integrity-recovered"
) {
execassState.receiptIntegrityQuarantined = false;
execassState.summaryRevision += 1;
broadcastExecassEvent("execass.v1.summary.changed", {
summary: "Receipt integrity verification recovered.",
});
sendJson(res, 200, { recovered: true });
return;
}

if (req.method === "POST" && requestUrl.pathname === "/api/v1/e2e/ws-event") {
const payload = await readJson(req);
const event = broadcastWsEvent({
Expand Down
Loading
Loading