feat(glass-office): Breakers & Scheduler as the fourth Basement room

apps/mission-control/e2e/core.spec.ts

@@ -417,6 +417,8 @@ test.describe("mission-control core onboarding + crash-proofing @core", () => {
     );
 
     await page.locator('[data-tour-id="nav-focus"]').click();
+    // Focus keeps its last mounted section, so select Queue explicitly.
+    await page.getByRole("tab", { name: /^Queue/ }).click();
     await expect(page.getByRole("button", { name: "Open task" }).first()).toBeVisible();
     await page.getByRole("button", { name: "Open task" }).first().click();
     await expect(page.getByTestId("strategy-page")).toBeVisible();

apps/mission-control/e2e/mockGateway.mjs

@@ -899,6 +899,21 @@ const providerOrder = new Map();
 let usageMode = "available";
 let usageUpdatedAtMs = Date.now();
 
+/**
+ * E2E-only operational posture: lets breaker/scheduler proofs drive real
+ * gateway payloads instead of inventing frontend-only fixtures.
+ */
+function defaultE2eOpsState() {
+  return {
+    circuit_breakers: [],
+    plugin_breakers: [],
+    top_stop_reasons: [],
+    scheduler_running: null,
+    fail_job_run_ids: [],
+  };
+}
+let e2eOpsState = defaultE2eOpsState();
+
 function cloneSeed(value) {
   return JSON.parse(JSON.stringify(value));
 }
@@ -1294,6 +1309,7 @@ function handleExecassResume(socket, request) {
 
 function resetMockState() {
   resetExecassState();
+  e2eOpsState = defaultE2eOpsState();
   Object.assign(board, cloneSeed(seedState.board));
   replaceArray(columns, seedState.columns);
   replaceArray(cards, seedState.cards);
@@ -2795,25 +2811,34 @@ function getStatusPayload() {
       drift_detected: false,
     },
     scheduler_lock: getSchedulerLock(),
-    open_circuit_breakers: 0,
-    circuit_breakers: [],
-    open_plugin_breakers: 0,
-    plugin_breakers: [],
-    top_stop_reasons: [],
+    open_circuit_breakers: e2eOpsState.circuit_breakers.filter(
+      (item) => String(item.state).toLowerCase() === "open"
+    ).length,
+    circuit_breakers: e2eOpsState.circuit_breakers,
+    open_plugin_breakers: e2eOpsState.plugin_breakers.filter(
+      (item) => item.faulted
+    ).length,
+    plugin_breakers: e2eOpsState.plugin_breakers,
+    top_stop_reasons: e2eOpsState.top_stop_reasons,
   };
 }
 
 function getJobsStatusPayload() {
   const jobsEnabled = jobs.filter((job) => job.enabled).length;
   return {
-    scheduler_running: true,
+    scheduler_running:
+      typeof e2eOpsState.scheduler_running === "boolean"
+        ? e2eOpsState.scheduler_running
+        : true,
     scheduler_lock: getSchedulerLock(),
     jobs_total: jobs.length,
     jobs_enabled: jobsEnabled,
     jobs_due: 0,
-    open_circuit_breakers: 0,
-    circuit_breakers: [],
-    top_stop_reasons: [],
+    open_circuit_breakers: e2eOpsState.circuit_breakers.filter(
+      (item) => String(item.state).toLowerCase() === "open"
+    ).length,
+    circuit_breakers: e2eOpsState.circuit_breakers,
+    top_stop_reasons: e2eOpsState.top_stop_reasons,
     now_utc: new Date().toISOString(),
   };
 }
@@ -2840,25 +2865,23 @@ function buildCalendarWeek() {
         last_run_at: jobs[0].last_run_at,
         last_error: jobs[0].last_error,
         lane: "scheduled",
-        primary_action: "pause",
-      },
-    ],
-    jobs: [
-      {
-        job_id: jobs[0].job_id,
-        name: jobs[0].name,
-        agent_id: jobs[0].agent_id,
-        enabled: jobs[0].enabled,
-        schedule_kind: jobs[0].schedule_kind,
-        interval_seconds: jobs[0].interval_seconds,
-        cron_expr: jobs[0].cron_expr,
-        next_run_at: jobs[0].next_run_at,
-        last_run_at: jobs[0].last_run_at,
-        last_error: jobs[0].last_error,
-        lane: "scheduled",
-        primary_action: "pause",
+        primary_action: jobs[0].enabled ? "pause" : "resume",
       },
     ],
+    jobs: jobs.map((job) => ({
+      job_id: job.job_id,
+      name: job.name,
+      agent_id: job.agent_id,
+      enabled: job.enabled,
+      schedule_kind: job.schedule_kind,
+      interval_seconds: job.interval_seconds,
+      cron_expr: job.cron_expr,
+      next_run_at: job.next_run_at,
+      last_run_at: job.last_run_at,
+      last_error: job.last_error,
+      lane: "scheduled",
+      primary_action: job.enabled ? "pause" : "resume",
+    })),
   };
 }
 
@@ -4751,6 +4774,10 @@ async function routeRequest(req, res) {
       sendJson(res, 404, { error: "job not found" });
       return;
     }
+    if (e2eOpsState.fail_job_run_ids.includes(jobId)) {
+      sendJson(res, 500, { error: "e2e forced job run failure" });
+      return;
+    }
     const now = Date.now();
     job.last_run_at = now;
     job.last_error = null;
@@ -5212,7 +5239,7 @@ async function routeRequest(req, res) {
   if (req.method === "GET" && requestUrl.pathname === "/api/v1/extensions/plugins/status") {
     sendJson(res, 200, {
       contract_version: "stub-v1",
-      items: [],
+      items: e2eOpsState.plugin_breakers,
     });
     return;
   }
@@ -5301,6 +5328,99 @@ async function routeRequest(req, res) {
     return;
   }
 
+  if (req.method === "POST" && requestUrl.pathname === "/api/v1/e2e/ops-state") {
+    const payload = await readJson(req);
+    const isRecord = (value) =>
+      value !== null && typeof value === "object" && !Array.isArray(value);
+    const validCircuitBreaker = (value) =>
+      isRecord(value) &&
+      typeof value.scope === "string" &&
+      typeof value.target_id === "string" &&
+      typeof value.state === "string" &&
+      Number.isFinite(value.consecutive_failures) &&
+      (value.cooldown_until === null || Number.isFinite(value.cooldown_until)) &&
+      Number.isFinite(value.updated_at);
+    const validPluginBreaker = (value) =>
+      isRecord(value) &&
+      typeof value.plugin_id === "string" &&
+      typeof value.enabled === "boolean" &&
+      typeof value.faulted === "boolean" &&
+      Number.isFinite(value.consecutive_failures);
+    const validStopReason = (value) =>
+      isRecord(value) &&
+      typeof value.code === "string" &&
+      Number.isFinite(value.count);
+    const invalidArrayElement =
+      (Array.isArray(payload.circuit_breakers) &&
+        !payload.circuit_breakers.every(validCircuitBreaker)) ||
+      (Array.isArray(payload.plugin_breakers) &&
+        !payload.plugin_breakers.every(validPluginBreaker)) ||
+      (Array.isArray(payload.top_stop_reasons) &&
+        !payload.top_stop_reasons.every(validStopReason));
+    if (invalidArrayElement) {
+      sendJson(res, 400, {
+        error:
+          "ops-state breaker and stop-reason arrays contain an invalid element",
+      });
+      return;
+    }
+    if (Array.isArray(payload.circuit_breakers)) {
+      e2eOpsState.circuit_breakers = payload.circuit_breakers;
+    }
+    if (Array.isArray(payload.plugin_breakers)) {
+      e2eOpsState.plugin_breakers = payload.plugin_breakers;
+    }
+    if (Array.isArray(payload.top_stop_reasons)) {
+      e2eOpsState.top_stop_reasons = payload.top_stop_reasons;
+    }
+    if (typeof payload.scheduler_running === "boolean" || payload.scheduler_running === null) {
+      e2eOpsState.scheduler_running = payload.scheduler_running;
+    }
+    if (Array.isArray(payload.fail_job_run_ids)) {
+      e2eOpsState.fail_job_run_ids = payload.fail_job_run_ids.filter(
+        (value) => typeof value === "string"
+      );
+    }
+    if (Array.isArray(payload.append_jobs)) {
+      const now = Date.now();
+      for (const [index, extra] of payload.append_jobs.entries()) {
+        if (!extra || typeof extra !== "object") continue;
+        const jobId =
+          typeof extra.job_id === "string" && extra.job_id.length > 0
+            ? extra.job_id
+            : `e2e-job-${index + 1}`;
+        if (jobs.some((item) => item.job_id === jobId)) continue;
+        jobs.push({
+          job_id: jobId,
+          agent_id: typeof extra.agent_id === "string" ? extra.agent_id : "default",
+          name: typeof extra.name === "string" ? extra.name : jobId,
+          enabled: typeof extra.enabled === "boolean" ? extra.enabled : true,
+          schedule_kind:
+            typeof extra.schedule_kind === "string" ? extra.schedule_kind : "interval",
+          interval_seconds: Number.isFinite(extra.interval_seconds)
+            ? extra.interval_seconds
+            : 600,
+          run_at_ms: null,
+          cron_expr: typeof extra.cron_expr === "string" ? extra.cron_expr : null,
+          next_run_at: now + 600_000,
+          payload_json: "{}",
+          max_retries: 1,
+          retry_backoff_ms: 1000,
+          timeout_ms: 30_000,
+          last_run_at: null,
+          last_error: typeof extra.last_error === "string" ? extra.last_error : null,
+          created_at: now,
+          updated_at: now,
+        });
+      }
+    }
+    sendJson(res, 200, {
+      ok: true,
+      ops_state: e2eOpsState,
+    });
+    return;
+  }
+
   if (req.method === "POST" && requestUrl.pathname === "/api/v1/e2e/ws-malformed") {
     const payload = await readJson(req);
     const raw =

apps/mission-control/e2e/p3-workflows.spec.ts

@@ -173,6 +173,8 @@ test.describe("mission-control phase 3 operator workflows @p3", () => {
     await waitForWsConnected(page);
 
     await page.locator('[data-tour-id="nav-focus"]').click();
+    // Focus keeps its last mounted section, so select Queue explicitly.
+    await page.getByRole("tab", { name: /^Queue/ }).click();
     await expect(page.getByText("Operator Focus Queue")).toBeVisible();
     await expect(page.getByText(/Approvals:\s*2/)).toBeVisible();