fix(onboard): require a healthy endpoint before the router startup poll succeeds

The Model Router startup poll accepted any 2xx /health, while the final
snapshot taken after the poll required the body to name at least one healthy
endpoint. When the routed credential is rejected, every upstream fails fast,
/health answers 200 with an empty healthy_endpoints list well inside the
3-second liveness budget, and onboarding reported the router started before
every sandbox request returned 401.

Read the body in the poll as well, so both acceptance paths apply the rule
the file already documents. The poll keeps its 3-second request budget and
its full retry window, so a router that needs longer to bring endpoints up is
still accepted as soon as /health names one.

Signed-off-by: Udaya Tejas <udayatejas2004@gmail.com>

Author: udsy19

src/lib/onboard/model-router-process.ts

@@ -41,8 +41,8 @@ const ROUTER_HEALTH_BODY_MAX_BYTES = 64 * 1024;
 
 /**
  * Fetch /health and keep the response body for diagnosis (#8962). Unlike
- * `isRouterHealthy`, this waits for the body, so callers pass a longer
- * timeout; `startModelRouter` uses 30 seconds. LiteLLM's /health probes
+ * `isRouterHealthy`, this waits for the body, so a caller that must read it
+ * budgets for it; the final startup snapshot uses 30 seconds. LiteLLM's /health probes
  * every upstream endpoint per request and can answer well after the
  * 3-second liveness budget. The timeout is a wall-clock deadline, not a
  * socket idle timeout, so a responder that trickles bytes cannot hold the

src/lib/onboard/model-router.ts

@@ -61,9 +61,9 @@ const ROUTER_HEALTH_INTERVAL_MS = 2000;
 const ROUTER_STARTUP_TIMEOUT_MS = 10 * 60_000;
 // LiteLLM's /health live-probes every upstream endpoint per request, so it
 // can need far longer than the 3-second liveness budget to answer (#8962).
-// The startup poll keeps the 3-second budget: the status-only liveness
-// probe must not accept a fast 200 that names zero healthy endpoints, so
-// recovery for a slow-but-healthy router runs through the body-checked
+// The startup poll keeps the 3-second budget and reads the body, so it
+// never accepts a fast 200 that names zero healthy endpoints; recovery for
+// a router whose /health outruns that budget runs through the body-checked
 // final snapshot after the poll exhausts its retries.
 const ROUTER_FINAL_HEALTH_SNAPSHOT_TIMEOUT_MS = 30_000;
 const ROUTER_LOG_TAIL_LINES = 20;
@@ -468,7 +468,8 @@ export async function startModelRouter(
       Math.min(ROUTER_HEALTH_REQUEST_TIMEOUT_MS, Math.ceil(remainingMs)),
     );
     healthAttempts += 1;
-    const healthy = await deps.isRouterHealthy(port, healthTimeoutMs);
+    const pollSnapshot = await deps.getRouterHealthSnapshot(port, healthTimeoutMs);
+    const healthy = pollSnapshot.healthy && hasHealthyEndpoint(pollSnapshot.body);
     const processAlive = deps.isProcessAlive(pid);
     if (healthy && processAlive) return pid;
     if (!processAlive) {

test/onboard-model-router.test.ts

@@ -43,6 +43,10 @@ const MODEL_ROUTER_FINGERPRINT_FILE = ".nemoclaw-source-fingerprint";
 const MODEL_ROUTER_TEST_SOURCE_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
 const MODEL_ROUTER_TEST_VERSION = "0.1.0";
 const NVIDIA_TEST_CREDENTIAL = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY";
+const ROUTER_HEALTHY_BODY = JSON.stringify({
+  healthy_endpoints: [{ api_base: "https://integrate.api.nvidia.com/v1" }],
+  unhealthy_endpoints: [],
+});
 
 type PrepareCall = {
   venvDir: string;
@@ -295,7 +299,6 @@ describe("onboard Model Router setup", () => {
     const port = 45_678;
     const healthChecks: number[] = [];
     const sleepCalls: number[] = [];
-    let healthProbe = 0;
     let pid: number | null = null;
 
     const blueprintDir = path.join(rootDir, "nemoclaw-blueprint");
@@ -350,8 +353,11 @@ describe("onboard Model Router setup", () => {
                 name === "ROUTER_API_KEY" ? "router-secret" : null,
               isRouterHealthy: async (routerPort) => {
                 healthChecks.push(routerPort);
-                healthProbe += 1;
-                return healthProbe > 1;
+                return false;
+              },
+              getRouterHealthSnapshot: async (routerPort) => {
+                healthChecks.push(routerPort);
+                return { healthy: true, body: ROUTER_HEALTHY_BODY };
               },
               sleep: async (milliseconds) => {
                 sleepCalls.push(milliseconds);
@@ -406,7 +412,6 @@ describe("onboard Model Router setup", () => {
     const homeDir = path.join(tmpDir, "home");
     const routerCommand = path.join(tmpDir, "managed", "model-router");
     const port = 45_692;
-    let healthProbe = 0;
     let pid: number | null = null;
 
     fs.mkdirSync(path.join(rootDir, "nemoclaw-blueprint", "router"), { recursive: true });
@@ -433,10 +438,8 @@ describe("onboard Model Router setup", () => {
           homeDir,
           ensureModelRouterCommand: () => routerCommand,
           resolveProviderCredential: () => null,
-          isRouterHealthy: async () => {
-            healthProbe += 1;
-            return healthProbe > 1;
-          },
+          isRouterHealthy: async () => false,
+          getRouterHealthSnapshot: async () => ({ healthy: true, body: ROUTER_HEALTHY_BODY }),
           sleep: async () => undefined,
         },
       );
@@ -475,9 +478,10 @@ describe("onboard Model Router setup", () => {
         }),
         resolveProviderCredential: () => null,
         buildSubprocessEnv: () => ({}),
-        isRouterHealthy: async () => {
+        isRouterHealthy: async () => false,
+        getRouterHealthSnapshot: async () => {
           healthProbe += 1;
-          return healthProbe > 61;
+          return { healthy: healthProbe > 60, body: ROUTER_HEALTHY_BODY };
         },
         sleep,
         isProcessAlive: () => true,
@@ -487,7 +491,7 @@ describe("onboard Model Router setup", () => {
     );
 
     assert.equal(startedPid, pid);
-    assert.equal(healthProbe, 62);
+    assert.equal(healthProbe, 61);
     assert.equal(sleep.mock.calls.length, 61);
     assert.equal(terminateProcess.mock.calls.length, 0);
   });
@@ -496,7 +500,7 @@ describe("onboard Model Router setup", () => {
     const pid = 12_345;
     const sleep = vi.fn(async () => undefined);
     const terminateProcess = vi.fn();
-    const isRouterHealthy = vi.fn(async () => false);
+    const getRouterHealthSnapshot = vi.fn(async () => ({ healthy: false, body: null }));
 
     await assert.rejects(
       startModelRouter(
@@ -515,8 +519,8 @@ describe("onboard Model Router setup", () => {
           }),
           resolveProviderCredential: () => null,
           buildSubprocessEnv: () => ({}),
-          isRouterHealthy,
-          getRouterHealthSnapshot: async () => ({ healthy: false, body: null }),
+          isRouterHealthy: async () => false,
+          getRouterHealthSnapshot,
           sleep,
           isProcessAlive: () => true,
           terminateProcess,
@@ -526,15 +530,14 @@ describe("onboard Model Router setup", () => {
       /failed to become healthy on port 45680 within 600 seconds \(completed health checks: 300\)/,
     );
 
-    assert.equal(isRouterHealthy.mock.calls.length, 301);
+    assert.equal(getRouterHealthSnapshot.mock.calls.length, 301);
     assert.equal(sleep.mock.calls.length, 300);
     assert.deepEqual(terminateProcess.mock.calls, [[pid]]);
   });
 
   it("sets OPENAI_API_KEY to the routed credential when an ambient OPENAI_API_KEY exists (#8962)", async () => {
     const pid = 12_345;
     let spawnedEnv: Record<string, string> | null = null;
-    let healthProbe = 0;
 
     await startModelRouter(
       { port: 45_690, pool_config_path: "router/test-pool.yaml", credential_env: "ROUTER_API_KEY" },
@@ -558,10 +561,8 @@ describe("onboard Model Router setup", () => {
         resolveProviderCredential: (name) =>
           ({ ROUTER_API_KEY: "router-secret", OPENAI_API_KEY: "stale-openai" })[name] ?? null,
         buildSubprocessEnv: (extra) => ({ ...extra }),
-        isRouterHealthy: async () => {
-          healthProbe += 1;
-          return healthProbe > 1;
-        },
+        isRouterHealthy: async () => false,
+        getRouterHealthSnapshot: async () => ({ healthy: true, body: ROUTER_HEALTHY_BODY }),
         sleep: async () => undefined,
         isProcessAlive: () => true,
         terminateProcess: () => undefined,
@@ -643,10 +644,6 @@ describe("onboard Model Router setup", () => {
   it("returns the router PID when the final health snapshot proves recovery (#8962)", async () => {
     const pid = 12_345;
     const terminateProcess = vi.fn();
-    const healthyBody = JSON.stringify({
-      healthy_endpoints: [{ api_base: "https://integrate.api.nvidia.com/v1" }],
-      unhealthy_endpoints: [],
-    });
 
     const startedPid = await startModelRouter(
       { port: 45_693, pool_config_path: "router/test-pool.yaml" },
@@ -665,7 +662,12 @@ describe("onboard Model Router setup", () => {
         resolveProviderCredential: () => null,
         buildSubprocessEnv: () => ({}),
         isRouterHealthy: async () => false,
-        getRouterHealthSnapshot: async () => ({ healthy: true, body: healthyBody }),
+        // /health outruns the poll's 3-second budget and answers only within
+        // the 30-second final-snapshot budget.
+        getRouterHealthSnapshot: async (_port: number, timeoutMs = 0) => ({
+          healthy: timeoutMs >= 30_000,
+          body: timeoutMs >= 30_000 ? ROUTER_HEALTHY_BODY : null,
+        }),
         sleep: async () => undefined,
         isProcessAlive: () => true,
         terminateProcess,
@@ -722,7 +724,7 @@ describe("onboard Model Router setup", () => {
     );
   });
 
-  it("still fails when the final snapshot is 2xx with zero healthy endpoints (#8962)", async () => {
+  it("still fails when the poll and final snapshot are 2xx with zero healthy endpoints (#8962)", async () => {
     const pid = 12_345;
     const terminateProcess = vi.fn();
     const allUnhealthyBody = JSON.stringify({
@@ -747,7 +749,9 @@ describe("onboard Model Router setup", () => {
           }),
           resolveProviderCredential: () => null,
           buildSubprocessEnv: () => ({}),
-          isRouterHealthy: async () => false,
+          // The pre-spawn port guard calls isRouterHealthy without a timeout;
+          // only the startup poll passes one. Answer 2xx for the poll alone.
+          isRouterHealthy: async (_port: number, timeoutMs) => timeoutMs !== undefined,
           getRouterHealthSnapshot: async () => ({ healthy: true, body: allUnhealthyBody }),
           sleep: async () => undefined,
           isProcessAlive: () => true,
@@ -827,7 +831,6 @@ describe("onboard Model Router setup", () => {
       const pid = 12_345;
       let spawnedEnv: Record<string, string> | null = null;
 
-      let healthProbe = 0;
       await startModelRouter(
         {
           port: 45_695,
@@ -853,10 +856,8 @@ describe("onboard Model Router setup", () => {
           resolveProviderCredential: (name) =>
             ({ ROUTER_API_KEY: "router-secret", OPENAI_API_KEY: "operator-openai" })[name] ?? null,
           buildSubprocessEnv: (extra) => ({ ...extra }),
-          isRouterHealthy: async () => {
-            healthProbe += 1;
-            return healthProbe > 1;
-          },
+          isRouterHealthy: async () => false,
+          getRouterHealthSnapshot: async () => ({ healthy: true, body: ROUTER_HEALTHY_BODY }),
           sleep: async () => undefined,
           isProcessAlive: () => true,
           terminateProcess: () => undefined,
@@ -874,7 +875,6 @@ describe("onboard Model Router setup", () => {
   it("preserves routed credential fallback for an unproven pool (#8962)", async () => {
     const pid = 12_345;
     let spawnedEnv: Record<string, string> | null = null;
-    let healthProbe = 0;
 
     await startModelRouter(
       {
@@ -900,10 +900,8 @@ describe("onboard Model Router setup", () => {
         readPoolConfig: () => 'models:\n  - litellm_model: "openai/gpt-test"\n',
         resolveProviderCredential: (name) => (name === "ROUTER_API_KEY" ? "router-secret" : null),
         buildSubprocessEnv: (extra) => ({ ...extra }),
-        isRouterHealthy: async () => {
-          healthProbe += 1;
-          return healthProbe > 1;
-        },
+        isRouterHealthy: async () => false,
+        getRouterHealthSnapshot: async () => ({ healthy: true, body: ROUTER_HEALTHY_BODY }),
         sleep: async () => undefined,
         isProcessAlive: () => true,
         terminateProcess: () => undefined,
@@ -924,9 +922,9 @@ describe("onboard Model Router setup", () => {
     const sleep = vi.fn(async (milliseconds: number) => {
       nowMs += milliseconds;
     });
-    const isRouterHealthy = vi.fn(async (_port: number, timeoutMs = 0) => {
+    const getRouterHealthSnapshot = vi.fn(async (_port: number, timeoutMs = 0) => {
       nowMs += timeoutMs;
-      return false;
+      return { healthy: false, body: null };
     });
 
     await assert.rejects(
@@ -946,11 +944,8 @@ describe("onboard Model Router setup", () => {
           }),
           resolveProviderCredential: () => null,
           buildSubprocessEnv: () => ({}),
-          isRouterHealthy,
-          getRouterHealthSnapshot: async (_port: number, timeoutMs = 0) => {
-            nowMs += timeoutMs;
-            return { healthy: false, body: null };
-          },
+          isRouterHealthy: async () => false,
+          getRouterHealthSnapshot,
           sleep,
           now: () => nowMs,
           isProcessAlive: () => true,
@@ -965,7 +960,7 @@ describe("onboard Model Router setup", () => {
     );
 
     assert.equal(nowMs, 600_000);
-    assert.equal(isRouterHealthy.mock.calls.length, 115);
+    assert.equal(getRouterHealthSnapshot.mock.calls.length, 115);
     assert.equal(sleep.mock.calls.length, 114);
     assert.deepEqual(terminateProcess.mock.calls, [[pid]]);
   });
@@ -980,7 +975,6 @@ describe("onboard Model Router setup", () => {
     const mkdirSync = vi.fn();
     const proxyConfigArgs: string[][] = [];
     const proxyArgs: string[][] = [];
-    let healthProbe = 0;
     vi.stubEnv("HOME", homeDir);
     vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "9123");
     vi.resetModules();
@@ -1008,10 +1002,8 @@ describe("onboard Model Router setup", () => {
         },
         resolveProviderCredential: () => null,
         buildSubprocessEnv: () => ({}),
-        isRouterHealthy: async () => {
-          healthProbe += 1;
-          return healthProbe > 1;
-        },
+        isRouterHealthy: async () => false,
+        getRouterHealthSnapshot: async () => ({ healthy: true, body: ROUTER_HEALTHY_BODY }),
         sleep: async () => undefined,
         isProcessAlive: () => true,
         terminateProcess: () => undefined,