perf(onboard): use deadlines for gateway health waits

Signed-off-by: Ho Lim <subhoya@gmail.com>

Author: HOYALIM

src/lib/onboard.ts

@@ -325,9 +325,8 @@ const {
   rejectUnsupportedWindowsHostOllama,
   shouldFrontOllamaWithProxy,
 }: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology");
-const {
-  waitForGatewayHealth,
-}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait");
+const { waitForGatewayHealth }: typeof import("./onboard/gateway-health-wait") =
+  require("./onboard/gateway-health-wait");
 const { resolveOpenshell } = require("./adapters/openshell/resolve");
 const credentials: typeof import("./credentials/store") = require("./credentials/store");
 const {
@@ -1896,7 +1895,7 @@ async function startGatewayWithOptions(
           return;
         }
 
-        throw new Error("Gateway failed to start");
+        throw new Error(`Gateway failed within ${healthWait.count * healthWait.interval}s.`);
       },
       {
         retries,
@@ -1912,11 +1911,9 @@ async function startGatewayWithOptions(
         },
       },
     );
-  } catch {
-    if (exitOnFailure) {
-      handleFinalGatewayStartFailure({ retries, dockerUnreachable });
-    }
-    throw new Error("Gateway failed to start");
+  } catch (error) {
+    if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable });
+    throw error instanceof Error ? error : new Error(String(error));
   }
 
   console.log("  ✓ Gateway is healthy");

src/lib/onboard/__test-helpers__/virtual-clock.ts

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { vi } from "vitest";
+
+export function createVirtualClock(startMs = 1_000_000_000_000) {
+  let currentMs = startMs;
+  const advance = (seconds: number) => {
+    currentMs += Math.max(0, seconds) * 1000;
+  };
+  return {
+    advance,
+    now: () => currentMs,
+    sleeper: vi.fn(advance),
+  };
+}

src/lib/onboard/docker-driver-gateway-service.test.ts

@@ -3,13 +3,14 @@
 
 import { describe, expect, it, vi } from "vitest";
 
+import { createVirtualClock } from "./__test-helpers__/virtual-clock";
 import {
   getOpenShellGatewayUserServiceBinaryPaths,
   getOpenShellGatewayUserServicePaths,
   hasOpenShellGatewayUserService,
-  startPackageManagedDockerDriverGateway,
-  startOpenShellGatewayUserService,
   type SpawnSyncLikeResult,
+  startOpenShellGatewayUserService,
+  startPackageManagedDockerDriverGateway,
 } from "./docker-driver-gateway-service";
 
 const STATUS_CONNECTED = `
@@ -254,6 +255,7 @@ describe("docker-driver-gateway-service", () => {
 
   it("uses the package-managed service only after endpoint, metadata, and gRPC health are ready", async () => {
     const events: string[] = [];
+    const clock = createVirtualClock();
     let registerCount = 0;
     const registerDockerDriverGatewayEndpoint = vi.fn(() => {
       events.push("register");
@@ -268,14 +270,18 @@ describe("docker-driver-gateway-service", () => {
         gatewayName: "nemoclaw",
         hasOpenShellGatewayUserService: () => true,
         healthPollCount: 3,
-        healthPollInterval: 0,
+        healthPollInterval: 1,
         isDockerDriverGatewayReady: async () => {
           events.push("ready");
           return true;
         },
+        now: clock.now,
         registerDockerDriverGatewayEndpoint,
         runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
-        sleepSeconds: () => events.push("sleep"),
+        sleepSeconds: (seconds) => {
+          events.push("sleep");
+          clock.advance(seconds);
+        },
         skipSandboxBridgeReachability: false,
         startOpenShellGatewayUserService: () => ({
           attempted: true,
@@ -318,6 +324,7 @@ describe("docker-driver-gateway-service", () => {
 
   it("keeps standalone runtime breadcrumbs when service health never becomes ready", async () => {
     const clearDockerDriverGatewayRuntimeFiles = vi.fn();
+    const clock = createVirtualClock();
 
     await expect(
       startPackageManagedDockerDriverGateway({
@@ -326,9 +333,12 @@ describe("docker-driver-gateway-service", () => {
         gatewayName: "nemoclaw",
         hasOpenShellGatewayUserService: () => true,
         healthPollCount: 1,
+        healthPollInterval: 1,
         isDockerDriverGatewayReady: async () => false,
+        now: clock.now,
         registerDockerDriverGatewayEndpoint: () => true,
         runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
+        sleepSeconds: clock.advance,
         skipSandboxBridgeReachability: false,
         startOpenShellGatewayUserService: () => ({
           attempted: true,
@@ -337,7 +347,7 @@ describe("docker-driver-gateway-service", () => {
         }),
         verifySandboxBridgeGatewayReachableOrExit: vi.fn(),
       }),
-    ).rejects.toThrow("did not become healthy");
+    ).rejects.toThrow("configured 1s health deadline");
 
     expect(clearDockerDriverGatewayRuntimeFiles).not.toHaveBeenCalled();
   });

src/lib/onboard/docker-driver-gateway-service.ts

@@ -1,13 +1,14 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { spawnSync, type SpawnSyncOptions } from "node:child_process";
+import { type SpawnSyncOptions, spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 
 import { sleepSeconds, waitUntilAsync } from "../core/wait";
 import { isGatewayHealthy } from "../state/gateway";
 import { envInt } from "./env";
+import { formatGatewayHealthWaitBudget, getGatewayHealthWaitBudgetMs } from "./gateway-health-wait";
 import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness";
 
 export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway";
@@ -49,6 +50,7 @@ export interface PackageManagedDockerDriverGatewayOptions {
   healthPollCount?: number;
   healthPollInterval?: number;
   isDockerDriverGatewayReady?: () => Promise<boolean>;
+  now?: () => number;
   registerDockerDriverGatewayEndpoint: () => boolean;
   runCaptureOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => string;
   sleepSeconds?: (seconds: number) => void;
@@ -291,6 +293,7 @@ export async function startPackageManagedDockerDriverGateway({
   healthPollCount,
   healthPollInterval,
   isDockerDriverGatewayReady = isDockerDriverGatewayHttpReady,
+  now = Date.now,
   registerDockerDriverGatewayEndpoint,
   runCaptureOpenshell,
   sleepSeconds: sleepSecondsImpl = sleepSeconds,
@@ -324,6 +327,7 @@ export async function startPackageManagedDockerDriverGateway({
   const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30);
   const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2);
   const pollIntervalMs = Math.max(0, pollInterval * 1000);
+  const waitBudgetMs = getGatewayHealthWaitBudgetMs(pollCount, pollInterval);
   const healthy =
     pollCount > 0 &&
     (await waitUntilAsync(
@@ -339,10 +343,11 @@ export async function startPackageManagedDockerDriverGateway({
         );
       },
       {
+        deadlineMs: now() + waitBudgetMs,
         initialIntervalMs: pollIntervalMs,
         maxIntervalMs: pollIntervalMs,
         backoffFactor: 1,
-        maxAttempts: pollCount,
+        now,
         sleep: (ms) => sleepSecondsImpl(ms / 1000),
       },
     ));
@@ -355,7 +360,10 @@ export async function startPackageManagedDockerDriverGateway({
     return true;
   }
 
-  const message = "OpenShell gateway user service started but did not become healthy.";
+  const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitBudget(
+    pollCount,
+    pollInterval,
+  )} health deadline.`;
   console.error(`  ${message}`);
   console.error("  Check: systemctl --user status openshell-gateway");
   if (exitOnFailure) process.exit(1);

src/lib/onboard/gateway-health-wait.test.ts

@@ -3,6 +3,7 @@
 
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
+import { createVirtualClock } from "./__test-helpers__/virtual-clock";
 import { type GatewayHealthWaitOptions, waitForGatewayHealth } from "./gateway-health-wait";
 
 function buildOptions(overrides: Partial<GatewayHealthWaitOptions> = {}): GatewayHealthWaitOptions {
@@ -49,16 +50,19 @@ describe("waitForGatewayHealth", () => {
   });
 
   it("returns false when HTTP readiness never follows healthy metadata", async () => {
+    const clock = createVirtualClock();
     const options = buildOptions({
       healthPollCount: 2,
       isGatewayHttpReady: vi.fn(async () => false),
+      now: clock.now,
+      sleepSeconds: clock.sleeper,
     });
 
     await expect(waitForGatewayHealth(options)).resolves.toBe(false);
 
     expect(options.isGatewayHealthy).toHaveBeenCalledTimes(2);
     expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(2);
-    expect(options.sleepSeconds).toHaveBeenCalledTimes(1);
+    expect(options.sleepSeconds).toHaveBeenCalledTimes(2);
   });
 
   it("force-refreshes metadata after bootstrap secret repair", async () => {
@@ -85,19 +89,27 @@ describe("waitForGatewayHealth", () => {
     expect(options.attachGatewayMetadataIfNeeded).toHaveBeenCalledWith();
   });
 
-  it("stops after healthPollCount attempts without sleeping after the final failed probe", async () => {
+  it("polls until the configured health deadline instead of stopping at the count cap (#3768)", async () => {
+    const clock = createVirtualClock();
+    const isGatewayHealthy = vi.fn(() => {
+      clock.advance(1);
+      return false;
+    });
     const options = buildOptions({
-      healthPollCount: 3,
-      isGatewayHealthy: vi.fn(() => false),
+      healthPollCount: 10,
+      healthPollIntervalSeconds: 1,
+      isGatewayHealthy,
+      now: clock.now,
+      sleepSeconds: clock.sleeper,
     });
 
     await expect(waitForGatewayHealth(options)).resolves.toBe(false);
 
-    expect(options.isGatewayHealthy).toHaveBeenCalledTimes(3);
+    expect(isGatewayHealthy).toHaveBeenCalled();
+    expect(isGatewayHealthy.mock.calls.length).toBeLessThan(10);
     expect(options.isGatewayHttpReady).not.toHaveBeenCalled();
-    expect(options.sleepSeconds).toHaveBeenCalledTimes(2);
-    expect(options.sleepSeconds).toHaveBeenNthCalledWith(1, 2);
-    expect(options.sleepSeconds).toHaveBeenNthCalledWith(2, 2);
+    expect(clock.sleeper).toHaveBeenCalled();
+    expect(clock.sleeper.mock.calls.every(([seconds]) => seconds === 1)).toBe(true);
   });
 
   it("returns false without probing when healthPollCount is zero", async () => {

src/lib/onboard/gateway-health-wait.ts

@@ -16,6 +16,29 @@ export interface GatewayHealthWaitOptions {
   repairGatewayBootstrapSecrets: () => { repaired: boolean };
   runCaptureOpenshell: RunCaptureOpenshell;
   sleepSeconds: (seconds: number) => void;
+  now?: () => number;
+}
+
+export function getGatewayHealthWaitBudgetMs(
+  healthPollCount: number,
+  healthPollIntervalSeconds: number,
+): number {
+  const normalizedCount = Number.isFinite(healthPollCount) ? Math.max(0, healthPollCount) : 0;
+  const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds)
+    ? Math.max(0, healthPollIntervalSeconds)
+    : 0;
+  return normalizedCount <= 0 ? 0 : Math.max(1, normalizedCount * normalizedIntervalSeconds * 1000);
+}
+
+export function formatGatewayHealthWaitBudget(
+  healthPollCount: number,
+  healthPollIntervalSeconds: number,
+): string {
+  const budgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds);
+  if (budgetMs <= 0) return "0s";
+  if (budgetMs < 1000) return `${Math.ceil(budgetMs)}ms`;
+  const seconds = budgetMs / 1000;
+  return Number.isInteger(seconds) ? `${seconds}s` : `${seconds.toFixed(1)}s`;
 }
 
 export async function waitForGatewayHealth({
@@ -29,8 +52,10 @@ export async function waitForGatewayHealth({
   repairGatewayBootstrapSecrets,
   runCaptureOpenshell,
   sleepSeconds,
+  now = Date.now,
 }: GatewayHealthWaitOptions): Promise<boolean> {
   const healthPollIntervalMs = Math.max(0, healthPollIntervalSeconds * 1000);
+  const waitBudgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds);
   return (
     healthPollCount > 0 &&
     (await waitUntilAsync(
@@ -50,10 +75,11 @@ export async function waitForGatewayHealth({
         return isGatewayHealthy(status, namedInfo, currentInfo) && (await isGatewayHttpReady());
       },
       {
+        deadlineMs: now() + waitBudgetMs,
         initialIntervalMs: healthPollIntervalMs,
         maxIntervalMs: healthPollIntervalMs,
         backoffFactor: 1,
-        maxAttempts: healthPollCount,
+        now,
         sleep: (ms) => sleepSeconds(ms / 1000),
       },
     ))