refactor(core): centralize result retries

Author: cv

src/lib/core/retry.ts

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+interface RetryUntilBaseOptions<T> {
+  /** Return true when the current result completes the retry operation. */
+  accept: (result: T, attempt: number) => boolean;
+  /** Delays before each additional attempt. */
+  retryDelaysMs: readonly number[];
+}
+
+export type RetryUntilOptions<T> = RetryUntilBaseOptions<T> & {
+  /** Sleep function used between attempts. */
+  sleep: (ms: number) => void;
+};
+
+export type RetryUntilAsyncOptions<T> = RetryUntilBaseOptions<T> & {
+  /** Sleep function used between attempts. */
+  sleep: (ms: number) => Promise<void>;
+};
+
+/**
+ * Retry a synchronous operation until its result is accepted or the delay
+ * schedule is exhausted. Returns the final operation result.
+ */
+export function retryUntil<T>(operation: (attempt: number) => T, options: RetryUntilOptions<T>): T {
+  let attempt = 1;
+  let result = operation(attempt);
+  if (options.accept(result, attempt)) return result;
+
+  for (const delayMs of options.retryDelaysMs) {
+    options.sleep(delayMs);
+    attempt += 1;
+    result = operation(attempt);
+    if (options.accept(result, attempt)) return result;
+  }
+  return result;
+}
+
+/**
+ * Retry an asynchronous operation until its result is accepted or the delay
+ * schedule is exhausted. Returns the final operation result.
+ */
+export async function retryUntilAsync<T>(
+  operation: (attempt: number) => T | Promise<T>,
+  options: RetryUntilAsyncOptions<T>,
+): Promise<T> {
+  let attempt = 1;
+  let result = await operation(attempt);
+  if (options.accept(result, attempt)) return result;
+
+  for (const delayMs of options.retryDelaysMs) {
+    await options.sleep(delayMs);
+    attempt += 1;
+    result = await operation(attempt);
+    if (options.accept(result, attempt)) return result;
+  }
+  return result;
+}

src/lib/shields/inference-convergence.ts

@@ -7,6 +7,8 @@ import {
 } from "../actions/sandbox/connect-inference-route-probe";
 import { buildOpenshellCommand } from "../adapters/openshell/command-argv";
 
+import { retryUntil } from "../core/retry";
+
 const DEFAULT_MAX_ATTEMPTS = 4;
 const DEFAULT_RETRY_DELAY_MS = 500;
 const INFERENCE_ROUTE_PROBE_TIMEOUT_MS = 10_000;
@@ -65,28 +67,34 @@ export function waitForHermesInferenceRouteConvergence(
     ? Math.max(0, Math.trunc(configuredRetryDelayMs))
     : DEFAULT_RETRY_DELAY_MS;
   const buildCommand = options.buildOpenshellCommand ?? buildOpenshellCommand;
-  const sleep = options.sleep ?? sleepMs;
-  let httpStatus = 0;
-
-  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-    const probe = options.run(
-      buildCommand(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" })),
-      {
-        ignoreError: true,
-        suppressOutput: true,
-        timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
-      },
-    );
-    const parsed = parseSandboxInferenceRouteProbeResult({
-      status: probe.status,
-      output: String(probe.stdout ?? ""),
-      stderr: String(probe.stderr ?? ""),
-    });
-    httpStatus = parsed.httpStatus;
-    const usable = parsed.healthy && httpStatus >= 200 && httpStatus < 300;
-    if (usable) return { ok: true, attempts: attempt, httpStatus };
-    if (attempt < maxAttempts) sleep(retryDelayMs);
-  }
+  const retryDelaysMs = Array.from({ length: maxAttempts - 1 }, () => retryDelayMs);
 
-  return { ok: false, attempts: maxAttempts, httpStatus };
+  return retryUntil(
+    (attempt) => {
+      const probe = options.run(
+        buildCommand(buildSandboxInferenceRouteProbeArgs(sandboxName, { name: "hermes" })),
+        {
+          ignoreError: true,
+          suppressOutput: true,
+          timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
+        },
+      );
+      const parsed = parseSandboxInferenceRouteProbeResult({
+        status: probe.status,
+        output: String(probe.stdout ?? ""),
+        stderr: String(probe.stderr ?? ""),
+      });
+      const httpStatus = parsed.httpStatus;
+      return {
+        ok: parsed.healthy && httpStatus >= 200 && httpStatus < 300,
+        attempts: attempt,
+        httpStatus,
+      };
+    },
+    {
+      accept: (result) => result.ok,
+      retryDelaysMs,
+      sleep: options.sleep ?? sleepMs,
+    },
+  );
 }

src/lib/verify-deployment.ts

@@ -21,6 +21,8 @@ import { parseVersionFromText } from "./adapters/openshell/client";
 import { compareChannelSets, type RuntimeChannelStatus } from "./channel-runtime-status";
 import type { DashboardDeliveryChain } from "./dashboard/contract";
 import { listMessagingChannelsWithoutCredentials } from "./messaging/channels";
+
+import { retryUntilAsync } from "./core/retry";
 import {
   buildCustomOpenClawRuntimeFailureHints,
   classifyOpenClawRuntimeFailure,
@@ -198,14 +200,11 @@ async function verifyGatewayInSandbox(
   retryDelaysMs: readonly number[],
   sleep: (ms: number) => Promise<void>,
 ): Promise<{ reachable: boolean; httpCode: number; detail: string }> {
-  let last = probeGatewayInSandboxOnce(sandboxName, chain, deps);
-  if (last.reachable) return last;
-  for (const delayMs of retryDelaysMs) {
-    await sleep(delayMs);
-    last = probeGatewayInSandboxOnce(sandboxName, chain, deps);
-    if (last.reachable) return last;
-  }
-  return last;
+  return retryUntilAsync(() => probeGatewayInSandboxOnce(sandboxName, chain, deps), {
+    accept: (result) => result.reachable,
+    retryDelaysMs,
+    sleep,
+  });
 }
 
 /**
@@ -253,14 +252,11 @@ async function verifyInferenceRoute(
   retryDelaysMs: readonly number[],
   sleep: (ms: number) => Promise<void>,
 ): Promise<{ status: InferenceRouteStatus; detail: string }> {
-  let last = probeInferenceRouteOnce(sandboxName, deps);
-  if (last.status === "ok") return last;
-  for (const delayMs of retryDelaysMs) {
-    await sleep(delayMs);
-    last = probeInferenceRouteOnce(sandboxName, deps);
-    if (last.status === "ok") return last;
-  }
-  return last;
+  return retryUntilAsync(() => probeInferenceRouteOnce(sandboxName, deps), {
+    accept: (result) => result.status === "ok",
+    retryDelaysMs,
+    sleep,
+  });
 }
 
 /**
@@ -289,14 +285,11 @@ async function verifyDashboardFromHost(
   retryDelaysMs: readonly number[],
   sleep: (ms: number) => Promise<void>,
 ): Promise<{ reachable: boolean; detail: string }> {
-  let last = probeDashboardFromHostOnce(chain, deps);
-  if (last.reachable) return last;
-  for (const delayMs of retryDelaysMs) {
-    await sleep(delayMs);
-    last = probeDashboardFromHostOnce(chain, deps);
-    if (last.reachable) return last;
-  }
-  return last;
+  return retryUntilAsync(() => probeDashboardFromHostOnce(chain, deps), {
+    accept: (result) => result.reachable,
+    retryDelaysMs,
+    sleep,
+  });
 }
 
 /**

test/wait.test.ts

@@ -4,6 +4,8 @@
 import assert from "node:assert";
 import { createServer, type AddressInfo } from "node:net";
 import { afterEach, describe, expect, it, vi } from "vitest";
+import { retryUntil, retryUntilAsync } from "../src/lib/core/retry.js";
+
 import {
   buildLoopbackProbeEnv,
   sleepMs,
@@ -47,6 +49,167 @@ describe("wait utility", () => {
     assert.ok(duration < 50, `duration ${duration}ms > 50ms`);
   });
 
+  it("retryUntil returns the first accepted result without sleeping (#9218)", () => {
+    const sleep = vi.fn();
+    const operation = vi.fn(() => "ready");
+
+    const result = retryUntil(operation, {
+      accept: (value) => value === "ready",
+      retryDelaysMs: [10, 20],
+      sleep,
+    });
+
+    expect(result).toBe("ready");
+    expect(operation).toHaveBeenCalledOnce();
+    expect(operation).toHaveBeenCalledWith(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("retryUntil applies exact delays before a later accepted result (#9218)", () => {
+    const sleep = vi.fn();
+
+    const result = retryUntil((attempt) => (attempt === 3 ? "ready" : "starting"), {
+      accept: (value) => value === "ready",
+      retryDelaysMs: [10, 20, 30],
+      sleep,
+    });
+
+    expect(result).toBe("ready");
+    expect(sleep.mock.calls).toEqual([[10], [20]]);
+  });
+
+  it("retryUntil returns the final unaccepted result after exhaustion (#9218)", () => {
+    const sleep = vi.fn();
+
+    const result = retryUntil((attempt) => `failure-${attempt}`, {
+      accept: () => false,
+      retryDelaysMs: [10, 20],
+      sleep,
+    });
+
+    expect(result).toBe("failure-3");
+    expect(sleep.mock.calls).toEqual([[10], [20]]);
+  });
+
+  it("retryUntil runs once with an empty retry schedule (#9218)", () => {
+    const operation = vi.fn(() => "failure");
+
+    const result = retryUntil(operation, {
+      accept: () => false,
+      retryDelaysMs: [],
+
+      sleep: vi.fn(),
+    });
+
+    expect(result).toBe("failure");
+    expect(operation).toHaveBeenCalledOnce();
+  });
+
+  it("retryUntil propagates an operation error without sleeping (#9218)", () => {
+    const sleep = vi.fn();
+    const error = new Error("operation failed");
+
+    expect(() =>
+      retryUntil(
+        () => {
+          throw error;
+        },
+        { accept: () => false, retryDelaysMs: [10], sleep },
+      ),
+    ).toThrow(error);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("retryUntil propagates a sleep error without another attempt (#9218)", () => {
+    const operation = vi.fn(() => "starting");
+    const error = new Error("sleep failed");
+
+    expect(() =>
+      retryUntil(operation, {
+        accept: () => false,
+        retryDelaysMs: [10],
+        sleep: () => {
+          throw error;
+        },
+      }),
+    ).toThrow(error);
+    expect(operation).toHaveBeenCalledOnce();
+  });
+
+  it("retryUntilAsync applies exact delays before a later accepted result (#9218)", async () => {
+    const sleep = vi.fn(async () => {});
+
+    const result = await retryUntilAsync(
+      async (attempt) => (attempt === 3 ? "ready" : "starting"),
+      {
+        accept: (value) => value === "ready",
+        retryDelaysMs: [10, 20, 30],
+        sleep,
+      },
+    );
+
+    expect(result).toBe("ready");
+    expect(sleep.mock.calls).toEqual([[10], [20]]);
+  });
+
+  it("retryUntilAsync runs once with an empty retry schedule (#9218)", async () => {
+    const operation = vi.fn(async () => "failure");
+
+    const result = await retryUntilAsync(operation, {
+      accept: () => false,
+      retryDelaysMs: [],
+
+      sleep: vi.fn(async () => {}),
+    });
+
+    expect(result).toBe("failure");
+    expect(operation).toHaveBeenCalledOnce();
+  });
+
+  it("retryUntilAsync returns the final unaccepted result after exhaustion (#9218)", async () => {
+    const sleep = vi.fn(async () => {});
+
+    const result = await retryUntilAsync(async (attempt) => `failure-${attempt}`, {
+      accept: () => false,
+      retryDelaysMs: [10, 20],
+      sleep,
+    });
+
+    expect(result).toBe("failure-3");
+    expect(sleep.mock.calls).toEqual([[10], [20]]);
+  });
+
+  it("retryUntilAsync propagates an operation error without sleeping (#9218)", async () => {
+    const sleep = vi.fn(async () => {});
+    const error = new Error("operation failed");
+
+    await expect(
+      retryUntilAsync(
+        async () => {
+          throw error;
+        },
+        { accept: () => false, retryDelaysMs: [10], sleep },
+      ),
+    ).rejects.toBe(error);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("retryUntilAsync propagates a sleep error without another attempt (#9218)", async () => {
+    const operation = vi.fn(async () => "starting");
+    const error = new Error("sleep failed");
+
+    await expect(
+      retryUntilAsync(operation, {
+        accept: () => false,
+        retryDelaysMs: [10],
+        sleep: async () => {
+          throw error;
+        },
+      }),
+    ).rejects.toBe(error);
+    expect(operation).toHaveBeenCalledOnce();
+  });
+
   it("waitUntil returns immediately when the condition is already true", () => {
     const sleeps: number[] = [];
     let attempts = 0;