fix(readiness): measure the reuse window from collection completion

A readiness collection stamped its start time, so a probe slower than the
30-second window aged out the facts it had just gathered and left no
retry that could succeed. Rebuild then had no working path at all: a
running gateway failed the readiness gate, and a stopped one failed the
route check, because an unanswerable gateway was treated as a route
mismatch.

The window now starts when collection finishes, the preflight
checkpoints collect gateway facts again instead of rescoring an old
snapshot, and the rebuild route check separates a gateway that cannot
answer from one that answers with another provider and model. Stale
evidence carries the measured age and the applied window.

This reverses the slow-collection rejection introduced by #8738.

Signed-off-by: Tinson Lai <tinsonl@nvidia.com>

Author: laitingsheng

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -262,7 +262,7 @@ $$nemoclaw <sandbox-name> rebuild
 
 ### Resolve Rebuild Preflight Stops
 
-Before it backs up or deletes the existing sandbox, `rebuild` validates the recorded sandbox, gateway, policy, MCP, agent, and operation-lock state.
+Before it backs up or deletes the existing sandbox, `rebuild` validates the recorded sandbox, gateway, inference route, policy, MCP, agent, and operation-lock state.
 When one of these checks fails, NemoClaw prints `Rebuild preflight failed`, explains how to recover, and ends with `Aborting rebuild`.
 At this boundary, the existing sandbox is unchanged and no sandbox data has been removed.
 
@@ -274,6 +274,10 @@ Use the recovery guidance that matches the reported check:
 - Resolve an incomplete MCP destroy transaction before retrying.
 - Back up the sandbox state and recreate it with `$$nemoclaw onboard` when the record contains multiple agents. Transactional multi-agent rebuild is not supported.
 - Wait for another onboarding or rebuild operation to finish before retrying. If verified stale-lock cleanup is still in progress, wait briefly and rerun the command. Do not delete the lock manually.
+- Set the live OpenShell inference route to the sandbox's recorded provider and model when rebuild reports route drift.
+
+A gateway that reports no live inference route does not stop the rebuild.
+Replacement onboarding configures and verifies the recorded route before it recreates the sandbox.
 
 <AgentOnly variant="openclaw">
 The rebuild command preserves the mounted workspace and registered policies while recreating the container.

docs/reference/system-readiness.mdx

@@ -289,8 +289,13 @@ Fresh onboarding and authoritative rebuilds use this order:
 8. Revalidate gateway authority immediately before gateway selection, recovery, reconciliation, or other lifecycle effects.
 
 The readiness gate runs before model-router cleanup, provider selection, credential registration, policy changes, image builds, or sandbox lifecycle effects.
-Host and gateway observations have a 30-second reuse window.
-If collection itself takes too long, onboarding rejects the stale composite instead of assigning a fresh timestamp to old facts.
+Host and gateway observations have a 30-second reuse window that starts when collection finishes.
+The readiness gate does not reject a collection for the time its own probes take.
+The `gateway.owner` evidence records the gateway collection duration as `collectionMs`.
+The readiness gate rejects an observation set that waits past the window for another collection.
+Onboarding then collects that set again instead of assigning a fresh timestamp to old facts.
+Bounded evidence for a rejected set appears under `host.probe.stale` or `gateway.probe.stale` with the applied `windowMs` and the measured `ageMs`.
+`ageMs` is `null` when the recorded time cannot be parsed or is later than the current time.
 
 The policy permits only these narrow exceptions:
 

src/lib/onboard.ts

@@ -929,6 +929,7 @@ const providerExistsInGateway = (name: string, gatewayName: string = GATEWAY_NAM
 const {
   verifyInferenceRoute,
   isInferenceRouteReady,
+  readInferenceRouteState,
   checkGatewayRouteCompatibility,
   preflightGatewayRouteDiscovery,
 } = inferenceRouteHelpers.createInferenceRouteHelpers(runCaptureOpenshell);
@@ -3024,7 +3025,7 @@ async function preflightAuthoritativeRebuildTarget(
             fail(`OpenShell component preflight exited with code ${String(code)}`),
           ),
         assertGatewayReadiness: onboardPreflightGatewayAuthority.collectGatewayReadiness,
-        inferenceRouteReady: (p, m) => isInferenceRouteReady(authoritativeGateway.name, p, m),
+        inferenceRouteState: (p, m) => readInferenceRouteState(authoritativeGateway.name, p, m),
         captureForwardList: () => runCaptureOpenshell(["forward", "list"], { ignoreError: true }),
         checkPort: (port) => checkPortAvailable(port),
       },

src/lib/onboard/authoritative-rebuild-target.test.ts

@@ -9,6 +9,7 @@ import {
   rebuildProviderFlowOptions,
   resolveAuthoritativeOnboardGatewayBinding,
 } from "./authoritative-rebuild-target";
+import type { InferenceRouteState } from "./inference-route";
 import {
   mintProviderRecoveryReceipt,
   type ProviderRecoveryReceiptTarget,
@@ -30,7 +31,7 @@ function deps(overrides: Partial<AuthoritativeRebuildTargetDeps> = {}) {
     runFatalRuntimePreflight: vi.fn(),
     ensureOpenshell: vi.fn(),
     assertGatewayReadiness: vi.fn(),
-    inferenceRouteReady: vi.fn(() => true),
+    inferenceRouteState: vi.fn((): InferenceRouteState => "matched"),
     captureForwardList: vi.fn(() => "alpha 127.0.0.1 18789 42 active"),
     checkPort: vi.fn(async () => ({ ok: true })),
     ...overrides,
@@ -228,7 +229,7 @@ describe("authoritative rebuild target preflight", () => {
     expect(targetDeps.bindGatewayAuthority).not.toHaveBeenCalled();
     expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled();
     expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled();
-    expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled();
+    expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled();
   });
 
   it("pins the requested gateway for route and forward checks, then restores it", async () => {
@@ -238,9 +239,9 @@ describe("authoritative rebuild target preflight", () => {
     await preflightAuthoritativeRebuildTarget(
       target,
       deps({
-        inferenceRouteReady: vi.fn(() => {
+        inferenceRouteState: vi.fn((): InferenceRouteState => {
           seen.push(`route:${process.env.OPENSHELL_GATEWAY}`);
-          return true;
+          return "matched";
         }),
         captureForwardList: vi.fn(() => {
           seen.push(`forward:${process.env.OPENSHELL_GATEWAY}`);
@@ -259,13 +260,25 @@ describe("authoritative rebuild target preflight", () => {
     await expect(
       preflightAuthoritativeRebuildTarget(
         target,
-        deps({ inferenceRouteReady: vi.fn(() => false) }),
+        deps({ inferenceRouteState: vi.fn((): InferenceRouteState => "mismatched") }),
       ),
     ).rejects.toThrow("inference route does not match");
   });
 
+  it("proceeds when the gateway cannot answer the route query (#9310)", async () => {
+    const targetDeps = deps({
+      inferenceRouteState: vi.fn((): InferenceRouteState => "unanswered"),
+    });
+
+    await expect(preflightAuthoritativeRebuildTarget(target, targetDeps)).resolves.toBeUndefined();
+
+    expect(targetDeps.inferenceRouteState).toHaveBeenCalledOnce();
+  });
+
   it("defers route validation for prepared recovery until authoritative onboard (#6114)", async () => {
-    const targetDeps = deps({ inferenceRouteReady: vi.fn(() => false) });
+    const targetDeps = deps({
+      inferenceRouteState: vi.fn((): InferenceRouteState => "mismatched"),
+    });
 
     await expect(
       preflightAuthoritativeRebuildTarget(
@@ -274,7 +287,7 @@ describe("authoritative rebuild target preflight", () => {
       ),
     ).resolves.toBeUndefined();
 
-    expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled();
+    expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled();
     expect(targetDeps.runFatalRuntimePreflight).toHaveBeenCalledOnce();
     expect(targetDeps.ensureOpenshell).toHaveBeenCalledOnce();
   });
@@ -329,9 +342,9 @@ describe("authoritative rebuild target preflight", () => {
       }),
       ensureOpenshell: vi.fn(() => calls.push("openshell")),
       assertGatewayReadiness: vi.fn(() => calls.push("gateway")),
-      inferenceRouteReady: vi.fn(() => {
+      inferenceRouteState: vi.fn((): InferenceRouteState => {
         calls.push("route");
-        return true;
+        return "matched";
       }),
     });
 
@@ -356,6 +369,6 @@ describe("authoritative rebuild target preflight", () => {
     );
     expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled();
     expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled();
-    expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled();
+    expect(targetDeps.inferenceRouteState).not.toHaveBeenCalled();
   });
 });

src/lib/onboard/authoritative-rebuild-target.ts

@@ -3,6 +3,7 @@
 
 import { findDashboardForwardOwner } from "./dashboard-port";
 import { resolveGatewayName } from "./gateway-binding";
+import type { InferenceRouteState } from "./inference-route";
 import type { PortProbeResult } from "./preflight";
 import { assertDashboardPortNotReserved } from "./preflight-ports";
 import {
@@ -181,7 +182,7 @@ export type AuthoritativeRebuildTargetDeps = {
   runFatalRuntimePreflight(): unknown | Promise<unknown>;
   ensureOpenshell(): unknown;
   assertGatewayReadiness(): unknown | Promise<unknown>;
-  inferenceRouteReady(provider: string, model: string): boolean;
+  inferenceRouteState(provider: string, model: string): InferenceRouteState;
   captureForwardList(): string | null;
   checkPort(port: number): Promise<PortProbeResult>;
   env?: NodeJS.ProcessEnv;
@@ -209,10 +210,12 @@ export async function preflightAuthoritativeRebuildTarget(
     // Prepared-backup recovery can run after the installer has replaced a
     // legacy gateway. That fresh gateway has no inference route to validate
     // yet; authoritative onboarding configures and verifies the pinned route
-    // before recreating the sandbox. Normal rebuilds must still match here.
+    // before recreating the sandbox. A gateway that cannot answer at all leaves
+    // the route unknown, which onboarding resolves the same way. Only a gateway
+    // that answers with a different route contradicts the rebuild target.
     if (
       target.deferInferenceRouteUntilOnboard !== true &&
-      !deps.inferenceRouteReady(target.provider, target.model)
+      deps.inferenceRouteState(target.provider, target.model) === "mismatched"
     ) {
       fail(
         `OpenShell inference route does not match provider '${target.provider}' and model '${target.model}'.`,

src/lib/onboard/fatal-runtime-preflight.test.ts

@@ -553,9 +553,11 @@ describe("readiness-gated runtime preflight", () => {
       "gateway-admission",
       "host-observation",
       "gpu-observation",
+      "gateway-admission",
       "gpu-runtime-proof",
       "gateway-admission",
       "host-observation",
+      "gateway-admission",
       "gpu-validation",
       "bridge-dns",
     ]);
@@ -638,7 +640,16 @@ describe("readiness-gated runtime preflight", () => {
       },
     );
 
-    expect(calls).toEqual(["gateway", "host", "gateway", "host", "gpu", "bridge"]);
+    expect(calls).toEqual([
+      "gateway",
+      "host",
+      "gateway",
+      "host",
+      "gateway",
+      "gateway",
+      "gpu",
+      "bridge",
+    ]);
   });
 
   it("uses the already-qualified portable host facts for runtime probe effects", async () => {
@@ -664,7 +675,16 @@ describe("readiness-gated runtime preflight", () => {
       },
     );
 
-    expect(calls).toEqual(["gateway", "host", "gateway", "host", "gpu", "bridge"]);
+    expect(calls).toEqual([
+      "gateway",
+      "host",
+      "gateway",
+      "host",
+      "gateway",
+      "gateway",
+      "gpu",
+      "bridge",
+    ]);
     expect(mocks.preparePortableExperimentalHost).not.toHaveBeenCalled();
   });
 

src/lib/onboard/fatal-runtime-preflight.ts

@@ -6,7 +6,6 @@ import { detectGpu, type GpuDetection } from "../inference/nim";
 import {
   createGatewayReadinessProjection,
   type GatewayReadinessProjection,
-  refreshGatewayReadinessProjection,
 } from "../readiness/gateway";
 import {
   createProductionGatewayReadinessDependencies,
@@ -360,10 +359,11 @@ export async function runReadinessGatedRuntimePreflight(
   let gatewayReadiness = await context.collectGatewayReadiness();
   assertOnboardGatewayReadiness(gatewayReadiness, exitProcess);
   let managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness);
-  // Gateway collection can be slow. Replace the earlier host observation so
-  // the composite gate never stamps an old assessment with a fresh timestamp.
+  // Gateway collection can be slow. Replace both observations so the composite
+  // gate never stamps an old assessment with a fresh timestamp, and never
+  // condemns the run over facts it could have observed again.
   let refreshedResult = refreshOnboardHostReadiness(options, context, managedGatewayReadiness);
-  gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness);
+  gatewayReadiness = await context.collectGatewayReadiness();
   assertOnboardGatewayReadiness(gatewayReadiness, exitProcess);
   managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness);
   let readinessReport = composeSystemReadinessReport(
@@ -413,7 +413,7 @@ export async function runReadinessGatedRuntimePreflight(
         : {}),
     };
   }
-  gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness);
+  gatewayReadiness = await context.collectGatewayReadiness();
   assertOnboardGatewayReadiness(gatewayReadiness, exitProcess);
   managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness);
   readinessReport = composeSystemReadinessReport(refreshedResult.readinessReport, gatewayReadiness);

src/lib/onboard/inference-route.test.ts

@@ -50,3 +50,37 @@ describe("verifyInferenceRoute", () => {
     );
   });
 });
+
+describe("readInferenceRouteState", () => {
+  it("reports a matched route", () => {
+    const helpers = createInferenceRouteHelpers(() =>
+      gatewayRoute("compatible-endpoint", "test-model"),
+    );
+
+    expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe(
+      "matched",
+    );
+  });
+
+  it.each([
+    ["openai-api", "test-model"],
+    ["compatible-endpoint", "other-model"],
+  ])("reports a route answered as %s/%s as mismatched", (provider, model) => {
+    const helpers = createInferenceRouteHelpers(() => gatewayRoute(provider, model));
+
+    expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe(
+      "mismatched",
+    );
+  });
+
+  it("separates a gateway that cannot answer from a mismatched route (#9310)", () => {
+    const helpers = createInferenceRouteHelpers(() => null);
+
+    expect(helpers.readInferenceRouteState("nemoclaw", "compatible-endpoint", "test-model")).toBe(
+      "unanswered",
+    );
+    expect(helpers.isInferenceRouteReady("nemoclaw", "compatible-endpoint", "test-model")).toBe(
+      false,
+    );
+  });
+});

src/lib/onboard/inference-route.ts

@@ -16,6 +16,9 @@ import { listSandboxes } from "../state/registry";
 
 type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string | null;
 
+/** A gateway that cannot answer is distinct from one that answers with another route. */
+export type InferenceRouteState = "matched" | "mismatched" | "unanswered";
+
 /** Resolve the exact portable inference route used by managed clone preparation. */
 export function resolveManagedStartupInferenceRoute(
   agentName: string,
@@ -50,11 +53,20 @@ export function createInferenceRouteHelpers(
     }
   }
 
-  function isInferenceRouteReady(gatewayName: string, provider: string, model: string): boolean {
+  function readInferenceRouteState(
+    gatewayName: string,
+    provider: string,
+    model: string,
+  ): InferenceRouteState {
     const live = parseGatewayInference(
       runCaptureOpenshell(["inference", "get", "-g", gatewayName], { ignoreError: true }),
     );
-    return Boolean(live && live.provider === provider && live.model === model);
+    if (!live) return "unanswered";
+    return live.provider === provider && live.model === model ? "matched" : "mismatched";
+  }
+
+  function isInferenceRouteReady(gatewayName: string, provider: string, model: string): boolean {
+    return readInferenceRouteState(gatewayName, provider, model) === "matched";
   }
 
   const checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck = (request) =>
@@ -72,6 +84,7 @@ export function createInferenceRouteHelpers(
   return {
     verifyInferenceRoute,
     isInferenceRouteReady,
+    readInferenceRouteState,
     checkGatewayRouteCompatibility,
     preflightGatewayRouteDiscovery,
   };