fix(onboard): verify the agent API host forward before reporting ready

Deployment verification probed the agent gateway only from inside the
sandbox and probed just the dashboard port on the host. For Hermes, whose
manifest forwards a second host port for the OpenAI-compatible API, a
failed API forward therefore left every checked link green: onboarding
printed "Deployment verified" and "Hermes is ready", and advertised an
API URL that refused every connection.

Probe the agent API port from the host too, whenever the agent declares
one distinct from the dashboard port, and fold it into the deployment
health result so a dead forward pauses onboarding with an actionable
diagnostic instead of a false success. Agents without a separate API
port keep their single dashboard probe unchanged.

Resolve that port per sandbox rather than from the manifest default, so
a second Hermes sandbox holding a reallocated port from the 8642-8652
range is not reported as unreachable.

Fixes #9290

Signed-off-by: Yanyun Liao <yanyunl@nvidia.com>

Author: yanyunl1991

docs/reference/troubleshooting.mdx

@@ -4893,6 +4893,26 @@ Expected output:
 Point an OpenAI-compatible client at `http://127.0.0.1:8642/v1` for chat completions.
 For terminal use, run `nemohermes launch <name>`.
 
+### Onboarding reports "Hermes is not ready" with an unreachable API port
+
+Deployment verification probes the OpenAI-compatible API on its host forward, not only inside the sandbox.
+When the sandbox gateway is healthy but the host forward never came up, verification reports the API link as failed and onboarding finishes with `Hermes is not ready` instead of a successful handoff:
+
+```text
+✗ api: port forward not working (connection refused)
+  The OpenAI-compatible API on port 8642 is not reachable from the host. Run: openshell forward start --background 8642 <name>
+```
+
+The sandbox itself is intact, so start the forward and re-check:
+
+```bash
+openshell forward list                        # confirm no row owns the API port
+openshell forward start --background 8642 <name>
+curl -sf http://127.0.0.1:8642/health
+```
+
+If the forward still refuses connections, a foreign host listener usually holds the port; stop it, then rerun `$$nemoclaw onboard` to complete verification.
+
 ### `docker port` shows no mapping for 8642 even though forwarding works
 
 OpenShell port forwards are host-side relays managed by the OpenShell gateway process, not Docker `-p` publish mappings on the sandbox container.

src/lib/onboard.ts

@@ -2906,6 +2906,7 @@ const setupOpenclaw = createOpenclawSetup({
 
 const {
   buildChain,
+  resolveVerifyAgentApiPort,
   buildControlUiUrls,
   buildOrphanedSandboxRollbackMessage,
   ensureDashboardForward,
@@ -3644,8 +3645,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
         removeLegacyCredentialsFile,
         cleanupStaleHostFiles,
         getChatUiUrl: () => process.env.CHAT_UI_URL || `http://127.0.0.1:${DASHBOARD_PORT}`,
-        buildVerifyChain: (chatUiUrl) =>
-          buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: getWslHostAddress(), dashboardHealthEndpoint: agent?.dashboard.healthPath, gatewayPort: agent?.healthProbe?.port, gatewayHealthEndpoint: agent?.healthProbe?.url }),
+        buildVerifyChain: (chatUiUrl, verifiedSandboxName) =>
+          buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: getWslHostAddress(), dashboardHealthEndpoint: agent?.dashboard.healthPath, gatewayPort: resolveVerifyAgentApiPort(verifiedSandboxName, agent), gatewayHealthEndpoint: agent?.healthProbe?.url }),
         verifyDeployment: async (name, chain) => {
           const verifyDeploymentModule: typeof import("./verify-deployment") =
             require("./verify-deployment");

src/lib/onboard/agent-dashboard-forward.ts

@@ -11,6 +11,11 @@ import {
 } from "./dashboard-runtime";
 import { resolveOnboardHermesApiPort } from "./hermes-api-port";
 
+// The port deployment verification must probe lives with the rest of this
+// module's "which host port does this agent publish" logic, so onboarding
+// reaches it through the dashboard helpers it already consumes (#9290).
+export { resolveVerifyAgentApiPort } from "./hermes-api-port";
+
 export type EnsureDashboardForward = (
   sandboxName: string,
   chatUiUrl?: string,

src/lib/onboard/dashboard.ts

@@ -14,6 +14,7 @@ import { runCapture as defaultRunCapture } from "../runner";
 import {
   ensureAgentDashboardForward as ensureAgentDashboardForwardForAgent,
   replaceUrlPort,
+  resolveVerifyAgentApiPort,
 } from "./agent-dashboard-forward";
 import { ensureAgentFixedForward as ensureFixedAgentForward } from "./agent-fixed-forward";
 import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token";
@@ -88,6 +89,7 @@ export interface OnboardDashboardDeps {
 
 export interface OnboardDashboardHelpers {
   buildChain: typeof buildChain;
+  resolveVerifyAgentApiPort: typeof resolveVerifyAgentApiPort;
   buildControlUiUrls: typeof buildControlUiUrls;
   buildOrphanedSandboxRollbackMessage(
     sandboxName: string,
@@ -630,6 +632,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa
 
   return {
     buildChain,
+    resolveVerifyAgentApiPort,
     buildControlUiUrls,
     buildOrphanedSandboxRollbackMessage,
     ensureDashboardForward,

src/lib/onboard/hermes-api-port.test.ts

@@ -13,6 +13,7 @@ import {
   reserveCreateSandboxHermesApiPort,
   resolveOnboardHermesApiPort,
   resolveSandboxHermesApiPort,
+  resolveVerifyAgentApiPort,
   retargetHermesApiPortInUrl,
   withHermesApiPortReservationScope,
 } from "./hermes-api-port";
@@ -323,3 +324,34 @@ describe("retargetHermesApiPortInUrl", () => {
     );
   });
 });
+
+describe("resolveVerifyAgentApiPort (#9290)", () => {
+  const hermes = { name: "hermes", healthProbe: { port: 8642 } };
+
+  it("targets the port this Hermes sandbox actually owns", () => {
+    // A second Hermes sandbox serves its API on a reallocated port; probing the
+    // manifest default would report a sibling sandbox's port as unreachable.
+    expect(
+      resolveVerifyAgentApiPort("second", hermes, { getSandbox: () => ({ hermesApiPort: 8643 }) }),
+    ).toBe(8643);
+  });
+
+  it("falls back to the manifest default when the sandbox is not registered yet", () => {
+    expect(resolveVerifyAgentApiPort("fresh", hermes, { getSandbox: () => null })).toBe(8642);
+  });
+
+  it("keeps a non-Hermes agent's declared probe port", () => {
+    expect(
+      resolveVerifyAgentApiPort("sb", { name: "other", healthProbe: { port: 9000 } }, {
+        getSandbox: () => ({ hermesApiPort: 8643 }),
+      }),
+    ).toBe(9000);
+  });
+
+  it("returns undefined when the agent declares no health probe port", () => {
+    expect(resolveVerifyAgentApiPort("sb", { name: "openclaw" }, { getSandbox: () => null })).toBe(
+      undefined,
+    );
+    expect(resolveVerifyAgentApiPort("sb", null, { getSandbox: () => null })).toBe(undefined);
+  });
+});

src/lib/onboard/hermes-api-port.ts

@@ -388,3 +388,27 @@ export function resolveOnboardHermesApiPort(
   }
   return publish(port);
 }
+
+/**
+ * Resolve the API port deployment verification must probe for `agent`.
+ *
+ * Returns the agent's declared health-probe port, except for Hermes, whose
+ * per-sandbox allocation from the 8642-8652 range means the manifest default
+ * would name a sibling sandbox's port. Returns undefined when the agent
+ * declares no health-probe port, which leaves `buildChain` on its dashboard-port
+ * fallback so agents without a separate API surface keep their existing single
+ * host probe (#9290).
+ */
+export function resolveVerifyAgentApiPort(
+  sandboxName: string,
+  agent: { name?: string; healthProbe?: { port?: number } | null } | null | undefined,
+  options: {
+    getSandbox?: (name: string) => { hermesApiPort?: number | null } | null | undefined;
+  } = {},
+): number | undefined {
+  const declared = agent?.healthProbe?.port;
+  if (!Number.isInteger(declared)) return undefined;
+  if (agent?.name !== "hermes" || declared !== HERMES_OPENAI_API_PORT) return declared;
+  const getSandbox = options.getSandbox ?? registry.getSandbox;
+  return resolveSandboxHermesApiPort(getSandbox(sandboxName) ?? {});
+}

src/lib/onboard/machine/final-flow-phases.runtime.test.ts

@@ -21,6 +21,7 @@ function deploymentResult(healthy: boolean): VerifyDeploymentResult {
       gatewayVersion: "test",
       inferenceRouteWorking: healthy,
       dashboardReachable: true,
+      agentApiReachable: null,
       messagingBridgesHealthy: true,
       messagingRuntimeChannelsMissing: null,
       messagingConfigChannelsMissing: null,
@@ -269,6 +270,7 @@ describe("final onboard flow runtime boundary", () => {
             gatewayVersion: "test",
             inferenceRouteWorking: true,
             dashboardReachable: true,
+            agentApiReachable: null,
             messagingBridgesHealthy: true,
             messagingRuntimeChannelsMissing: null,
             messagingConfigChannelsMissing: null,

src/lib/onboard/machine/handlers/finalization.test.ts

@@ -133,7 +133,9 @@ describe("finalization handlers", () => {
     );
     expect(calls.cleanupHost).toHaveBeenCalledOnce();
     expect(calls.recoverProcesses).toHaveBeenCalledWith("my-assistant", { quiet: true });
-    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789");
+    // The sandbox name lets the chain resolve this sandbox's own agent API
+    // port rather than the agent manifest default (#9290).
+    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789", "my-assistant");
     expect(calls.verify).toHaveBeenCalledWith("my-assistant", { port: 18789 });
     expect(calls.log).toHaveBeenCalledWith("  ✓ verified");
     expect(calls.dashboard).toHaveBeenCalledWith(

src/lib/onboard/machine/handlers/finalization.ts

@@ -61,7 +61,12 @@ export interface FinalizationStateOptions<Agent, VerifyChain, VerificationResult
      */
     warmupScopeUpgrade(sandboxName: string): void;
     getChatUiUrl(): string;
-    buildVerifyChain(chatUiUrl: string): VerifyChain;
+    /**
+     * `sandboxName` lets the chain target the API port this sandbox actually
+     * owns: Hermes allocates a per-sandbox port from the 8642-8652 range, so
+     * the manifest default would probe a sibling sandbox's port (#9290).
+     */
+    buildVerifyChain(chatUiUrl: string, sandboxName: string): VerifyChain;
     verifyDeployment(sandboxName: string, chain: VerifyChain): Promise<VerificationResult>;
     formatVerificationDiagnostics(result: VerificationResult): string[];
     isDeploymentHealthy(result: VerificationResult): boolean;
@@ -231,7 +236,7 @@ export async function handlePostVerifyState<Agent, VerifyChain, VerificationResu
       (webSearchProvider !== null &&
         deps.verifyWebSearchInsideSandbox(sandboxName, agent, webSearchProvider));
     // Confirm the delivered sandbox is reachable before printing the live dashboard (#2342).
-    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl());
+    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl(), sandboxName);
     const verificationResult = await deps.verifyDeployment(sandboxName, verifyChain);
     deploymentHealthy =
       webSearchCredentialBoundarySafe && deps.isDeploymentHealthy(verificationResult);

src/lib/verify-deployment-agent.test.ts

@@ -54,3 +54,88 @@ describe("verifyDeployment agent dashboard probes", () => {
     expect(hostProbes).toContainEqual({ port: 18789, path: "/api/status" });
   });
 });
+
+describe("verifyDeployment agent OpenAI-compatible API host forward (#9290)", () => {
+  const agentChain = buildChain({
+    chatUiUrl: "http://127.0.0.1:18789",
+    dashboardHealthEndpoint: "/api/status",
+    gatewayPort: 8642,
+    gatewayHealthEndpoint: "/health",
+  });
+
+  function runWithApiHostCode(apiCode: number) {
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return port === 8642 ? apiCode : 200;
+      },
+    });
+    return { hostProbes, run: () => verifyDeployment("my-sandbox", agentChain, deps, NO_RETRY) };
+  }
+
+  it("probes the API port on the host, not just inside the sandbox", async () => {
+    const { hostProbes, run } = runWithApiHostCode(200);
+
+    const result = await run();
+
+    // The in-sandbox gateway probe only proves the API answers inside the
+    // sandbox; the host forward is what operators actually connect through.
+    expect(hostProbes).toContainEqual({ port: 8642, path: "/health" });
+    expect(result.healthy).toBe(true);
+    expect(result.verification.agentApiReachable).toBe(true);
+  });
+
+  it("fails verification when the API host forward refuses connections", async () => {
+    // The reported regression: the sandbox gateway is healthy and the dashboard
+    // forward is up, but port 8642 never became reachable on the host.
+    const { run } = runWithApiHostCode(0);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.verification.agentApiReachable).toBe(false);
+    expect(result.verification.gatewayReachable).toBe(true);
+    expect(result.verification.dashboardReachable).toBe(true);
+    const api = result.diagnostics.find((d) => d.link === "api");
+    expect(api?.status).toBe("fail");
+    expect(api?.hint).toContain("openshell forward start --background 8642 my-sandbox");
+  });
+
+  it("fails verification when the API host forward answers with a server error", async () => {
+    const { run } = runWithApiHostCode(502);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.diagnostics.find((d) => d.link === "api")?.detail).toContain("502");
+  });
+
+  it("accepts an authenticated API host forward (HTTP 401)", async () => {
+    // Bearer auth is enabled on the Hermes API; 401 still proves the forward
+    // reaches a live listener.
+    const { run } = runWithApiHostCode(401);
+
+    expect((await run()).healthy).toBe(true);
+  });
+
+  it("keeps a single host probe for agents without a separate API port", async () => {
+    // Regression lock: OpenClaw declares no gateway port, so buildChain falls
+    // back to the dashboard port and no API probe or diagnostic is added.
+    const openClawChain = buildChain({ chatUiUrl: "http://127.0.0.1:18789" });
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return 200;
+      },
+    });
+
+    const result = await verifyDeployment("my-sandbox", openClawChain, deps, NO_RETRY);
+
+    expect(hostProbes).toEqual([{ port: 18789, path: "/health" }]);
+    expect(result.diagnostics.some((d) => d.link === "api")).toBe(false);
+    expect(result.verification.agentApiReachable).toBeNull();
+    expect(result.healthy).toBe(true);
+  });
+});