fix(onboard): verify the agent API host forward before reporting ready

docs/reference/troubleshooting.mdx

@@ -4893,6 +4893,52 @@ Expected output:
 Point an OpenAI-compatible client at `http://127.0.0.1:8642/v1` for chat completions.
 For terminal use, run `nemohermes launch <name>`.
 
+### Onboarding Reports Hermes Is Not Ready With an Unreachable API Port
+
+Deployment verification probes the OpenAI-compatible API inside the sandbox and through its host-side API port forward.
+When the API answers inside the sandbox but its host-side API port forward is unreachable, verification reports the API port forward as failed.
+Onboarding then prints `Hermes is not ready` and exits with a nonzero status:
+
+```text
+✗ api: port forward not working (connection refused)
+  The OpenAI-compatible API on port 8642 is not reachable from the host. Run: openshell forward start --background 8642 <name>
+```
+
+Use the forward recovery below only when the in-sandbox `gateway` check passed.
+The output omits passing checks, so confirm that it contains no `gateway` failure.
+If the output contains a `gateway` failure, follow that diagnostic first.
+Do not restart the host forward until the in-sandbox API responds.
+
+When the `gateway` check passed and only the `api` check failed, the API remains reachable inside the sandbox.
+Each Hermes sandbox owns an API port allocated from `8642` through `8652`, so use the port from the `api` diagnostic instead of assuming the default.
+First, list the active OpenShell port forwards:
+
+```bash
+openshell forward list
+```
+
+If no row owns the API port, start its port forward:
+
+```bash
+openshell forward start --background <port> <name>
+```
+
+Continue only when the command exits with status `0`.
+If the command reports that the port is in use, follow [Port Already in Use](#port-already-in-use) immediately.
+Replace that section's example port `18789` with the API port from the `api` diagnostic.
+Apply its process-ownership, service-manager, and active-work conditions before you stop a listener.
+Do not run the health probe after `openshell forward start` fails.
+
+After `openshell forward start` exits with status `0`, probe the health endpoint:
+
+```bash
+curl -sS -o /dev/null -w '%{http_code}\n' --max-time 3 http://127.0.0.1:<port>/health
+```
+
+An HTTP status of `200` or `401` means the host port forward is reachable.
+Any other status, `000`, or connection failure requires fresh diagnostics.
+Rerun `$$nemoclaw onboard` and follow the reported `gateway` or `api` failure.
+
 ### `docker port` shows no mapping for 8642 even though forwarding works
 
 OpenShell port forwards are host-side relays managed by the OpenShell gateway process, not Docker `-p` publish mappings on the sandbox container.

src/lib/onboard.ts

@@ -2906,6 +2906,7 @@ const setupOpenclaw = createOpenclawSetup({
 
 const {
   buildChain,
+  buildAgentVerifyChain,
   buildControlUiUrls,
   buildOrphanedSandboxRollbackMessage,
   ensureDashboardForward,
@@ -3641,8 +3642,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise<void> {
         removeLegacyCredentialsFile,
         cleanupStaleHostFiles,
         getChatUiUrl: () => process.env.CHAT_UI_URL || `http://127.0.0.1:${DASHBOARD_PORT}`,
-        buildVerifyChain: (chatUiUrl) =>
-          buildChain({ chatUiUrl, isWsl: isWsl(), wslHostAddress: getWslHostAddress(), dashboardHealthEndpoint: agent?.dashboard.healthPath, gatewayPort: agent?.healthProbe?.port, gatewayHealthEndpoint: agent?.healthProbe?.url }),
+        buildVerifyChain: (chatUiUrl, name) => buildAgentVerifyChain(chatUiUrl, name, agent),
         verifyDeployment: async (name, chain) => {
           const verifyDeploymentModule: typeof import("./verify-deployment") =
             require("./verify-deployment");

src/lib/onboard/agent-dashboard-forward.ts

@@ -11,6 +11,11 @@ import {
 } from "./dashboard-runtime";
 import { resolveOnboardHermesApiPort } from "./hermes-api-port";
 
+// The port deployment verification must probe lives with the rest of this
+// module's "which host port does this agent publish" logic, so onboarding
+// reaches it through the dashboard helpers it already consumes (#9290).
+export { resolveVerifyAgentApiPort } from "./hermes-api-port";
+
 export type EnsureDashboardForward = (
   sandboxName: string,
   chatUiUrl?: string,

src/lib/onboard/dashboard.ts

@@ -14,6 +14,7 @@ import { runCapture as defaultRunCapture } from "../runner";
 import {
   ensureAgentDashboardForward as ensureAgentDashboardForwardForAgent,
   replaceUrlPort,
+  resolveVerifyAgentApiPort,
 } from "./agent-dashboard-forward";
 import { ensureAgentFixedForward as ensureFixedAgentForward } from "./agent-fixed-forward";
 import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token";
@@ -74,6 +75,8 @@ export interface OnboardDashboardDeps {
   listSandboxes?: ListSandboxesFn;
   /** Host-listener probe injected by forward release race tests. */
   isPortBoundOnHost?: typeof isPortBoundOnHost;
+  /** Sandbox lookup used to resolve the per-sandbox Hermes API port. */
+  getSandbox?(name: string): { hermesApiPort?: number | null } | null | undefined;
   printAgentDashboardUi(
     sandboxName: string,
     token: string | null,
@@ -86,8 +89,20 @@ export interface OnboardDashboardDeps {
   ): void;
 }
 
+/** Agent fields the deployment-verification chain reads. */
+export type VerifyChainAgent = {
+  name?: string;
+  dashboard?: { healthPath?: string } | null;
+  healthProbe?: { url?: string; port?: number } | null;
+};
+
 export interface OnboardDashboardHelpers {
   buildChain: typeof buildChain;
+  buildAgentVerifyChain(
+    chatUiUrl: string,
+    sandboxName: string,
+    agent: VerifyChainAgent | null | undefined,
+  ): ReturnType<typeof buildChain>;
   buildControlUiUrls: typeof buildControlUiUrls;
   buildOrphanedSandboxRollbackMessage(
     sandboxName: string,
@@ -224,6 +239,33 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa
     });
   }
 
+  /**
+   * Build the delivery chain deployment verification probes for `sandboxName`.
+   *
+   * Resolves the agent's OpenAI-compatible API port for this sandbox rather
+   * than the agent manifest default, so verification probes the port this
+   * sandbox actually publishes on the host (#9290).
+   */
+  function buildAgentVerifyChain(
+    chatUiUrl: string,
+    sandboxName: string,
+    agent: VerifyChainAgent | null | undefined,
+  ): ReturnType<typeof buildChain> {
+    // Resolve WSL once: `buildChain` and the host-address lookup must agree, or
+    // the chain can claim WSL while dropping the fallback URL that pairs with it.
+    const isWsl = deps.isWsl();
+    return buildChain({
+      chatUiUrl,
+      isWsl,
+      wslHostAddress: getWslHostAddress({ isWsl }),
+      dashboardHealthEndpoint: agent?.dashboard?.healthPath,
+      gatewayPort: resolveVerifyAgentApiPort(sandboxName, agent, {
+        getSandbox: deps.getSandbox,
+      }),
+      gatewayHealthEndpoint: agent?.healthProbe?.url,
+    });
+  }
+
   function stopAllDashboardForwards(): void {
     const forwardList = deps.runCaptureOpenshell(["forward", "list"], { ignoreError: true });
     for (const port of getRunningForwardPorts(forwardList)) {
@@ -630,6 +672,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa
 
   return {
     buildChain,
+    buildAgentVerifyChain,
     buildControlUiUrls,
     buildOrphanedSandboxRollbackMessage,
     ensureDashboardForward,

src/lib/onboard/hermes-api-port.test.ts

@@ -13,6 +13,7 @@ import {
   reserveCreateSandboxHermesApiPort,
   resolveOnboardHermesApiPort,
   resolveSandboxHermesApiPort,
+  resolveVerifyAgentApiPort,
   retargetHermesApiPortInUrl,
   withHermesApiPortReservationScope,
 } from "./hermes-api-port";
@@ -323,3 +324,34 @@ describe("retargetHermesApiPortInUrl", () => {
     );
   });
 });
+
+describe("resolveVerifyAgentApiPort (#9290)", () => {
+  const hermes = { name: "hermes", healthProbe: { port: 8642 } };
+
+  it("targets the port this Hermes sandbox actually owns", () => {
+    // A second Hermes sandbox serves its API on a reallocated port; probing the
+    // manifest default would report a sibling sandbox's port as unreachable.
+    expect(
+      resolveVerifyAgentApiPort("second", hermes, { getSandbox: () => ({ hermesApiPort: 8643 }) }),
+    ).toBe(8643);
+  });
+
+  it("falls back to the manifest default when the sandbox is not registered yet", () => {
+    expect(resolveVerifyAgentApiPort("fresh", hermes, { getSandbox: () => null })).toBe(8642);
+  });
+
+  it("keeps a non-Hermes agent's declared probe port", () => {
+    expect(
+      resolveVerifyAgentApiPort("sb", { name: "other", healthProbe: { port: 9000 } }, {
+        getSandbox: () => ({ hermesApiPort: 8643 }),
+      }),
+    ).toBe(9000);
+  });
+
+  it("returns undefined when the agent declares no health probe port", () => {
+    expect(resolveVerifyAgentApiPort("sb", { name: "openclaw" }, { getSandbox: () => null })).toBe(
+      undefined,
+    );
+    expect(resolveVerifyAgentApiPort("sb", null, { getSandbox: () => null })).toBe(undefined);
+  });
+});

src/lib/onboard/hermes-api-port.ts

@@ -388,3 +388,27 @@ export function resolveOnboardHermesApiPort(
   }
   return publish(port);
 }
+
+/**
+ * Resolve the API port deployment verification must probe for `agent`.
+ *
+ * Returns the agent's declared health-probe port, except for Hermes, whose
+ * per-sandbox allocation from the 8642-8652 range means the manifest default
+ * would name a sibling sandbox's port. Returns undefined when the agent
+ * declares no health-probe port, which leaves `buildChain` on its dashboard-port
+ * fallback so agents without a separate API surface keep their existing single
+ * host probe (#9290).
+ */
+export function resolveVerifyAgentApiPort(
+  sandboxName: string,
+  agent: { name?: string; healthProbe?: { port?: number } | null } | null | undefined,
+  options: {
+    getSandbox?: (name: string) => { hermesApiPort?: number | null } | null | undefined;
+  } = {},
+): number | undefined {
+  const declared = agent?.healthProbe?.port;
+  if (!Number.isInteger(declared)) return undefined;
+  if (agent?.name !== "hermes" || declared !== HERMES_OPENAI_API_PORT) return declared;
+  const getSandbox = options.getSandbox ?? registry.getSandbox;
+  return resolveSandboxHermesApiPort(getSandbox(sandboxName) ?? {});
+}

src/lib/onboard/machine/final-flow-phases.runtime.test.ts

@@ -21,6 +21,7 @@ function deploymentResult(healthy: boolean): VerifyDeploymentResult {
       gatewayVersion: "test",
       inferenceRouteWorking: healthy,
       dashboardReachable: true,
+      agentApiReachable: null,
       messagingBridgesHealthy: true,
       messagingRuntimeChannelsMissing: null,
       messagingConfigChannelsMissing: null,
@@ -269,6 +270,7 @@ describe("final onboard flow runtime boundary", () => {
             gatewayVersion: "test",
             inferenceRouteWorking: true,
             dashboardReachable: true,
+            agentApiReachable: null,
             messagingBridgesHealthy: true,
             messagingRuntimeChannelsMissing: null,
             messagingConfigChannelsMissing: null,

src/lib/onboard/machine/handlers/finalization.test.ts

@@ -133,7 +133,9 @@ describe("finalization handlers", () => {
     );
     expect(calls.cleanupHost).toHaveBeenCalledOnce();
     expect(calls.recoverProcesses).toHaveBeenCalledWith("my-assistant", { quiet: true });
-    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789");
+    // The sandbox name lets the chain resolve this sandbox's own agent API
+    // port rather than the agent manifest default (#9290).
+    expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789", "my-assistant");
     expect(calls.verify).toHaveBeenCalledWith("my-assistant", { port: 18789 });
     expect(calls.log).toHaveBeenCalledWith("  ✓ verified");
     expect(calls.dashboard).toHaveBeenCalledWith(

src/lib/onboard/machine/handlers/finalization.ts

@@ -61,7 +61,12 @@ export interface FinalizationStateOptions<Agent, VerifyChain, VerificationResult
      */
     warmupScopeUpgrade(sandboxName: string): void;
     getChatUiUrl(): string;
-    buildVerifyChain(chatUiUrl: string): VerifyChain;
+    /**
+     * `sandboxName` lets the chain target the API port this sandbox actually
+     * owns: Hermes allocates a per-sandbox port from the 8642-8652 range, so
+     * the manifest default would probe a sibling sandbox's port (#9290).
+     */
+    buildVerifyChain(chatUiUrl: string, sandboxName: string): VerifyChain;
     verifyDeployment(sandboxName: string, chain: VerifyChain): Promise<VerificationResult>;
     formatVerificationDiagnostics(result: VerificationResult): string[];
     isDeploymentHealthy(result: VerificationResult): boolean;
@@ -231,7 +236,7 @@ export async function handlePostVerifyState<Agent, VerifyChain, VerificationResu
       (webSearchProvider !== null &&
         deps.verifyWebSearchInsideSandbox(sandboxName, agent, webSearchProvider));
     // Confirm the delivered sandbox is reachable before printing the live dashboard (#2342).
-    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl());
+    const verifyChain = deps.buildVerifyChain(deps.getChatUiUrl(), sandboxName);
     const verificationResult = await deps.verifyDeployment(sandboxName, verifyChain);
     deploymentHealthy =
       webSearchCredentialBoundarySafe && deps.isDeploymentHealthy(verificationResult);

src/lib/onboard/machine/handlers/sandbox-messaging.test.ts

@@ -390,13 +390,13 @@ describe("reconcileReusedSandboxMessaging", () => {
 
   it("omits a retired host-backed channel from a reused sandbox selection (#9283)", () => {
     const plan = discordPlan(hashCredential("previous-discord-token") ?? "");
-    const clearPlanEnv = vi.fn();
+    const deps = { clearPlanEnv: vi.fn(), note: vi.fn(), writePlanToEnv: vi.fn() };
     vi.stubEnv("DISCORD_BOT_TOKEN", "");
 
     const result = reconcileReusedSandboxMessaging(
       structuredClone(plan),
       { name: "openclaw" },
-      { clearPlanEnv, note: vi.fn(), writePlanToEnv: vi.fn() },
+      deps,
       plan,
     );
 
@@ -407,7 +407,7 @@ describe("reconcileReusedSandboxMessaging", () => {
       selectedChannels: [],
       changed: true,
     });
-    expect(clearPlanEnv).not.toHaveBeenCalled();
+    expect(deps.clearPlanEnv).not.toHaveBeenCalled();
   });
 
   it("keeps a still-configured channel in a reused sandbox selection (#9283)", () => {

src/lib/verify-deployment-agent.test.ts

@@ -54,3 +54,88 @@ describe("verifyDeployment agent dashboard probes", () => {
     expect(hostProbes).toContainEqual({ port: 18789, path: "/api/status" });
   });
 });
+
+describe("verifyDeployment agent OpenAI-compatible API host forward (#9290)", () => {
+  const agentChain = buildChain({
+    chatUiUrl: "http://127.0.0.1:18789",
+    dashboardHealthEndpoint: "/api/status",
+    gatewayPort: 8642,
+    gatewayHealthEndpoint: "/health",
+  });
+
+  function runWithApiHostCode(apiCode: number) {
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return port === 8642 ? apiCode : 200;
+      },
+    });
+    return { hostProbes, run: () => verifyDeployment("my-sandbox", agentChain, deps, NO_RETRY) };
+  }
+
+  it("probes the API port on the host, not just inside the sandbox", async () => {
+    const { hostProbes, run } = runWithApiHostCode(200);
+
+    const result = await run();
+
+    // The in-sandbox gateway probe only proves the API answers inside the
+    // sandbox; the host forward is what operators actually connect through.
+    expect(hostProbes).toContainEqual({ port: 8642, path: "/health" });
+    expect(result.healthy).toBe(true);
+    expect(result.verification.agentApiReachable).toBe(true);
+  });
+
+  it("fails verification when the API host forward refuses connections", async () => {
+    // The reported regression: the sandbox gateway is healthy and the dashboard
+    // forward is up, but port 8642 never became reachable on the host.
+    const { run } = runWithApiHostCode(0);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.verification.agentApiReachable).toBe(false);
+    expect(result.verification.gatewayReachable).toBe(true);
+    expect(result.verification.dashboardReachable).toBe(true);
+    const api = result.diagnostics.find((d) => d.link === "api");
+    expect(api?.status).toBe("fail");
+    expect(api?.hint).toContain("openshell forward start --background 8642 my-sandbox");
+  });
+
+  it("fails verification when the API host forward answers with a server error", async () => {
+    const { run } = runWithApiHostCode(502);
+
+    const result = await run();
+
+    expect(result.healthy).toBe(false);
+    expect(result.diagnostics.find((d) => d.link === "api")?.detail).toContain("502");
+  });
+
+  it("accepts an authenticated API host forward (HTTP 401)", async () => {
+    // Bearer auth is enabled on the Hermes API; 401 still proves the forward
+    // reaches a live listener.
+    const { run } = runWithApiHostCode(401);
+
+    expect((await run()).healthy).toBe(true);
+  });
+
+  it("keeps a single host probe for agents without a separate API port", async () => {
+    // Regression lock: OpenClaw declares no gateway port, so buildChain falls
+    // back to the dashboard port and no API probe or diagnostic is added.
+    const openClawChain = buildChain({ chatUiUrl: "http://127.0.0.1:18789" });
+    const hostProbes: Array<{ port: number; path: string }> = [];
+    const deps = makeDeps({
+      probeHostPort: (port: number, path: string) => {
+        hostProbes.push({ port, path });
+        return 200;
+      },
+    });
+
+    const result = await verifyDeployment("my-sandbox", openClawChain, deps, NO_RETRY);
+
+    expect(hostProbes).toEqual([{ port: 18789, path: "/health" }]);
+    expect(result.diagnostics.some((d) => d.link === "api")).toBe(false);
+    expect(result.verification.agentApiReachable).toBeNull();
+    expect(result.healthy).toBe(true);
+  });
+});