fix(onboard): separate portable host gateway subnet (#9641)

Author: senthilr-nv

.github/workflows/podman-cpu-proof.yaml

@@ -28,6 +28,7 @@ on:
       - "src/lib/onboard/runtime-provider/container-state-mutation.ts"
       - "src/lib/onboard/runtime-provider/docker-state-mutation.ts"
       - "src/lib/onboard/experimental/portable-host-preparation*.ts"
+      - "src/lib/onboard/experimental/portable-profile.ts"
       - "src/lib/onboard/runtime-provider/podman*.ts"
       - "scripts/install-openshell.sh"
       - "scripts/checks/run-portable-cpu-delegation-proof.mts"
@@ -334,6 +335,21 @@ jobs:
             printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_dir"
           } >>"$GITHUB_ENV"
 
+      - name: Configure exact Portable host gateway alias
+        shell: bash
+        run: |
+          set -euo pipefail
+          portable_host_gateway_ip="$(
+            node --input-type=module --eval '
+              const { PORTABLE_HOST_GATEWAY_IP } =
+                await import("./dist/lib/onboard/docker-driver-platform.js");
+              process.stdout.write(PORTABLE_HOST_GATEWAY_IP);
+            '
+          )"
+          sudo ip address replace "$portable_host_gateway_ip/32" dev lo
+          ip -o -4 address show dev lo | awk '{print $4}' | grep -Fx "$portable_host_gateway_ip/32"
+          printf 'E2E_PORTABLE_HOST_GATEWAY_IP=%s\n' "$portable_host_gateway_ip" >>"$GITHUB_ENV"
+
       - name: Start the exact connected gateway required by portable retirement
         shell: bash
         run: |
@@ -458,6 +474,17 @@ jobs:
             esac
           done < <(podman --url "$endpoint" secret ls --format '{{.Name}}' 2>/dev/null || true)
           podman --url "$endpoint" network rm openshell-docker 2>/dev/null || true
+          portable_host_gateway_ip=""
+          if portable_host_gateway_ip="$(
+            node --input-type=module --eval '
+              const { PORTABLE_HOST_GATEWAY_IP } =
+                await import("./dist/lib/onboard/docker-driver-platform.js");
+              process.stdout.write(PORTABLE_HOST_GATEWAY_IP);
+            '
+          )" && [ -n "$portable_host_gateway_ip" ] \
+            && [ "${E2E_PORTABLE_HOST_GATEWAY_IP:-}" = "$portable_host_gateway_ip" ]; then
+            sudo ip address delete "$portable_host_gateway_ip/32" dev lo 2>/dev/null || true
+          fi
           service_pid="${E2E_PODMAN_SERVICE_PID:-}"
           if [[ "$service_pid" =~ ^[1-9][0-9]*$ ]]; then
             kill "$service_pid" 2>/dev/null || true

docs/inference/set-up-openai-compatible-endpoint.mdx

@@ -132,10 +132,11 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_AGENT=langchain-deepage
 
 </AgentOnly>
 
-The portable profile creates the `openshell-docker` network on `169.254.1.0/24` and assigns `169.254.1.2/32` to the host loopback interface.
-On that network, `169.254.1.2` is the host-gateway address for `host.openshell.internal` and carries authenticated mTLS plus sandbox-JWT callbacks to OpenShell on port `8080`.
+The portable profile creates the `openshell-docker` network on `169.254.1.0/24` and assigns `169.254.2.2/32` to the host loopback interface.
+The address `169.254.2.2` is outside the Portable sandbox subnet and is the host-gateway address for `host.openshell.internal`.
+It carries authenticated mTLS plus sandbox-JWT callbacks to OpenShell on port `8080`.
 The managed local registry uses the distinct address `169.254.1.3` on port `5000`.
-Keeping these addresses distinct prevents registry traffic from intercepting gateway callbacks.
+Keeping the host-gateway address outside the sandbox subnet prevents a Portable workload from receiving that address.
 
 The portable profile handles the descriptor as follows:
 

src/lib/onboard/docker-driver-gateway-env.test.ts

@@ -15,6 +15,7 @@ import {
   startPackageManagedDockerDriverGatewayWithEnvOverride,
   writeDockerGatewayDebEnvOverride,
 } from "./docker-driver-gateway-env";
+import { PORTABLE_HOST_GATEWAY_IP } from "./experimental/portable-profile";
 
 function homeEnv(home: string, xdgConfigHome = ""): NodeJS.ProcessEnv {
   return { HOME: home, XDG_CONFIG_HOME: xdgConfigHome } as NodeJS.ProcessEnv;
@@ -156,14 +157,14 @@ describe("buildDockerDriverGatewayEnv", () => {
         OPENSHELL_DRIVERS: "podman",
         CONTAINERS_CONF: "/tmp/nemoclaw-portable/containers.conf",
         OPENSHELL_BIND_ADDRESS: "0.0.0.0",
-        OPENSHELL_GRPC_ENDPOINT: "https://169.254.1.2:8080",
+        OPENSHELL_GRPC_ENDPOINT: `https://${PORTABLE_HOST_GATEWAY_IP}:8080`,
         NETAVARK_FW: "iptables",
         OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock",
       });
       const toml = fs.readFileSync(env.OPENSHELL_GATEWAY_CONFIG, "utf-8");
       expect(toml).toContain('compute_drivers = ["podman"]');
       expect(toml).toContain("[openshell.drivers.podman]");
-      expect(toml).toContain('host_gateway_ip = "169.254.1.2"');
+      expect(toml).toContain(`host_gateway_ip = "${PORTABLE_HOST_GATEWAY_IP}"`);
       expect(toml).toContain('socket_path = "/run/user/1001/podman/podman.sock"');
       expect(toml).not.toContain("supervisor_bin");
     } finally {

src/lib/onboard/docker-driver-gateway-launch.test.ts

@@ -20,6 +20,7 @@ import {
   resolveDriftGatewayBin,
   shouldUseContainerizedGateway,
 } from "./docker-driver-gateway-launch";
+import { PORTABLE_HOST_GATEWAY_IP } from "./experimental/portable-profile";
 import { gatewayProcessCmdlineMatches } from "./gateway-process-identity";
 
 function withTempBinaries<T>(
@@ -158,7 +159,7 @@ describe("docker-driver-gateway-launch", () => {
   it("writes the exact rootless socket only for the Podman driver", () => {
     const toml = buildDockerDriverGatewayConfigToml({
       OPENSHELL_DRIVERS: "podman",
-      OPENSHELL_GRPC_ENDPOINT: "https://169.254.1.2:8080",
+      OPENSHELL_GRPC_ENDPOINT: `https://${PORTABLE_HOST_GATEWAY_IP}:8080`,
       OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker",
       OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "supervisor:test",
       OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock",

src/lib/onboard/docker-driver-gateway-local-tls.test.ts

@@ -12,6 +12,7 @@ import {
   ensureDockerDriverGatewayLocalTlsBundle,
   getDockerDriverGatewayLocalTlsBundle,
 } from "./docker-driver-gateway-local-tls";
+import { PORTABLE_HOST_GATEWAY_IP } from "./experimental/portable-profile";
 
 const TEST_CERT_VALID_AT = new Date("2026-06-27T00:00:00.000Z");
 const TEST_CERT_SKEW_BOUNDARY_NOT_YET_VALID_AT = new Date("2026-06-26T20:38:47.000Z");
@@ -239,7 +240,7 @@ describe("docker-driver-gateway-local-tls", () => {
           }) as never,
         }),
       ).toThrow("did not create a complete");
-      expect(calls[0]).toEqual(expect.arrayContaining(["--server-san", "169.254.1.2"]));
+      expect(calls[0]).toEqual(expect.arrayContaining(["--server-san", PORTABLE_HOST_GATEWAY_IP]));
     } finally {
       fs.rmSync(stateDir, { recursive: true, force: true });
     }

src/lib/onboard/experimental/hermes-portable-ollama-inference.test.ts

@@ -36,6 +36,7 @@ import {
 } from "../../../../test/helpers/hermes-portable-ollama-test-harness";
 import { hermesPortableOllamaAuthorityInternals } from "./hermes-portable-ollama-authority";
 import { createHermesPortableOllamaInferenceResolver } from "./hermes-portable-ollama-inference";
+import { PORTABLE_HOST_GATEWAY_IP } from "./portable-profile";
 
 const PODMAN_PATH = "/usr/bin/podman";
 const PODMAN_BYTES = Buffer.from("portable-podman-5.7.0", "utf8");
@@ -378,7 +379,7 @@ describe("Hermes Portable Ollama inference activation", () => {
           networkName: "openshell-docker",
           networkId: NETWORK_ID,
           networkGatewayIp: "169.254.1.1",
-          networkListenerIp: "169.254.1.2",
+          networkListenerIp: PORTABLE_HOST_GATEWAY_IP,
           gpuDevices: [GPU_DEVICE],
         },
       });
@@ -739,7 +740,7 @@ describe("Hermes Portable Ollama inference activation", () => {
     );
     fixture.gatewayProvider.setLookupFailure(false);
     const mutation = await selection.prepareGatewayMutation(gatewayMutationInput);
-    expect(() => createExactGatewayProvider(mutation, "http://169.254.1.2:11434/v1")).toThrow(
+    expect(() => createExactGatewayProvider(mutation, "http://192.0.2.2:11434/v1")).toThrow(
       "provider mutation authority changed",
     );
     createExactGatewayProvider(mutation);

src/lib/onboard/experimental/portable-host-preparation.test.ts

@@ -3,6 +3,7 @@
 
 import { spawnSync } from "node:child_process";
 import fs from "node:fs";
+import { BlockList } from "node:net";
 import os from "node:os";
 import path from "node:path";
 
@@ -137,6 +138,20 @@ describe("preparePortableExperimentalHost", () => {
     for (const tempDir of tempDirs) fs.rmSync(tempDir, { recursive: true, force: true });
   });
 
+  it("keeps the host gateway outside every Portable sandbox subnet (#9587)", () => {
+    const [networkAddress, prefixText] = PORTABLE_DOCKER_NETWORK_SUBNET.split("/");
+    const portableSandboxAddresses = new BlockList();
+    portableSandboxAddresses.addSubnet(networkAddress!, Number(prefixText), "ipv4");
+
+    expect(PORTABLE_DOCKER_NETWORK_SUBNET).toBe("169.254.1.0/24");
+    expect(PORTABLE_REGISTRY_IP).toBe("169.254.1.3");
+    expect(PORTABLE_HOST_GATEWAY_IP).toBe("169.254.2.2");
+    expect(portableSandboxAddresses.check("169.254.1.2", "ipv4")).toBe(true);
+    expect(portableSandboxAddresses.check(PORTABLE_REGISTRY_IP, "ipv4")).toBe(true);
+    expect(portableSandboxAddresses.check(PORTABLE_HOST_GATEWAY_IP, "ipv4")).toBe(false);
+    expect(PORTABLE_HOST_GATEWAY_IP).not.toBe(PORTABLE_REGISTRY_IP);
+  });
+
   it("does nothing unless the portable profile is explicit", () => {
     const systemctl = vi.fn();
     const docker = vi.fn();
@@ -419,7 +434,7 @@ describe("preparePortableExperimentalHost", () => {
         PORTABLE_REGISTRY_IP,
       ]),
     ]);
-    expect(ip).toHaveBeenCalledTimes(2);
+    expect(ip).toHaveBeenCalledTimes(4);
     expect(sudo).not.toHaveBeenCalled();
   });
 
@@ -473,6 +488,61 @@ describe("preparePortableExperimentalHost", () => {
     expect(commands).toEqual(["create network", "add host gateway"]);
   });
 
+  it.each([
+    ["missing network", false, "1: lo    inet 169.254.1.2/32 scope global lo\n"],
+    [
+      "existing network and replacement gateway",
+      true,
+      `1: lo    inet 169.254.1.2/32 scope global lo\n1: lo    inet ${PORTABLE_HOST_GATEWAY_IP}/32 scope global lo\n`,
+    ],
+  ])(
+    "rejects the retired gateway alias before inspecting the %s (#9587)",
+    (_case, simulateExistingPortableNetwork, addresses) => {
+      const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-"));
+      tempDirs.push(home);
+      const docker = vi.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>(() =>
+        result(),
+      );
+      const ip = vi.fn(() => result(0, addresses));
+      const sudo = vi.fn(() => result());
+
+      expect(() =>
+        preparePortableExperimentalHost(
+          { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
+          portablePreparationDeps(home, docker, { ip, sudo }),
+          undefined,
+          { simulateExistingPortableNetwork },
+        ),
+      ).toThrow(
+        /sudo ip address delete 169\.254\.1\.2\/32 dev lo.*nemoclaw onboard --experimental-profile portable/u,
+      );
+      expect(docker.mock.calls.map(([args]) => args)).toEqual([["--version"]]);
+      expect(ip).toHaveBeenCalledTimes(1);
+      expect(sudo).not.toHaveBeenCalled();
+    },
+  );
+
+  it("does not remove a conflicting retired gateway assignment (#9587)", () => {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-"));
+    tempDirs.push(home);
+    const docker = vi.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>(() =>
+      result(),
+    );
+    const sudo = vi.fn(() => result());
+
+    expect(() =>
+      preparePortableExperimentalHost(
+        { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" },
+        portablePreparationDeps(home, docker, {
+          ip: () => result(0, "2: eth0    inet 169.254.1.2/24 scope global eth0\n"),
+          sudo,
+        }),
+      ),
+    ).toThrow(/retired portable host gateway address 169\.254\.1\.2 has a conflicting/u);
+    expect(docker.mock.calls.map(([args]) => args)).toEqual([["--version"]]);
+    expect(sudo).not.toHaveBeenCalled();
+  });
+
   it("configures and verifies the portable gateway loopback alias before registry mutation (#9461)", () => {
     const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-"));
     tempDirs.push(home);
@@ -484,6 +554,7 @@ describe("preparePortableExperimentalHost", () => {
     const ip = vi
       .fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>()
       .mockReturnValueOnce(result())
+      .mockReturnValueOnce(result())
       .mockReturnValueOnce(
         result(0, `1: lo    inet ${PORTABLE_HOST_GATEWAY_IP}/32 scope global lo\n`),
       );
@@ -499,6 +570,7 @@ describe("preparePortableExperimentalHost", () => {
     expect(ip.mock.calls.map(([args]) => args)).toEqual([
       ["-o", "-4", "address", "show"],
       ["-o", "-4", "address", "show"],
+      ["-o", "-4", "address", "show"],
     ]);
     expect(sudo).toHaveBeenCalledWith(
       ["--", "ip", "address", "replace", `${PORTABLE_HOST_GATEWAY_IP}/32`, "dev", "lo"],

src/lib/onboard/experimental/portable-host-preparation.ts

@@ -41,6 +41,10 @@ import {
 
 const REGISTRY_CONTAINER = "nemoclaw-portable-registry";
 const REGISTRY_LABEL = "com.nvidia.nemoclaw.portable=1";
+// Portable onboarding assigned this address to loopback before #9587 moved the
+// host gateway outside the sandbox subnet. Keep the retired value here so an
+// upgraded host cannot silently retain a route that captures sandbox traffic.
+const RETIRED_PORTABLE_HOST_GATEWAY_IP = "169.254.1.2";
 const REGISTRY_IMAGE =
   "docker.io/library/registry:2@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
 const HOST_COMMAND_TIMEOUT_MS = 30_000;
@@ -128,8 +132,11 @@ function requireDockerCompatibleCli(
   );
 }
 
-function portableHostGatewayAliasState(output: string): "absent" | "configured" {
-  const escapedGatewayIp = PORTABLE_HOST_GATEWAY_IP.replaceAll(".", "\\.");
+function portableHostGatewayAliasState(
+  output: string,
+  gatewayIp = PORTABLE_HOST_GATEWAY_IP,
+): "absent" | "configured" {
+  const escapedGatewayIp = gatewayIp.replaceAll(".", "\\.");
   const addressPattern = new RegExp(`\\binet\\s+${escapedGatewayIp}/(\\d+)\\b`, "u");
   const assignments = output
     .split("\n")
@@ -143,7 +150,33 @@ function portableHostGatewayAliasState(output: string): "absent" | "configured"
     return "configured";
   }
   throw new Error(
-    `Refusing to configure ${PORTABLE_HOST_GATEWAY_IP}/32 because the address already has a conflicting host assignment.`,
+    `Refusing to configure ${gatewayIp}/32 because the address already has a conflicting host assignment.`,
+  );
+}
+
+function rejectRetiredPortableHostGatewayAlias(
+  env: NodeJS.ProcessEnv,
+  ip: NonNullable<PortableHostPreparationDeps["ip"]>,
+): void {
+  const result = ip(["-o", "-4", "address", "show"], env);
+  requireCommand(result, "Inspecting the retired portable host gateway address");
+  let state: "absent" | "configured";
+  try {
+    state = portableHostGatewayAliasState(
+      String(result.stdout ?? ""),
+      RETIRED_PORTABLE_HOST_GATEWAY_IP,
+    );
+  } catch {
+    throw new Error(
+      `The retired portable host gateway address ${RETIRED_PORTABLE_HOST_GATEWAY_IP} has a conflicting host assignment. ` +
+        "Resolve that assignment, then rerun `nemoclaw onboard --experimental-profile portable`.",
+    );
+  }
+  if (state === "absent") return;
+  throw new Error(
+    `The retired portable host gateway address ${RETIRED_PORTABLE_HOST_GATEWAY_IP}/32 is still assigned to loopback. ` +
+      `Remove it with \`sudo ip address delete ${RETIRED_PORTABLE_HOST_GATEWAY_IP}/32 dev lo\`, then rerun ` +
+      "`nemoclaw onboard --experimental-profile portable`.",
   );
 }
 
@@ -643,6 +676,7 @@ export function preparePortableExperimentalHost(
         env: childEnv,
         timeout: HOST_COMMAND_TIMEOUT_MS,
       }));
+  rejectRetiredPortableHostGatewayAlias(podmanEnv, ip);
   ensurePortableSandboxNetwork(podmanEnv, docker, dockerNetworkName);
   ensurePortableHostGatewayAlias(podmanEnv, ip, sudo);
   ensureRegistryContainer(podmanEnv, docker, dockerNetworkName);

src/lib/onboard/experimental/portable-profile.ts

@@ -5,7 +5,7 @@ import { DEFAULT_DOCKER_DRIVER_NETWORK_NAME } from "./docker-network-authority";
 
 export const EXPERIMENTAL_PROFILE_ENV = "NEMOCLAW_EXPERIMENTAL_PROFILE";
 export const PORTABLE_EXPERIMENTAL_PROFILE = "portable";
-export const PORTABLE_HOST_GATEWAY_IP = "169.254.1.2";
+export const PORTABLE_HOST_GATEWAY_IP = "169.254.2.2";
 export const PORTABLE_REGISTRY_IP = "169.254.1.3";
 export const PORTABLE_DOCKER_NETWORK_NAME = DEFAULT_DOCKER_DRIVER_NETWORK_NAME;
 export const PORTABLE_DOCKER_NETWORK_SUBNET = "169.254.1.0/24";