fix(inference): pin recovered vLLM container identity

Signed-off-by: prekshivyas <prekshiv@nvidia.com>

Author: prekshivyas

src/lib/inference/serving/vllm-host-local-lifecycle.test.ts

@@ -113,7 +113,11 @@ describe("host-local managed vLLM recovery", () => {
         dockerCapture: capture,
         loadApiKey: () => API_KEY,
       }),
-    ).toEqual({ baseUrl: "http://127.0.0.1:8000", apiKey: API_KEY });
+    ).toEqual({
+      baseUrl: "http://127.0.0.1:8000",
+      apiKey: API_KEY,
+      containerId: "a".repeat(64),
+    });
 
     expect(capture).toHaveBeenCalledOnce();
     const dockerOptions = capture.mock.calls[0]?.[1];
@@ -132,7 +136,11 @@ describe("host-local managed vLLM recovery", () => {
         loadApiKey: () => API_KEY,
         onManagedContainerObserved: observed,
       }),
-    ).toEqual({ baseUrl: "http://127.0.0.1:8000", apiKey: API_KEY });
+    ).toEqual({
+      baseUrl: "http://127.0.0.1:8000",
+      apiKey: API_KEY,
+      containerId: "a".repeat(64),
+    });
     expect(observed).toHaveBeenCalledOnce();
   });
 
@@ -171,7 +179,11 @@ describe("host-local managed vLLM recovery", () => {
         loadApiKey: () => API_KEY,
         stateDir: directory,
       }),
-    ).toEqual({ baseUrl: "http://127.0.0.1:8000", apiKey: API_KEY });
+    ).toEqual({
+      baseUrl: "http://127.0.0.1:8000",
+      apiKey: API_KEY,
+      containerId: "a".repeat(64),
+    });
   });
 
   it("rejects a profile-labeled runtime when its ownership receipt is missing", () => {

src/lib/inference/serving/vllm-host-local-lifecycle.ts

@@ -276,7 +276,7 @@ function inspectHostLocalContainer(
 /** Recover only the exact authenticated host-local container with bounded host bindings. */
 export function recoverHostLocalManagedVllmEndpoint(
   options: RecoverHostLocalManagedVllmOptions = {},
-): { baseUrl: string; apiKey: string } | null {
+): { baseUrl: string; apiKey: string; containerId: string } | null {
   const capture = options.dockerCapture ?? dockerCapture;
   const dockerEnv = buildLocalManagedVllmDockerEnv();
   const source = (
@@ -364,5 +364,9 @@ export function recoverHostLocalManagedVllmEndpoint(
   ) {
     throw new Error("Managed host-local vLLM authentication is missing or mismatched.");
   }
-  return { baseUrl: `http://127.0.0.1:${String(HOST_LOCAL_VLLM_PORT)}`, apiKey };
+  return {
+    baseUrl: `http://127.0.0.1:${String(HOST_LOCAL_VLLM_PORT)}`,
+    apiKey,
+    containerId: row.Id,
+  };
 }

src/lib/inference/vllm-serving-port.test.ts

@@ -130,6 +130,7 @@ describe("managed vLLM serving-port guard (#8685)", () => {
     mocks.recoverHostLocalManagedVllmEndpoint.mockReturnValue({
       baseUrl: "http://127.0.0.1:8000",
       apiKey: "b".repeat(64),
+      containerId: MANAGED_CONTAINER_ID,
     });
     const checkServingPort = vi.fn(async () => ({ ok: false, reason: "port 8000 is held" }));
 
@@ -151,6 +152,31 @@ describe("managed vLLM serving-port guard (#8685)", () => {
     expect(errSpy.mock.calls.flat().join("\n")).not.toContain("another process");
   });
 
+  it("fails closed when the managed container changes after recovery", async () => {
+    const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!;
+    mockSuccessfulVllmInstall(mocks, profile.containerName, [
+      () => vllmContainerRow(profile.containerName, { id: "c".repeat(64) }),
+    ]);
+    mocks.recoverHostLocalManagedVllmEndpoint.mockReturnValue({
+      baseUrl: "http://127.0.0.1:8000",
+      apiKey: "b".repeat(64),
+      containerId: MANAGED_CONTAINER_ID,
+    });
+
+    const result = await installVllm(profile, {
+      hasImage: true,
+      nonInteractive: true,
+      promptFn: vi.fn<(q: string) => Promise<string>>(),
+      checkServingPort: async () => ({ ok: false, reason: "port 8000 is held" }),
+    });
+
+    expect(result).toEqual({ ok: false });
+    expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled();
+    expect(mocks.dockerForceRm).not.toHaveBeenCalled();
+    expect(mocks.dockerRunDetached).not.toHaveBeenCalled();
+    expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("changed after recovery"));
+  });
+
   it("fails closed when a managed container holding the serving port cannot be recovered", async () => {
     const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!;
     mockSuccessfulVllmInstall(mocks, profile.containerName);

src/lib/inference/vllm.ts

@@ -738,6 +738,7 @@ function inspectVllmContainerOwnership(containerName: string): VllmContainerOwne
 function vllmContainerReplacementTarget(
   containerName: string,
   dockerEnv?: Record<string, string>,
+  expectedContainerId?: string,
 ): { ok: true; containerId?: string } | { ok: false; reason: string } {
   const ownership = dockerEnv
     ? inspectVllmContainerOwnershipInDockerEnv(containerName, dockerEnv)
@@ -762,6 +763,15 @@ function vllmContainerReplacementTarget(
         `Refusing single-host replacement because it would orphan the peer worker. Restore ${NEMOCLAW_DGX_STATION_PEER_ENV} and select Nemotron Ultra to manage the pair.`,
     };
   }
+  if (
+    expectedContainerId &&
+    (ownership.kind !== "managed" || ownership.containerId !== expectedContainerId)
+  ) {
+    return {
+      ok: false,
+      reason: `Managed vLLM container "${containerName}" changed after recovery. NemoClaw will not remove it. Retry onboarding.`,
+    };
+  }
   return ownership.kind === "managed"
     ? { ok: true, containerId: ownership.containerId }
     : { ok: true };
@@ -868,6 +878,7 @@ function startContainer(
   dockerEnv: Record<string, string> = buildVllmDockerEnv(),
   resolveBridgeHost: (dockerEnv: Record<string, string>) => string = (env) =>
     resolveManagedVllmBridgeHost(dockerCapture, env),
+  expectedReplacementContainerId?: string,
 ): { ok: true; containerId: string } | { ok: false; reason: string } {
   emit(`Starting vLLM container (${profile.containerName})`);
   // The explicit download completed before this long-lived container starts,
@@ -892,6 +903,7 @@ function startContainer(
   const replacement = vllmContainerReplacementTarget(
     profile.containerName,
     model.managedBearerAuth ? dockerEnv : undefined,
+    expectedReplacementContainerId,
   );
   if (!replacement.ok) return replacement;
   if (replacement.containerId) {
@@ -1735,6 +1747,7 @@ async function runVllmInstall(
   // the guard first keeps a refused install free of both side effects.
   // Port 25000 is not checked here: it belongs to the managed-cluster
   // rendezvous contract and this single-node path never binds it.
+  let recoveredHostLocalContainerId: string | undefined;
   const servingPort = await opts.checkServingPort?.(VLLM_PORT);
   if (servingPort && !servingPort.ok) {
     // An interrupted host-local install can leave its authenticated managed
@@ -1743,7 +1756,9 @@ async function runVllmInstall(
     // credential fingerprint. The replacement guard below then removes the
     // inspected container ID immediately before the new launch.
     try {
-      if (recoverHostLocalManagedVllmEndpoint()) {
+      const recovered = recoverHostLocalManagedVllmEndpoint();
+      if (recovered) {
+        recoveredHostLocalContainerId = recovered.containerId;
         // Continue through the ordinary managed-container replacement path.
       } else {
         printServingPortConflict(servingPort);
@@ -1830,6 +1845,7 @@ async function runVllmInstall(
     const replacement = vllmContainerReplacementTarget(
       runtimeProfile.containerName,
       model.managedBearerAuth ? localDockerEnv : undefined,
+      recoveredHostLocalContainerId,
     );
     if (!replacement.ok) {
       console.error(`  vLLM install failed: ${replacement.reason}`);
@@ -2074,6 +2090,7 @@ async function runVllmInstall(
     model,
     localDockerEnv,
     opts.resolveManagedBridgeHost,
+    recoveredHostLocalContainerId,
   );
   if (!start.ok) {
     console.error(`  vLLM install failed: ${String(start.reason)}`);