Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/lib/inference/local-vllm-auth.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,9 @@ describe("managed vLLM authentication", () => {
.filter((argv) => argv[0] === "docker");

expect(result.ok).toBe(false);
expect(dockerCommands).toHaveLength(5);
// 3 reachability probes, 2 diagnostic re-probes, and the daemon probe of
// the image-pull classifier (#9308) — every one context-pinned.
expect(dockerCommands).toHaveLength(6);
expect(
dockerCommands.every((argv) => argv.slice(0, 3).join(" ") === "docker --context default"),
).toBe(true);
Expand Down
54 changes: 54 additions & 0 deletions src/lib/inference/local.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,60 @@ describe("local inference helpers", () => {
expect(result.diagnostic).toMatch(/Docker command failed/);
});

it("reports an image-pull failure instead of an Ollama networking failure when Docker cannot provide the probe image (#9308)", () => {
const mockCapture = (cmd: readonly string[]) =>
cmd.includes("version")
? "29.6.2"
: cmd.includes("inspect")
? ""
: cmd.includes("run")
? ""
: '{"models":[]}';
const noopSleep = () => {};
const result = validateLocalProvider("ollama-local", mockCapture, noopSleep);
expect(result.ok).toBe(false);
expect(result.message).toMatch(/Docker image-pull failure/);
expect(result.message).toMatch(/not an Ollama networking failure/);
expect(result.message).not.toMatch(/Docker container reachability check failed/);
expect(result.message).not.toMatch(/sandbox uses a different network path/);
expect(result.diagnostic).toMatch(/DOCKER_CONFIG=\$\(mktemp -d\) docker pull curlimages\/curl/);
expect(result.diagnostic).toMatch(/credential helper/);
expect(result.diagnostic).toMatch(/onboard --resume/);
});

it("reports an image-pull failure instead of a vLLM networking failure when Docker cannot provide the probe image (#9308)", () => {
const mockCapture = (cmd: readonly string[]) =>
cmd.includes("version")
? "29.6.2"
: cmd.includes("inspect")
? ""
: cmd.includes("run")
? ""
: '{"data":[]}';
const noopSleep = () => {};
const result = validateLocalProvider("vllm-local", mockCapture, noopSleep);
expect(result.ok).toBe(false);
expect(result.message).toMatch(/Docker image-pull failure/);
expect(result.message).toMatch(/not a vLLM networking failure/);
expect(result.diagnostic).toMatch(/docker pull curlimages\/curl/);
});

it("keeps the runtime-failure report when the probe image is present locally (#9308)", () => {
const mockCapture = (cmd: readonly string[]) =>
cmd.includes("version")
? "29.6.2"
: cmd.includes("inspect")
? "sha256:0d9b7ef1"
: cmd.includes("run")
? ""
: '{"models":[]}';
const noopSleep = () => {};
const result = validateLocalProvider("ollama-local", mockCapture, noopSleep);
expect(result.ok).toBe(false);
expect(result.message).toMatch(/Docker container reachability check failed/);
expect(result.diagnostic).toMatch(/image pull error or runtime failure/);
});

it("succeeds after container check retry", () => {
let callCount = 0;
const mockCapture = () => {
Expand Down
90 changes: 82 additions & 8 deletions src/lib/inference/local.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1164,28 +1164,51 @@ export function validateLocalProvider(
// All retries exhausted — collect diagnostics
const diagnostic = collectContainerDiagnostic(containerCommand, capture);

if (diagnostic.probeImageUnavailable) {
return probeImageUnavailableResult(provider, diagnostic.text);
}

switch (provider) {
case "vllm-local":
return {
ok: false,
message: `Local vLLM is responding on the host, but the Docker container reachability check failed for ${getContainerCheckUrl(provider)}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`,
diagnostic,
diagnostic: diagnostic.text,
};
case "ollama-local":
return {
ok: false,
message: `Local Ollama is responding on ${getResolvedOllamaHost()}, but the Docker container reachability check failed for http://host.openshell.internal:${getOllamaContainerPort()}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`,
diagnostic,
diagnostic: diagnostic.text,
};
default:
return {
ok: false,
message: "The selected local inference provider is unavailable from containers.",
diagnostic,
diagnostic: diagnostic.text,
};
}
}

/**
* Report a reachability check that never ran because Docker could not
* provide the probe image (#9308). Blaming the provider's network path here
* is a misreport: the reporter's environment had a working path once the
* image existed.
*/
function probeImageUnavailableResult(provider: string, diagnostic: string): ValidationResult {
const responding =
provider === "vllm-local"
? "Local vLLM is responding on the host"
: `Local Ollama is responding on ${getResolvedOllamaHost()}`;
const providerLabel = provider === "vllm-local" ? "a vLLM" : "an Ollama";
return {
ok: false,
message: `${responding}, but the container reachability check could not run because Docker could not provide its probe image. This is a Docker image-pull failure, not ${providerLabel} networking failure.`,
diagnostic,
};
}

function getContainerCheckUrl(provider: string): string | null {
switch (provider) {
case "vllm-local": {
Expand All @@ -1203,13 +1226,62 @@ function getContainerCheckUrl(provider: string): string | null {
}
}

function collectContainerDiagnostic(containerCommand: string[], capture: RunCaptureFn): string {
type ContainerDiagnostic = { text: string; probeImageUnavailable: boolean };

function containerRuntimeFailureDiagnostic(text: string): ContainerDiagnostic {
return { text, probeImageUnavailable: false };
}

/**
* Distinguish "Docker could not provide the probe image" from a general
* runtime failure using the stdout-only capture seam: the daemon answers
* `docker version` while `docker image inspect` finds no local copy of the
* probe image. Credential-helper failures land here (#9308) — a remote login
* session can lose access to Docker Desktop's credential store, so the pull
* fails and every probe run produces empty stdout.
*
* Image-absent-after-run-attempts proves the pull failed: `docker run` pulls
* an absent image before it creates the container, and `--add-host`, policy,
* and seccomp failures all happen after that pull. Five runs precede this
* check (three probes, two diagnostics), so a pullable image would be in the
* cache by now and a run that failed for any post-pull reason keeps the
* generic runtime diagnostic.
*/
function classifyContainerRunFailure(
dockerCommand: string[],
capture: RunCaptureFn,
): ContainerDiagnostic {
const runtimeFailure = containerRuntimeFailureDiagnostic(
`Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`,
);
const daemonVersion = capture(
[...dockerCommand, "version", "--format", "{{.Server.Version}}"],
{ ignoreError: true },
);
if (!daemonVersion) return runtimeFailure;
const probeImageId = capture(
[...dockerCommand, "image", "inspect", "--format", "{{.Id}}", CONTAINER_REACHABILITY_IMAGE],
{ ignoreError: true },
);
if (probeImageId) return runtimeFailure;
return {
text: `The probe image ${CONTAINER_REACHABILITY_IMAGE} is not in the local Docker image cache, and Docker could not pull it. The image is public and needs no credentials, but a Docker credential helper (credsStore in ~/.docker/config.json) can fail in a remote login session and block every pull. Pre-pull the image with an isolated Docker config, then resume: DOCKER_CONFIG=$(mktemp -d) docker pull ${CONTAINER_REACHABILITY_IMAGE} && nemoclaw onboard --resume`,
probeImageUnavailable: true,
};
}

function collectContainerDiagnostic(
containerCommand: string[],
capture: RunCaptureFn,
): ContainerDiagnostic {
const url = containerCommand.at(-1);
const dockerRunIndex = containerCommand.indexOf("run");
const addHostIndex = containerCommand.indexOf("--add-host");
const hostAlias = containerCommand[addHostIndex + 1];
if (!url || dockerRunIndex < 1 || addHostIndex < 0 || !hostAlias) {
return `Docker command failed (invalid reachability command). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`;
return containerRuntimeFailureDiagnostic(
`Docker command failed (invalid reachability command). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`,
);
}
const dockerCommand = containerCommand.slice(0, dockerRunIndex);
try {
Expand Down Expand Up @@ -1252,7 +1324,7 @@ function collectContainerDiagnostic(containerCommand: string[], capture: RunCapt
);

if (!httpStatus && !hostsOutput) {
return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`;
return classifyContainerRunFailure(dockerCommand, capture);
}

const parts: string[] = [];
Expand All @@ -1270,9 +1342,11 @@ function collectContainerDiagnostic(containerCommand: string[], capture: RunCapt
parts.push(
`Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times over ~${(CONTAINER_CHECK_MAX_ATTEMPTS - 1) * CONTAINER_CHECK_RETRY_DELAY_SECS}s`,
);
return parts.join(". ") + ".";
return containerRuntimeFailureDiagnostic(parts.join(". ") + ".");
} catch {
return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`;
return containerRuntimeFailureDiagnostic(
`Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`,
);
}
}

Expand Down
Loading