fix(onboard): name the failed GPU trust-gate check in preflight

When the GPU trust gate rejected an nvidia-smi report, preflight printed
only the bare "no GPU detected", which points the user at hardware and
drivers that can be working. The check that actually failed — an absent
/proc/driver/nvidia interface, an unattempted proof on multiple GPU
rows, an unavailable prover, or a failed bounded CUDA workload — was
known at rejection time and then discarded.

detectGpu() now reports the rejecting check through an injected
onTrustGateRejection callback. The reason is fixed text only and never
echoes nvidia-smi output, which is untrusted. The readiness-gated
runtime preflight carries the reason from the newest detection on its
result, and onboard preflight prints it under the existing
"Local NIM unavailable — no GPU detected" line.

Closes #9000

Signed-off-by: Dongni Yang <dongniy@nvidia.com>

Author: Dongni-Yang

docs/reference/troubleshooting.mdx

@@ -951,6 +951,7 @@ A failed or timed-out workload leaves the GPU unproven and does not enable GPU p
 The names-only unified-memory fallback does not run this workload and rejects denylisted names.
 WSL hosts that are not Docker Desktop-backed do not run the workload and continue to report the GPU as unavailable.
 
+When GPU detection rejects the `nvidia-smi` report, preflight prints the failed check under the `Local NIM unavailable — no GPU detected` line, for example an absent `/proc/driver/nvidia` interface or a failed bounded CUDA proof.
 If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers.
 
 Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route.

src/lib/inference/nim.gpu-proof-plausible-name.test.ts

@@ -188,3 +188,110 @@ describe("detectGpu CUDA proof for a plausible, non-placeholder NVIDIA GPU name
     });
   });
 });
+
+describe("detectGpu trust-gate rejection reasons (#9000)", () => {
+  const collectReasons = () => {
+    const reasons: string[] = [];
+    return { reasons, onTrustGateRejection: (reason: string) => reasons.push(reason) };
+  };
+
+  it("reports the absent kernel interface when the CUDA proof fails (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: failingProver(),
+          runCaptureImpl: makeRunCapture(`${PLAUSIBLE_NAME}, 8128, 7000\n`),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).toBeNull();
+    });
+    expect(reasons).toEqual(["/proc/driver/nvidia is absent and the bounded CUDA proof failed"]);
+  });
+
+  it("reports an unattempted proof when no prover is available (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: null,
+          runCaptureImpl: makeRunCapture(`${PLAUSIBLE_NAME}, 8128, 7000\n`),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).toBeNull();
+    });
+    expect(reasons).toEqual([
+      "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted",
+    ]);
+  });
+
+  it("reports an unrecognized GPU name without attempting the proof (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: passingProver(),
+          runCaptureImpl: makeRunCapture("Graphics Device, 8128, 7000\n"),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).toBeNull();
+    });
+    expect(reasons).toEqual([
+      "nvidia-smi reported a GPU name that is not a recognized NVIDIA product and the bounded CUDA proof was not attempted",
+    ]);
+  });
+
+  it("reports multiple GPU rows without attempting the proof (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: passingProver(),
+          runCaptureImpl: makeRunCapture(
+            `${PLAUSIBLE_NAME}, 8128, 7000\nNVIDIA GeForce RTX 4090 Laptop GPU, 16376, 15000\n`,
+          ),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).toBeNull();
+    });
+    expect(reasons).toEqual([
+      "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted for multiple GPU rows",
+    ]);
+  });
+
+  it("reports a placeholder GPU name when its CUDA proof fails (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: failingProver(),
+          runCaptureImpl: makeRunCapture("JMJWOA-Generic-GPU, 65471, 65000\n"),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).toBeNull();
+    });
+    expect(reasons).toEqual([
+      "nvidia-smi reported a placeholder GPU name and the bounded CUDA proof failed",
+    ]);
+  });
+
+  it("emits no rejection reason when the CUDA proof passes (#9000)", () => {
+    const { reasons, onTrustGateRejection } = collectReasons();
+    onWsl2Arm64WithoutKernelInterface(() => {
+      expect(
+        detectGpu({
+          proveArm64WslDockerDesktopGpu: passingProver(),
+          runCaptureImpl: makeRunCapture(`${PLAUSIBLE_NAME}, 8128, 7000\n`),
+          isWsl: true,
+          onTrustGateRejection,
+        }),
+      ).not.toBeNull();
+    });
+    expect(reasons).toEqual([]);
+  });
+});

src/lib/inference/nim.ts

@@ -109,6 +109,11 @@ export interface DetectGpuDeps {
   runCaptureImpl?: typeof runCapture;
   /** Override WSL detection for deterministic tests. */
   isWsl?: boolean;
+  // Receives one sentence naming the check that rejected an nvidia-smi probe
+  // when the trust gate returns no GPU, so preflight can say which check
+  // failed instead of the bare "no GPU detected" (#9000). The reason is built
+  // from fixed text only — never from nvidia-smi output, which is untrusted.
+  onTrustGateRejection?: (reason: string) => void;
 }
 
 // Lazily construct the default ARM64 Linux GPU prover. Keep it behind a require
@@ -467,16 +472,21 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null {
         // The all-GPU CUDA workload proves that at least one usable device
         // exists. It does not establish which nvidia-smi rows or capacities
         // are genuine, so a multi-row response stays untrusted.
-        const passesBoundedCudaProof = (): boolean => {
+        // Null when the proof passed; otherwise a fixed-text fragment naming
+        // why it did not, composed into the onTrustGateRejection reason.
+        const boundedCudaProofRejection = (): string | null => {
           if (parsed.length !== 1) {
-            return false;
+            return "the bounded CUDA proof was not attempted for multiple GPU rows";
           }
           const prover =
             deps.proveArm64WslDockerDesktopGpu === undefined
               ? defaultArm64WslDockerDesktopGpuProver()
               : deps.proveArm64WslDockerDesktopGpu;
           const proof = prover ? prover(parsed.map((p: ParsedGpu) => p.name)) : null;
-          return !!proof && proof.passed;
+          if (!proof) {
+            return "the bounded CUDA proof was not attempted";
+          }
+          return proof.passed ? null : "the bounded CUDA proof failed";
         };
         let trusted: ParsedGpu[];
         let wslDockerDesktopGpuProofPassed = false;
@@ -489,7 +499,11 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null {
           // A bounded Docker `--gpus` workload proves that the single reported
           // row has a usable CUDA device. The Snapdragon shim cannot pass it
           // (#4565 without reopening #3988/#4424).
-          if (!passesBoundedCudaProof()) {
+          const proofRejection = boundedCudaProofRejection();
+          if (proofRejection) {
+            deps.onTrustGateRejection?.(
+              `nvidia-smi reported a placeholder GPU name and ${proofRejection}`,
+            );
             return null;
           }
           trusted = parsed;
@@ -504,7 +518,15 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null {
             // name the filter below would discard never starts the Docker workload.
             // Without a passing proof this path stays fail-closed as before.
             const plausible = parsed.every((p: ParsedGpu) => isPlausibleNvidiaGpuName(p.name));
-            if (!plausible || !passesBoundedCudaProof()) {
+            if (!plausible) {
+              deps.onTrustGateRejection?.(
+                "nvidia-smi reported a GPU name that is not a recognized NVIDIA product and the bounded CUDA proof was not attempted",
+              );
+              return null;
+            }
+            const proofRejection = boundedCudaProofRejection();
+            if (proofRejection) {
+              deps.onTrustGateRejection?.(`/proc/driver/nvidia is absent and ${proofRejection}`);
               return null;
             }
             wslDockerDesktopGpuProofPassed = true;

src/lib/onboard.ts

@@ -1361,7 +1361,7 @@ async function preflight(
   preflightOpts: PreflightOptions = {},
 ): Promise<ReturnType<typeof nim.detectGpu>> {
   step(1, 8, "Preflight checks");
-  const { gpu, host, sandboxGpuConfig } =
+  const { gpu, host, sandboxGpuConfig, gpuTrustGateRejection } =
     await onboardPreflightGatewayAuthority.runRuntimePreflight(preflightOpts);
 
   await preflightUtils.checkContainerRuntimeResources(host, {
@@ -1515,6 +1515,9 @@ async function preflight(
     console.log("  ⓘ Local NIM unavailable — requires NVIDIA GPU");
   } else {
     console.log("  ⓘ Local NIM unavailable — no GPU detected");
+    if (gpuTrustGateRejection) {
+      console.log(`    GPU detection rejected the nvidia-smi report: ${gpuTrustGateRejection}`);
+    }
   }
 
   if (sandboxGpuConfig.sandboxGpuEnabled) {

src/lib/onboard/fatal-runtime-preflight.test.ts

@@ -670,3 +670,89 @@ describe("readiness-gated runtime preflight", () => {
     expect(gpu).not.toHaveBeenCalled();
   });
 });
+
+describe("GPU trust-gate rejection reason propagation (#9000)", () => {
+  const gatedContext = (detectGpu: (deps?: DetectGpuDeps) => GpuDetection | null, host: HostAssessment) => ({
+    nonInteractive: true,
+    collectGatewayReadiness: async () => managedGatewayReadiness(),
+    assessHost: () => host,
+    detectGpu,
+    warnIfHostProxyMissesLoopback: vi.fn(),
+    assertDockerBridgeAndContainerDnsHealthy: vi.fn(),
+    validateSandboxGpuPreflight: vi.fn(),
+  });
+
+  it("carries the runtime-proof rejection reason when the bounded proof fails (#9000)", async () => {
+    const detectGpu = vi.fn((deps?: DetectGpuDeps): GpuDetection | null => {
+      const isObservation = deps?.proveArm64WslDockerDesktopGpu === null;
+      deps?.onTrustGateRejection?.(
+        isObservation
+          ? "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted"
+          : "/proc/driver/nvidia is absent and the bounded CUDA proof failed",
+      );
+      return null;
+    });
+
+    const result = await runReadinessGatedRuntimePreflight(
+      {},
+      gatedContext(detectGpu, wslDockerDesktopHost()),
+    );
+
+    expect(result.gpu).toBeNull();
+    expect(result.gpuTrustGateRejection).toBe(
+      "/proc/driver/nvidia is absent and the bounded CUDA proof failed",
+    );
+  });
+
+  it("carries the observation rejection reason when no runtime proof is required (#9000)", async () => {
+    const detectGpu = vi.fn((deps?: DetectGpuDeps): GpuDetection | null => {
+      deps?.onTrustGateRejection?.(
+        "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted",
+      );
+      return null;
+    });
+
+    const result = await runReadinessGatedRuntimePreflight(
+      {},
+      gatedContext(detectGpu, {
+        ...hostWithRuntime("docker"),
+        hasNvidiaGpu: true,
+        nvidiaContainerToolkitInstalled: true,
+        dockerCdiSpecDirs: ["/etc/cdi"],
+      }),
+    );
+
+    expect(result.gpu).toBeNull();
+    expect(result.gpuTrustGateRejection).toBe(
+      "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted",
+    );
+  });
+
+  it("omits the rejection reason when the runtime proof passes (#9000)", async () => {
+    const detectGpu = vi.fn((deps?: DetectGpuDeps): GpuDetection | null => {
+      const isObservation = deps?.proveArm64WslDockerDesktopGpu === null;
+      isObservation &&
+        deps?.onTrustGateRejection?.(
+          "/proc/driver/nvidia is absent and the bounded CUDA proof was not attempted",
+        );
+      return isObservation
+        ? null
+        : {
+            type: "nvidia",
+            count: 1,
+            totalMemoryMB: 32_768,
+            perGpuMB: 32_768,
+            nimCapable: true,
+            wslDockerDesktopGpuProofPassed: true,
+          };
+    });
+
+    const result = await runReadinessGatedRuntimePreflight(
+      {},
+      gatedContext(detectGpu, wslDockerDesktopHost()),
+    );
+
+    expect(result.gpu).toMatchObject({ wslDockerDesktopGpuProofPassed: true });
+    expect(result.gpuTrustGateRejection).toBeUndefined();
+  });
+});

src/lib/onboard/fatal-runtime-preflight.ts

@@ -74,6 +74,11 @@ export interface FatalRuntimePreflightResult {
   host: HostAssessment;
   readinessReport: SystemReadinessReport;
   sandboxGpuConfig: SandboxGpuConfig;
+  // Which trust-gate check rejected the newest GPU detection, so preflight can
+  // name the failed check instead of the bare "no GPU detected" (#9000).
+  // Absent when detection found a GPU, never ran the trust gate, or discarded
+  // every row through the plausible-name filter, which records no reason.
+  gpuTrustGateRejection?: string;
 }
 
 export type ReadinessGatedRuntimePreflightContext = Omit<
@@ -239,9 +244,15 @@ function refreshOnboardHostReadiness(
   const now = context.now ?? (() => new Date());
   const observedAt = now().toISOString();
   const host = (context.assessHost ?? assessHost)();
+  let gpuTrustGateRejection: string | undefined;
   const gpu = runtimeGpu
     ? runtimeGpu.value
-    : (context.detectGpu ?? detectGpu)({ proveArm64WslDockerDesktopGpu: null });
+    : (context.detectGpu ?? detectGpu)({
+        proveArm64WslDockerDesktopGpu: null,
+        onTrustGateRejection: (reason) => {
+          gpuTrustGateRejection = reason;
+        },
+      });
   const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, {
     flag: resolveSandboxGpuFlagFromOptions(options),
     device: options.sandboxGpuDevice ?? null,
@@ -256,7 +267,13 @@ function refreshOnboardHostReadiness(
     observedAt,
     now,
   });
-  return { gpu, host, readinessReport, sandboxGpuConfig };
+  return {
+    gpu,
+    host,
+    readinessReport,
+    sandboxGpuConfig,
+    ...(gpuTrustGateRejection ? { gpuTrustGateRejection } : {}),
+  };
 }
 
 /** Resolve the bounded WSL GPU proof only after canonical readiness admission. */
@@ -266,13 +283,21 @@ function resolveRuntimeGpuProof(
   context: FatalRuntimePreflightContext,
 ): { result: FatalRuntimePreflightResult; proofRan: boolean } {
   if (!requiresRuntimeGpuProof(result, options)) return { result, proofRan: false };
-  const gpu = (context.detectGpu ?? detectGpu)();
+  let gpuTrustGateRejection: string | undefined;
+  const gpu = (context.detectGpu ?? detectGpu)({
+    onTrustGateRejection: (reason) => {
+      gpuTrustGateRejection = reason;
+    },
+  });
   const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, {
     flag: resolveSandboxGpuFlagFromOptions(options),
     device: options.sandboxGpuDevice ?? null,
   });
+  // The proof-phase detection replaces the observation-phase result, so its
+  // rejection reason (or its absence, when the proof passed) replaces the
+  // observation-phase reason too.
   return {
-    result: { ...result, gpu, sandboxGpuConfig },
+    result: { ...result, gpu, sandboxGpuConfig, gpuTrustGateRejection },
     proofRan: true,
   };
 }
@@ -380,6 +405,14 @@ export async function runReadinessGatedRuntimePreflight(
           ? false
           : runtimeGpu.result.gpu.wslDockerDesktopGpuProofPassed,
     });
+    // The refresh reuses the proof-phase GPU value without re-detecting, so
+    // carry the proof-phase rejection reason alongside it (#9000).
+    refreshedResult = {
+      ...refreshedResult,
+      ...(runtimeGpu.result.gpuTrustGateRejection
+        ? { gpuTrustGateRejection: runtimeGpu.result.gpuTrustGateRejection }
+        : {}),
+    };
   }
   gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness);
   assertOnboardGatewayReadiness(gatewayReadiness, exitProcess);