fix(uat): append GKE driver lib path for served vLLM worker

The #1644 proving run (32732018329) reached phase_serve for the first
time and the VllmDecodeWorker crash-looped before binding its health
port. GKE mounts the node driver at /usr/local/nvidia without putting
it on LD_LIBRARY_PATH, so vLLM cannot dlopen libcuda.so.1 — the same
hole inference-perf already works around, which is why it served the
same runtime on the same cluster minutes earlier.

Wrap the worker command with the append used by the
validators/performance Dynamo templates, and capture --previous
container logs in serve_debug so a crash loop is diagnosable instead
of dumping the empty stdout of its replacement container.

Signed-off-by: Rohit Rajani <rorajani@nvidia.com>

Author: rorajani

tests/uat/lib/phases.sh

@@ -1307,10 +1307,15 @@ serve_debug() {
     kubectl describe pods -n "${SERVE_NAMESPACE}" 2>&1 || true
     echo "--- events ---"
     kubectl get events -n "${SERVE_NAMESPACE}" --sort-by=.lastTimestamp 2>&1 || true
-    echo "--- logs (all pods, all containers, last 200 lines) ---"
+    echo "--- logs (all pods, all containers, last 200 lines; --previous for crash loops) ---"
     for p in $(kubectl get pods -n "${SERVE_NAMESPACE}" -o name 2>/dev/null); do
       echo "=== ${p} ==="
       kubectl logs -n "${SERVE_NAMESPACE}" "${p#pod/}" --all-containers --tail=200 2>&1 || true
+      # CrashLoopBackOff replaces the current container before this dump
+      # runs, so the dying process's stdout is on --previous. Missing
+      # previous (first start, or already GC'd) is a no-op.
+      echo "=== ${p} (previous) ==="
+      kubectl logs -n "${SERVE_NAMESPACE}" "${p#pod/}" --all-containers --previous --tail=200 2>&1 || true
     done
   } | tee serve-logs/"${SERVE_NAME}".log
 }
@@ -1384,7 +1389,24 @@ spec:
             - name: main
               image: ${SERVE_RUNTIME_IMAGE}
               workingDir: /workspace/examples/backends/vllm
-              command: ["python3", "-m", "dynamo.vllm"]
+              # A bare `python3 -m dynamo.vllm` here crash-looped on GKE before
+              # binding its health port (run 32732018329), while inference-perf
+              # served the same runtime on the SAME cluster minutes earlier
+              # using this wrapper. GKE mounts the node driver at
+              # /usr/local/nvidia without putting it on LD_LIBRARY_PATH, so
+              # vLLM cannot dlopen libcuda.so.1 ("Failed to infer device type",
+              # observed live in gke-default qualification — see the same
+              # wrapper in validators/performance/testdata/inference).
+              # Shell APPEND with \${VAR:+} so we do not clobber
+              # the image's nixl/ucx/cuda entries or create a leading empty
+              # ld.so entry. Harmless no-op when the path is absent (EKS/AKS
+              # GPU Operator toolkit). \$ so the unquoted heredoc does not
+              # expand LD_LIBRARY_PATH at render time.
+              command:
+                - /bin/bash
+                - -c
+                - export LD_LIBRARY_PATH="\${LD_LIBRARY_PATH:+\${LD_LIBRARY_PATH}:}/usr/local/nvidia/lib64"; exec python3 -m dynamo.vllm "\$@"
+                - dynamo.vllm
               args:
                 - --model
                 - ${SERVE_MODEL}
@@ -1402,9 +1424,12 @@ phase_serve() {
   # DynamoGraphDeployment in demos/cuj2-inference.md
   # (demos/workloads/inference/vllm-agg.yaml): the KAI queue and a
   # two-component (Frontend + decode Worker) graph serving an OpenAI-compatible
-  # endpoint. Frontend placement intentionally diverges — the demo pins
-  # nodeGroup=cpu-worker; this graph selects the GPU pool for both components
-  # (pool-selection note below, #1644). The worker requests its GPU as a scalar
+  # endpoint. Two intentional divergences from the demo: Frontend placement
+  # (the demo pins nodeGroup=cpu-worker; this graph selects the GPU pool for
+  # both components — pool-selection note below, #1644), and the worker
+  # command (the demo uses python3 -m dynamo.vllm; this graph wraps it with
+  # the GKE driver-lib append used by inference-perf, or vLLM cannot see
+  # libcuda.so.1 on gke-default). The worker requests its GPU as a scalar
   # nvidia.com/gpu limit — the device-plugin production default (#1327).
   #
   # Tolerations are a portable SUPERSET of the taints across all UAT clusters
@@ -1424,7 +1449,9 @@ phase_serve() {
   # no device: the Frontend declares no nvidia.com/gpu limit, so the device
   # plugin never allocates one to it. This matches how the inference-perf
   # validator places every component on the GPU cohort
-  # (validators/performance/inference_perf_constraint.go).
+  # (validators/performance/inference_perf_constraint.go). The worker command
+  # is the same GKE driver-lib append as that validator's Dynamo templates;
+  # without it, vLLM crash-loops on gke-default (run 32732018329).
   #
   # This selects the POOL, not a node: the pool holds two GPU nodes and nothing
   # constrains the two components to the same one, so they may be split. That is

tests/uat/serve_manifest_test.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -164,6 +165,39 @@ func TestServeGraphSelectorIsOverridable(t *testing.T) {
 	}
 }
 
+// TestServeGraphWorkerDriverLibPathAppend pins the GKE driver-lib wrapper on
+// the decode worker. GKE's managed device plugin mounts /usr/local/nvidia
+// without setting LD_LIBRARY_PATH; a bare `python3 -m dynamo.vllm` then
+// crash-loops before it binds its health port (UAT run 32732018329). The
+// shape matches validators/performance testdata Dynamo templates: a shell
+// APPEND with ${VAR:+}, argv0 consumed by the trailing "dynamo.vllm" element
+// so --model stays args[0]. A revert to ["python3", "-m", "dynamo.vllm"]
+// must fail here.
+func TestServeGraphWorkerDriverLibPathAppend(t *testing.T) {
+	graph := renderServeGraph(t, nil)
+	worker := graph.component(t, workerComponent)
+
+	wantCommand := []string{
+		"/bin/bash",
+		"-c",
+		`export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}/usr/local/nvidia/lib64"; exec python3 -m dynamo.vllm "$@"`,
+		"dynamo.vllm",
+	}
+	for _, c := range worker.PodTemplate.Spec.Containers {
+		if c.Name != "main" {
+			continue
+		}
+		if !reflect.DeepEqual(c.Command, wantCommand) {
+			t.Errorf("worker main command = %#v, want %#v", c.Command, wantCommand)
+		}
+		if len(c.Args) == 0 || c.Args[0] != "--model" {
+			t.Errorf("worker main args = %v, want first element %q", c.Args, "--model")
+		}
+		return
+	}
+	t.Fatal("main container not found in VllmDecodeWorker")
+}
+
 // renderServeGraph sources phases.sh, runs serve_render_manifest, and returns
 // the DynamoGraphDeployment document. env entries (KEY=VALUE) override the
 // script's defaults. It also asserts the render produced the Queue and Namespace
@@ -279,7 +313,9 @@ type dynamoComponent struct {
 			NodeSelector map[string]string `yaml:"nodeSelector"`
 			Tolerations  []toleration      `yaml:"tolerations"`
 			Containers   []struct {
-				Name      string `yaml:"name"`
+				Name      string   `yaml:"name"`
+				Command   []string `yaml:"command"`
+				Args      []string `yaml:"args"`
 				Resources struct {
 					// Quantities are `any`: YAML renders `nvidia.com/gpu: 1` as an
 					// int, while a chart-style "1" would be a string. Comparison is