test(uat): tighten serve GPU claim assert and clarify rollout comments

Count nvidia.com/gpu limits across worker containers so two containers
each claiming 1 GPU cannot pass as "exactly one". Drop the AWS/Azure
"lockstep with uat-gcp" wording now that only GCP runs the serve step.

Signed-off-by: Rohit Rajani <rorajani@nvidia.com>

Author: rorajani

.github/workflows/uat-aws.yaml

@@ -813,18 +813,15 @@ jobs:
           RUN_ID: ${{ github.run_id }}
         run: ./tests/uat/aws/run train "${TEST_CONFIG}"
 
-      # STILL DISABLED (step commented out below), but the root cause is now
-      # addressed: phase_serve selects the GPU pool for both graph components, so
-      # the Frontend no longer cold-pulls the ~12GB vllm-runtime image on a
-      # small CPU-pool node and wedges in ContainerCreating past the 30m
-      # readiness budget. uat-gcp.yaml re-enables its step first (#1644 tracks
-      # the GKE x inference cell); this one stays commented until that lane
-      # demonstrates a served completion — re-enable then, and restore the Serve
-      # summary row below to `${{ steps.serve.outcome }}`. Until then the
-      # inference STACK (dynamo + KAI scheduler + DRA driver) is still stood up
-      # and validated by prep/install/conformance/verify; only the
-      # served-workload run is skipped (the Serve summary row is a static
-      # `disabled`). Kept in lockstep with uat-gcp.yaml. Tracked by #1644.
+      # STILL DISABLED (step commented out below). phase_serve already selects the
+      # GPU pool for both graph components (#1644), so the Frontend no longer
+      # cold-pulls the ~12GB vllm-runtime image on a small CPU-pool node. Only
+      # uat-gcp.yaml re-enables the serve *step* for now (GKE x inference is what
+      # #1644 tracks); re-enable this one after that lane has a served completion,
+      # and restore the Serve summary row below to `${{ steps.serve.outcome }}`.
+      # Until then the inference STACK (dynamo + KAI scheduler + DRA driver) is
+      # still stood up and validated by prep/install/conformance/verify; only the
+      # served-workload run is skipped (Serve summary row is a static `disabled`).
       #
       # - name: UAT - serve (DynamoGraphDeployment + endpoint)
       #   id: serve

.github/workflows/uat-azure.yaml

@@ -797,18 +797,15 @@ jobs:
           RUN_ID: ${{ github.run_id }}
         run: ./tests/uat/azure/run train "${TEST_CONFIG}"
 
-      # STILL DISABLED (step commented out below), but the root cause is now
-      # addressed: phase_serve selects the GPU pool for both graph components, so
-      # the Frontend no longer cold-pulls the ~12GB vllm-runtime image on a
-      # small CPU-pool node and wedges in ContainerCreating past the 30m
-      # readiness budget. uat-gcp.yaml re-enables its step first (#1644 tracks
-      # the GKE x inference cell); this one stays commented until that lane
-      # demonstrates a served completion — re-enable then, and restore the Serve
-      # summary row below to `${{ steps.serve.outcome }}`. Until then the
-      # inference STACK (dynamo + KAI scheduler + DRA driver) is still stood up
-      # and validated by prep/install/conformance/verify; only the
-      # served-workload run is skipped (the Serve summary row is a static
-      # `disabled`). Kept in lockstep with uat-gcp.yaml. Tracked by #1644.
+      # STILL DISABLED (step commented out below). phase_serve already selects the
+      # GPU pool for both graph components (#1644), so the Frontend no longer
+      # cold-pulls the ~12GB vllm-runtime image on a small CPU-pool node. Only
+      # uat-gcp.yaml re-enables the serve *step* for now (GKE x inference is what
+      # #1644 tracks); re-enable this one after that lane has a served completion,
+      # and restore the Serve summary row below to `${{ steps.serve.outcome }}`.
+      # Until then the inference STACK (dynamo + KAI scheduler + DRA driver) is
+      # still stood up and validated by prep/install/conformance/verify; only the
+      # served-workload run is skipped (Serve summary row is a static `disabled`).
       #
       # - name: UAT - serve (DynamoGraphDeployment + endpoint)
       #   id: serve

.github/workflows/uat-gcp.yaml

@@ -694,13 +694,12 @@ jobs:
           RUN_ID: ${{ github.run_id }}
         run: ./tests/uat/gcp/run train "${TEST_CONFIG}"
 
-      # RE-ENABLED (#1644). It was disabled because the Frontend cold-pulled the
-      # ~12GB vllm-runtime image on a CPU-pool node and wedged in
-      # ContainerCreating past the readiness budget; phase_serve now selects the
-      # GPU pool for both graph components, whose a3-megagpu-8g class pulls that
-      # image in a fraction of the budget. GCP goes first because #1644 tracks
-      # the GKE x inference cell specifically; aws/azure stay commented until
-      # this lane demonstrates a served completion.
+      # Live in the GCP inference phase chain (#1644). Was disabled because the
+      # Frontend cold-pulled the ~12GB vllm-runtime image on a CPU-pool node and
+      # wedged in ContainerCreating past the readiness budget; phase_serve now
+      # selects the GPU pool for both graph components, whose a3-megagpu-8g class
+      # pulls that image in a fraction of the budget. AWS/Azure serve steps stay
+      # commented until this lane demonstrates a served completion.
       - name: UAT - serve (DynamoGraphDeployment + endpoint)
         id: serve
         if: steps.conformance.outcome == 'success' && inputs.intent == 'inference'

tests/uat/serve_manifest_test.go

@@ -124,19 +124,22 @@ func TestServeGraphGPUPlacement(t *testing.T) {
 	})
 
 	// Guards the blast radius of the pin: the worker's GPU request is the reason
-	// the pool exists, and the placement edit sits directly above it.
+	// the pool exists, and the placement edit sits directly above it. Count
+	// claims across containers — two containers each declaring nvidia.com/gpu: 1
+	// would request two GPUs, not "exactly one".
 	t.Run("worker still requests exactly one GPU", func(t *testing.T) {
-		var found bool
+		claims := 0
 		for _, c := range worker.PodTemplate.Spec.Containers {
 			if q, ok := c.Resources.Limits[gpuResource]; ok {
-				found = true
+				claims++
 				if fmt.Sprint(q) != "1" {
 					t.Errorf("worker container %q %s limit = %v, want 1", c.Name, gpuResource, q)
 				}
 			}
 		}
-		if !found {
-			t.Errorf("no worker container requests %s; the served graph would run on CPU", gpuResource)
+		if claims != 1 {
+			t.Errorf("worker declares %d %s limit(s) across containers, want exactly 1",
+				claims, gpuResource)
 		}
 	})
 }