fix(evidence): gate verdicts on cleanup and stop over-claiming (NVIDIA#2222)

Signed-off-by: Yuan Chen <yuanchen97@gmail.com>

Author: yuanchen8911

pkg/evidence/cncf/result_propagation_test.go

@@ -248,6 +248,101 @@ case "${mode}" in
     autoscaler-absent)
         SECTION="cluster-autoscaling"
         ;;
+    dynamo-dispatch-list-failed)
+        # Drives the REAL collect_service_metrics dispatcher (no override): the
+        # Dynamo pod list fails while the namespace exists. The dispatcher must
+        # route to the Dynamo collector so the failure surfaces, rather than
+        # falling through to NIM/trainer and emitting unrelated evidence.
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            case " $* " in
+                *"dynamo-component-type=worker"*) return 1 ;;
+                *" get namespace dynamo-workload "*) return 0 ;;
+            esac
+            return 0
+        }
+        SECTION="service-metrics"
+        ;;
+    dynamo-dispatch-both-queries-failed)
+        # Both Dynamo probes fail (cluster-wide read failure: RBAC revoked,
+        # expired credentials, apiserver outage). The namespace state cannot be
+        # classified, so the dispatcher must NOT read that as "no workload" and
+        # fall through to NIM/trainer — it routes to Dynamo, which fails closed.
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            case " $* " in
+                *"dynamo-component-type=worker"*) return 1 ;;
+                *" get namespace dynamo-workload "*)
+                    echo "Error from server (Forbidden): namespaces \"dynamo-workload\" is forbidden" >&2
+                    return 1 ;;
+            esac
+            return 0
+        }
+        SECTION="service-metrics"
+        ;;
+    dynamo-workload-absent)
+        # No worker pods: the section MEASURES an existing workload and must
+        # never deploy one, so an absent workload is an absent prerequisite
+        # (SKIP), not a cue to apply the manifest.
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            return 0
+        }
+        collect_service_metrics() {
+            EVIDENCE_FILE="${EVIDENCE_DIR}/ai-service-metrics.md"
+            collect_service_metrics_dynamo
+        }
+        SECTION="service-metrics"
+        ;;
+    dynamo-workload-list-failed)
+        # The pod list fails: a read error must fail closed rather than be
+        # flattened into "no workload present".
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            case " $* " in
+                *"dynamo-component-type=worker"*) return 1 ;;
+            esac
+            return 0
+        }
+        collect_service_metrics() {
+            EVIDENCE_FILE="${EVIDENCE_DIR}/ai-service-metrics.md"
+            collect_service_metrics_dynamo
+        }
+        SECTION="service-metrics"
+        ;;
+    hpa-scaled-cleanup-ok|hpa-scaled-cleanup-failed)
+        # Both lanes observe a real scale-up (replicas>1 with a current metric),
+        # so hpa_scaled is true and the ONLY difference is whether the test
+        # namespace could be deleted. The section must not report PASS while its
+        # unbounded CUDA workload may still be running, so the cleanup lane has
+        # to flip the verdict to FAIL — and the control lane proves the FAIL is
+        # attributable to cleanup rather than to a stub that never passes.
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            case " $* " in
+                *"averageValue"*) echo "75" ;;
+                *"currentReplicas"*) echo "2" ;;
+            esac
+            return 0
+        }
+        sleep() { return 0; }
+        if [ "${mode}" = "hpa-scaled-cleanup-failed" ]; then
+            cleanup_ns() { return 1; }
+        else
+            cleanup_ns() { return 0; }
+        fi
+        SECTION="hpa"
+        ;;
     dynamo-unhealthy)
         kubectl() {
             if [ "${1:-}" = "cluster-info" ]; then
@@ -526,6 +621,58 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 			wantStatus:     "FAIL",
 			wantErr:        true,
 		},
+		{
+			name:              "dispatcher routes a failed Dynamo list to the Dynamo collector",
+			mode:              "dynamo-dispatch-list-failed",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			wantEvidence:      "the workload state is unknown",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
+		},
+		{
+			name:              "unclassifiable Dynamo namespace state fails closed",
+			mode:              "dynamo-dispatch-both-queries-failed",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			wantEvidence:      "the workload state is unknown",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
+		},
+		{
+			name:              "absent Dynamo workload skips instead of deploying",
+			mode:              "dynamo-workload-absent",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "SKIP",
+			wantEvidence:      "no running Dynamo workload in dynamo-workload",
+			singleVerdictFile: "ai-service-metrics.md",
+		},
+		{
+			name:              "Dynamo worker pod list failure fails closed",
+			mode:              "dynamo-workload-list-failed",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			wantEvidence:      "the workload state is unknown",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
+		},
+		{
+			name:              "HPA scale-up with successful cleanup is pass",
+			mode:              "hpa-scaled-cleanup-ok",
+			displayName:       "Pod Autoscaling (HPA)",
+			wantStatus:        "PASS",
+			singleVerdictFile: "pod-autoscaling.md",
+			wantErr:           false,
+		},
+		{
+			name:              "HPA scale-up with failed cleanup is fail",
+			mode:              "hpa-scaled-cleanup-failed",
+			displayName:       "Pod Autoscaling (HPA)",
+			wantStatus:        "FAIL",
+			wantEvidence:      "the hpa-test namespace could not be deleted",
+			singleVerdictFile: "pod-autoscaling.md",
+			wantErr:           true,
+		},
 		{
 			name:              "present but unhealthy Dynamo is fail",
 			mode:              "dynamo-unhealthy",
@@ -598,6 +745,16 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 					t.Fatalf("write trainer manifest fixture: %v", err)
 				}
 			}
+			if strings.HasPrefix(tt.mode, "hpa-") {
+				manifestDir := filepath.Join(dir, "manifests")
+				if err := os.MkdirAll(manifestDir, 0o755); err != nil {
+					t.Fatalf("create manifest directory: %v", err)
+				}
+				manifestPath := filepath.Join(manifestDir, "hpa-gpu-test.yaml")
+				if err := os.WriteFile(manifestPath, []byte("---\n"), 0o600); err != nil {
+					t.Fatalf("write HPA manifest fixture: %v", err)
+				}
+			}
 			if strings.HasPrefix(tt.mode, "gang-") {
 				manifestDir := filepath.Join(dir, "manifests")
 				if err := os.MkdirAll(manifestDir, 0o755); err != nil {
@@ -652,7 +809,13 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 				t.Errorf("result = %q, want %q", got, want)
 			}
 			if tt.wantEvidence != "" {
-				evidencePath := filepath.Join(outputDir, "ai-service-metrics.md")
+				// Assert against the artifact the case names, falling back to
+				// the service-metrics file the original lanes all used.
+				evidenceName := tt.singleVerdictFile
+				if evidenceName == "" {
+					evidenceName = "ai-service-metrics.md"
+				}
+				evidencePath := filepath.Join(outputDir, evidenceName)
 				evidence, evidenceErr := os.ReadFile(evidencePath)
 				if evidenceErr != nil {
 					t.Fatalf("read evidence artifact: %v", evidenceErr)