fix(evidence): SKIP absent Dynamo workload instead of failing submission

The robust-operator section recorded Result: FAIL when the Dynamo operator
was present but no DynamoGraphDeployment existed, collapsing a failed API
query and a successful zero-row query into the same 0. With section FAILs
now propagating to a nonzero exit, a stock Dynamo inference bundle +
--cncf-submission failed on the default path: the service-metrics section
deploys and cleans up its own DGD before the operator section runs, so the
operator section saw zero DGDs.

Distinguish a failed DGD query (fail closed) from a successful query with
zero rows (absent prerequisite -> SKIP), matching the SKIP treatment given
to absent gateway/operator/autoscaler prerequisites. Add a subprocess test
for the new branch and assert each section collector emits exactly one
column-zero verdict.

Fixes the cross-review finding on PR #1730.

Signed-off-by: Yuan Chen <yuanchen97@gmail.com>

Author: yuanchen8911

pkg/evidence/cncf/result_propagation_test.go

@@ -64,6 +64,22 @@ case "${mode}" in
     operator-absent)
         SECTION="operator"
         ;;
+    operator-dynamo-no-dgd)
+        # Dynamo operator is installed but no DynamoGraphDeployment exists and the
+        # DGD query itself succeeds: an absent inference workload is an absent
+        # prerequisite (SKIP), not a failure.
+        kubectl() {
+            if [ "${1:-}" = "cluster-info" ]; then
+                return 0
+            fi
+            case " $* " in
+                *"dynamo-platform-dynamo-operator-controller-manager"*) echo "dynamo-operator 1/1" ;;
+                *" dynamographdeployments "*) return 0 ;;
+            esac
+            return 0
+        }
+        SECTION="operator"
+        ;;
     autoscaler-absent)
         SECTION="cluster-autoscaling"
         ;;
@@ -170,7 +186,13 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 		staleOutput    bool
 		wantStatus     string
 		wantEvidence   string
-		wantErr        bool
+		// singleVerdictFile, when set, is an evidence artifact (relative to the
+		// evidence dir) that must contain exactly one column-zero "**Result:"
+		// verdict line — the invariant evidence_result relies on. Only set for
+		// section collectors that emit their own verdict, not for injected
+		// fixtures that deliberately carry zero or multiple verdicts.
+		singleVerdictFile string
+		wantErr           bool
 	}{
 		{
 			name:         "pass",
@@ -191,22 +213,32 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 			wantStatus:   "SKIP",
 		},
 		{
-			name:        "absent gateway emits explicit skip",
-			mode:        "gateway-absent",
-			displayName: "Inference Gateway",
-			wantStatus:  "SKIP",
+			name:              "absent gateway emits explicit skip",
+			mode:              "gateway-absent",
+			displayName:       "Inference Gateway",
+			wantStatus:        "SKIP",
+			singleVerdictFile: "inference-gateway.md",
 		},
 		{
-			name:        "absent operator emits explicit skip",
-			mode:        "operator-absent",
-			displayName: "Robust AI Operator",
-			wantStatus:  "SKIP",
+			name:              "absent operator emits explicit skip",
+			mode:              "operator-absent",
+			displayName:       "Robust AI Operator",
+			wantStatus:        "SKIP",
+			singleVerdictFile: "robust-operator.md",
 		},
 		{
-			name:        "absent autoscaler emits explicit skip",
-			mode:        "autoscaler-absent",
-			displayName: "Cluster Autoscaling",
-			wantStatus:  "SKIP",
+			name:              "present operator without DynamoGraphDeployment skips",
+			mode:              "operator-dynamo-no-dgd",
+			displayName:       "Robust AI Operator",
+			wantStatus:        "SKIP",
+			singleVerdictFile: "robust-operator.md",
+		},
+		{
+			name:              "absent autoscaler emits explicit skip",
+			mode:              "autoscaler-absent",
+			displayName:       "Cluster Autoscaling",
+			wantStatus:        "SKIP",
+			singleVerdictFile: "cluster-autoscaling.md",
 		},
 		{
 			name: "fail is not masked by overview prose",
@@ -273,34 +305,38 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 			wantErr:        true,
 		},
 		{
-			name:        "present but unhealthy Dynamo is fail",
-			mode:        "dynamo-unhealthy",
-			displayName: "AI Service Metrics",
-			wantStatus:  "FAIL",
-			wantErr:     true,
+			name:              "present but unhealthy Dynamo is fail",
+			mode:              "dynamo-unhealthy",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
 		},
 		{
-			name:        "present but unhealthy NIM is fail",
-			mode:        "nim-unhealthy",
-			displayName: "AI Service Metrics",
-			wantStatus:  "FAIL",
-			wantErr:     true,
+			name:              "present but unhealthy NIM is fail",
+			mode:              "nim-unhealthy",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
 		},
 		{
-			name:         "Dynamo Prometheus connection failure is explicit fail",
-			mode:         "dynamo-prometheus-unavailable",
-			displayName:  "AI Service Metrics",
-			wantStatus:   "FAIL",
-			wantEvidence: "**Result: FAIL** — Could not connect to Prometheus.",
-			wantErr:      true,
+			name:              "Dynamo Prometheus connection failure is explicit fail",
+			mode:              "dynamo-prometheus-unavailable",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			wantEvidence:      "**Result: FAIL** — Could not connect to Prometheus.",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
 		},
 		{
-			name:         "trainer Prometheus connection failure is explicit fail",
-			mode:         "trainer-prometheus-unavailable",
-			displayName:  "AI Service Metrics",
-			wantStatus:   "FAIL",
-			wantEvidence: "**Result: FAIL** — Could not connect to Prometheus.",
-			wantErr:      true,
+			name:              "trainer Prometheus connection failure is explicit fail",
+			mode:              "trainer-prometheus-unavailable",
+			displayName:       "AI Service Metrics",
+			wantStatus:        "FAIL",
+			wantEvidence:      "**Result: FAIL** — Could not connect to Prometheus.",
+			singleVerdictFile: "ai-service-metrics.md",
+			wantErr:           true,
 		},
 	}
 
@@ -393,6 +429,30 @@ func TestEvidenceResultPropagatesThroughCollector(t *testing.T) {
 					t.Errorf("evidence does not contain %q", tt.wantEvidence)
 				}
 			}
+			if tt.singleVerdictFile != "" {
+				evidencePath := filepath.Join(outputDir, tt.singleVerdictFile)
+				evidence, evidenceErr := os.ReadFile(evidencePath)
+				if evidenceErr != nil {
+					t.Fatalf("read evidence artifact %s: %v", tt.singleVerdictFile, evidenceErr)
+				}
+				if got := countColumnZeroVerdicts(string(evidence)); got != 1 {
+					t.Errorf("evidence %s has %d column-zero **Result: verdicts, want exactly 1", tt.singleVerdictFile, got)
+				}
+			}
 		})
 	}
 }
+
+// countColumnZeroVerdicts counts lines that begin at column zero with the
+// "**Result:" verdict marker — the same anchoring evidence_result() uses to
+// ignore numbered overview prose. Section collectors must emit exactly one so
+// the parser never fails closed on a healthy check.
+func countColumnZeroVerdicts(evidence string) int {
+	count := 0
+	for _, line := range strings.Split(evidence, "\n") {
+		if strings.HasPrefix(line, "**Result:") {
+			count++
+		}
+	}
+	return count
+}

pkg/evidence/cncf/scripts/collect-evidence.sh

@@ -2470,23 +2470,33 @@ INVALID_CR
         echo "WARNING: Webhook did not reject the invalid resource." >> "${EVIDENCE_FILE}"
     fi
 
-    # Verdict — require DGD + healthy workload pods; webhook rejection strengthens but is optional
+    # Verdict — require DGD + healthy workload pods; webhook rejection strengthens but is optional.
+    # Distinguish a failed DGD query (fail closed) from a successful query returning
+    # zero rows: an absent inference workload is an absent prerequisite (SKIP), not a
+    # failure, because a stock inference recipe deploys the operator but no persistent
+    # DynamoGraphDeployment (the service-metrics section deploys and cleans up its own).
     echo "" >> "${EVIDENCE_FILE}"
-    local dgd_count
-    dgd_count=$(kubectl get dynamographdeployments -A --no-headers 2>/dev/null | wc -l | tr -d ' ')
+    local dgd_query_ok=true dgd_count=0 dgd_out
+    if dgd_out=$(kubectl get dynamographdeployments -A --no-headers 2>/dev/null); then
+        dgd_count=$(printf '%s' "${dgd_out}" | grep -c . || true)
+    else
+        dgd_query_ok=false
+    fi
     local running_pods
     running_pods=$(kubectl get pods -n dynamo-workload -l nvidia.com/dynamo-graph-deployment-name --no-headers 2>/dev/null | grep -c "Running" || true)
     local webhook_ok
     webhook_ok=$(echo "${webhook_result}" | grep -ci "denied\|forbidden\|invalid\|error" || true)
 
-    if [ "${dgd_count}" -gt 0 ] && [ "${running_pods}" -gt 0 ] && [ "${webhook_ok}" -gt 0 ]; then
+    if [ "${dgd_query_ok}" != "true" ]; then
+        echo "**Result: FAIL** — DynamoGraphDeployment query failed (fail closed); rerun against a reachable API server." >> "${EVIDENCE_FILE}"
+    elif [ "${dgd_count}" -gt 0 ] && [ "${running_pods}" -gt 0 ] && [ "${webhook_ok}" -gt 0 ]; then
         echo "**Result: PASS** — Dynamo operator running, webhooks operational (rejection verified), CRDs registered, DynamoGraphDeployment reconciled with ${running_pods} healthy workload pod(s)." >> "${EVIDENCE_FILE}"
     elif [ "${dgd_count}" -gt 0 ] && [ "${running_pods}" -gt 0 ]; then
         echo "**Result: PASS** — Dynamo operator running, CRDs registered, DynamoGraphDeployment reconciled with ${running_pods} healthy workload pod(s)." >> "${EVIDENCE_FILE}"
     elif [ "${dgd_count}" -gt 0 ]; then
         echo "**Result: FAIL** — DynamoGraphDeployment found but no healthy workload pods." >> "${EVIDENCE_FILE}"
     else
-        echo "**Result: FAIL** — No DynamoGraphDeployment found." >> "${EVIDENCE_FILE}"
+        echo "**Result: SKIP (prerequisite absent)** — Dynamo operator present but no DynamoGraphDeployment exists; deploy an inference workload (e.g. demos/workloads/inference/vllm-agg.yaml) to exercise operator reconciliation." >> "${EVIDENCE_FILE}"
     fi
 
     log_info "Robust operator evidence collection complete."