test(recipe): classify strict failures from context, not message text

Signed-off-by: Brian Lockwood <lockwobr@gmail.com>

Author: lockwobr

pkg/recipe/coverage.go

@@ -482,12 +482,11 @@ func (s *MetadataStore) strictDimensionGaps(criteria *Criteria, applied []string
 		for _, value := range s.dimensionValues(dim) {
 			probe := *criteria
 			setCriteriaDimension(&probe, dim.name, value)
-			for _, match := range s.FindMatchingOverlays(&probe) {
-				for _, name := range s.inheritanceChainNames(match) {
-					if _, already := appliedSet[name]; !already {
-						reachable[value] = struct{}{}
-					}
-				}
+			// One overlay outside the applied set is enough to establish that
+			// this value reaches something currently being skipped; the rest
+			// of the matches and their chains cannot change the answer.
+			if s.reachesUnappliedOverlay(&probe, appliedSet) {
+				reachable[value] = struct{}{}
 			}
 		}
 		if len(reachable) == 0 {
@@ -506,6 +505,19 @@ func (s *MetadataStore) strictDimensionGaps(criteria *Criteria, applied []string
 	return gaps
 }
 
+// reachesUnappliedOverlay reports whether resolving probe would pull in any
+// overlay that is not already applied.
+func (s *MetadataStore) reachesUnappliedOverlay(probe *Criteria, applied map[string]struct{}) bool {
+	for _, match := range s.FindMatchingOverlays(probe) {
+		for _, name := range s.inheritanceChainNames(match) {
+			if _, already := applied[name]; !already {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // dimensionValues returns every value the catalog declares for a dimension,
 // sorted. Iteration order of s.Overlays is randomized, so the sort is what
 // makes strictDimensionGaps deterministic.

pkg/recipe/coverage_matrix_test.go

@@ -19,7 +19,6 @@ import (
 	stderrors "errors"
 	"os"
 	"sort"
-	"strings"
 	"testing"
 
 	aicrerrors "github.com/NVIDIA/aicr/pkg/errors"
@@ -30,9 +29,13 @@ const goldenPath = "testdata/coverage_golden.yaml"
 
 // coverageClassification is the golden outcome for one projected query.
 type coverageClassification struct {
-	Outcome    string   `yaml:"outcome"`              // "success" | "error"
-	Uncovered  []string `yaml:"uncovered,omitempty"`  // coverage errors
-	RequiresOS bool     `yaml:"requiresOS,omitempty"` // requireOSIfNeeded guard errors
+	Outcome   string   `yaml:"outcome"`             // "success" | "error"
+	Uncovered []string `yaml:"uncovered,omitempty"` // completeness failures
+	// StrictDimensions names the dimensions a joint-sufficiency failure
+	// demanded, read from the error's structured context rather than its
+	// message text so rewording the message cannot silently reclassify a
+	// projection and retire the golden matrix's guard on the rule.
+	StrictDimensions []string `yaml:"strictDimensions,omitempty"`
 	// ValidCompletions pins the completion-suggestion content per uncovered
 	// dimension (canonical tupleKey strings, in minimalTuples order) so a
 	// regression in the suggestion machinery on the real catalog flips a
@@ -112,7 +115,8 @@ func TestCoverageGoldenMatrix(t *testing.T) {
 			continue
 		}
 		if w.Outcome != got[k].Outcome || !equalStrings(w.Uncovered, got[k].Uncovered) ||
-			w.RequiresOS != got[k].RequiresOS || !equalCompletions(w.ValidCompletions, got[k].ValidCompletions) {
+			!equalStrings(w.StrictDimensions, got[k].StrictDimensions) ||
+			!equalCompletions(w.ValidCompletions, got[k].ValidCompletions) {
 
 			t.Errorf("projection %q flipped: golden %+v, now %+v", k, w, got[k])
 		}
@@ -151,10 +155,8 @@ func classify(ctx context.Context, t *testing.T, store *MetadataStore, q *Criter
 	if err == nil {
 		return coverageClassification{Outcome: "success"}
 	}
-	msg := err.Error()
-	if strings.Contains(msg, "; specify ") {
-		// Joint-sufficiency failure (formerly the requireOSIfNeeded guard).
-		return coverageClassification{Outcome: "error", RequiresOS: true}
+	if strict := strictDimensionsFromError(err); len(strict) > 0 {
+		return coverageClassification{Outcome: "error", StrictDimensions: strict}
 	}
 	uncovered, completions := coverageDetailsFromError(err)
 	if len(uncovered) == 0 {
@@ -163,6 +165,28 @@ func classify(ctx context.Context, t *testing.T, store *MetadataStore, q *Criter
 	return coverageClassification{Outcome: "error", Uncovered: uncovered, ValidCompletions: completions}
 }
 
+// strictDimensionsFromError extracts the dimension names a joint-sufficiency
+// failure demanded, from the error's `strictDimensions` context, or nil when
+// err is not one. Reading the structured context rather than the message keeps
+// the golden matrix pinned to the rule instead of to its phrasing.
+func strictDimensionsFromError(err error) []string {
+	var se *aicrerrors.StructuredError
+	if !stderrors.As(err, &se) || se.Context == nil {
+		return nil
+	}
+	entries, ok := se.Context["strictDimensions"].([]map[string]any)
+	if !ok {
+		return nil
+	}
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if name, ok := entry["dimension"].(string); ok {
+			names = append(names, name)
+		}
+	}
+	return names
+}
+
 // coverageDetailsFromError extracts the uncovered dimension names and their
 // completion suggestions (as canonical tupleKey strings, preserving
 // minimalTuples order) from a StructuredError's "uncovered" context entries,

pkg/recipe/coverage_subsumption_test.go

@@ -124,7 +124,7 @@ func TestJointSufficiencySubsumesRetiredGuard(t *testing.T) {
 	base := &RecipeMetadata{}
 	base.Metadata.Name = testRecipeBase
 
-	checked, guardFired := 0, 0
+	guardFired := 0
 	// Catalogs of three overlays drawn from the shape space. Three is enough
 	// to express the split-coverage catalog (service alone, accelerator alone,
 	// both plus an os) that the guard was written for.
@@ -147,7 +147,6 @@ func TestJointSufficiencySubsumesRetiredGuard(t *testing.T) {
 						continue
 					}
 					guardFired++
-					checked++
 					if _, err := store.BuildRecipeResult(ctx, &query); err == nil {
 						t.Fatalf("SUBSUMPTION VIOLATED: retired guard rejected %s but resolution succeeded\n"+
 							"  catalog: %s | %s | %s",
@@ -160,5 +159,5 @@ func TestJointSufficiencySubsumesRetiredGuard(t *testing.T) {
 	if guardFired == 0 {
 		t.Fatal("generated no catalog where the retired guard fires; the test proves nothing")
 	}
-	t.Logf("subsumption held: retired guard fired on %d (catalog, query) pairs, all still rejected", checked)
+	t.Logf("subsumption held: retired guard fired on %d (catalog, query) pairs, all still rejected", guardFired)
 }