test(infra): e2e also creates an instance of each template

The template e2e proved only that kro ACCEPTS each authored RGD
(GraphAccepted=True). Extend it to also CREATE a sample instance per
template and assert kro reconciles it WITHOUT an apply error.

This is the exact failure that motivated the schema-default image
convention ("apply results contain errors: ... image: Required value"):
RGD acceptance alone never exercised it, because the empty-image fault
only surfaces when kro applies a child Deployment. The test now catches
it.

Per template: build an instance from sampleValues when present, else a
minimal valid spec (e2eMinimalSpecs); retry Create until the fresh CRD
is servable; poll status conditions and fail on any apply-error marker
(apply results contain errors / is invalid / Required value / failed to
apply / admission webhook). Readiness is out of scope — images need not
pull in CI; apply validates the spec.

Validated end-to-end on a fresh standalone kro kind cluster (ok 20.7s,
all five templates).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: mjudeikis

providers/infrastructure/backend/kro/e2e_test.go

@@ -16,42 +16,69 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// E2E coverage for the seeded infrastructure Templates against a real kro
-// cluster. Build-tagged `e2e` so it never runs in the normal `go test ./...`
-// unit pass — it needs a running kro controller (see `make e2e-infrastructure`).
+// E2E coverage for the seeded infrastructure Templates against a real, CLEAN kro
+// cluster (one fresh cluster per run — see `make e2e-infrastructure`). Build-
+// tagged `e2e` so it never runs in the normal `go test ./...` unit pass.
 //
-// What it proves: every Template the provider ships authors a kro
-// ResourceGraphDefinition that kro ACCEPTS (status GraphAccepted=True). This is
-// the exact class of failure that unit tests (which only check buildRGD output)
-// miss — e.g. an integer field fed a string, an includeWhen that isn't a
-// standalone CEL expression, or a container image that resolved to "". kro's
-// graph/schema validation is independent of its watch source, so this works
-// against any kro cluster (standalone kind in CI, or a kcp-wired dev cluster).
+// For every Template the provider ships it proves two things unit tests can't
+// (they only check buildRGD's output):
+//
+//  1. kro ACCEPTS the authored ResourceGraphDefinition (status GraphAccepted=
+//     True) and establishes the per-template instance CRD. Catches malformed
+//     graphs — an integer field fed a string, an includeWhen that isn't a
+//     standalone CEL expression, etc.
+//  2. kro can CREATE an instance's objects: a sample instance reconciles WITHOUT
+//     an apply error. This is the exact failure that motivated the schema-default
+//     image convention ("apply results contain errors: ... image: Required
+//     value"). It does NOT require the images to pull — apply validates the spec.
 package kro
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/tools/clientcmd"
+
+	infrav1alpha1 "github.com/faroshq/provider-infrastructure/apis/v1alpha1"
 )
 
 const (
-	e2eRGDPrefix  = "e2e-"
-	e2eAcceptWait = 90 * time.Second
-	e2ePollEvery  = 2 * time.Second
+	e2eGraphWait    = 90 * time.Second
+	e2eCRDWait      = 60 * time.Second
+	e2eInstanceWait = 120 * time.Second
+	e2ePollEvery    = 2 * time.Second
 )
 
-// TestE2ESeedTemplatesAcceptedByKro builds each seeded Template's RGD with the
-// real buildRGD path and asserts kro accepts the graph.
-func TestE2ESeedTemplatesAcceptedByKro(t *testing.T) {
+// e2eMinimalSpecs supplies a valid instance spec for templates that ship no
+// sampleValues (those that do — application, database — use them directly).
+var e2eMinimalSpecs = map[string]map[string]any{
+	"redis-cache":    {"name": "e2e-redis"},
+	"simple-webapp":  {"name": "e2e-web"},
+	"sandbox-runner": {"name": "kedge-sandbox-0000111122223333", "projectRef": "e2e"},
+}
+
+// e2eApplyErrorMarkers are substrings kro puts in an instance condition when it
+// FAILS to apply a child resource (the bug class we guard against). Readiness
+// waits (pods not up because an image can't pull in CI) do not contain these.
+var e2eApplyErrorMarkers = []string{
+	"apply results contain errors",
+	"is invalid",
+	"Required value",
+	"failed to apply",
+	"admission webhook",
+}
+
+func TestE2ESeedTemplates(t *testing.T) {
 	dyn := e2eDynamicClient(t)
 	dir := filepath.Join("..", "..", "install", "templates")
 	entries, err := os.ReadDir(dir)
@@ -71,25 +98,34 @@ func TestE2ESeedTemplatesAcceptedByKro(t *testing.T) {
 				t.Fatalf("read %s: %v", entry.Name(), err)
 			}
 			tmpl := decodeTemplate(t, raw)
+
 			rgd, err := buildRGD(tmpl, testTokens())
 			if err != nil {
 				t.Fatalf("buildRGD(%s): %v", tmpl.Name, err)
 			}
-			// Apply under a throwaway name so the test doesn't clobber any
-			// RGDs a dev cluster already seeded; GraphAccepted is name-
-			// independent. The per-template instance CRD it would create may
-			// collide with a seeded one (KindReady), which is fine — we only
-			// assert the graph is accepted.
-			name := e2eRGDPrefix + tmpl.Name
-			rgd.SetName(name)
-			applyRGDForTest(t, dyn, rgd)
-			t.Cleanup(func() { deleteRGDForTest(dyn, name) })
-
-			reason, msg := waitGraphAccepted(t, dyn, name)
-			if reason != "True" {
-				t.Fatalf("kro rejected RGD for template %q: GraphAccepted=%s: %s", tmpl.Name, reason, msg)
+			applyRGD(t, dyn, rgd)
+			t.Cleanup(func() { _ = dyn.Resource(rgdGVR).Delete(context.Background(), rgd.GetName(), metav1.DeleteOptions{}) })
+
+			// 1. kro accepts the graph.
+			if status, msg := waitGraphAccepted(t, dyn, rgd.GetName()); status != "True" {
+				t.Fatalf("kro rejected RGD for template %q: GraphAccepted=%s: %s", tmpl.Name, status, msg)
+			}
+			t.Logf("template %q: RGD accepted", tmpl.Name)
+
+			// 2. kro creates an instance's objects without an apply error.
+			instGVR := schema.GroupVersionResource{
+				Group:    tmpl.Spec.InstanceCRD.Group,
+				Version:  tmpl.Spec.InstanceCRD.Version,
+				Resource: tmpl.Spec.InstanceCRD.Resource,
 			}
-			t.Logf("template %q: kro accepted the RGD", tmpl.Name)
+			inst := e2eInstance(t, tmpl)
+			createInstance(t, dyn, instGVR, inst)
+			t.Cleanup(func() {
+				_ = dyn.Resource(instGVR).Delete(context.Background(), inst.GetName(), metav1.DeleteOptions{})
+			})
+
+			waitInstanceApplied(t, dyn, instGVR, inst.GetName(), tmpl.Name)
+			t.Logf("template %q: instance reconciled (objects applied)", tmpl.Name)
 		})
 	}
 	if seen == 0 {
@@ -101,7 +137,7 @@ func e2eDynamicClient(t *testing.T) dynamic.Interface {
 	t.Helper()
 	kubeconfig := os.Getenv("KUBECONFIG")
 	if kubeconfig == "" {
-		t.Skip("KUBECONFIG not set; this e2e needs a kro cluster (see make e2e-infrastructure)")
+		t.Skip("KUBECONFIG not set; this e2e needs a clean kro cluster (see make e2e-infrastructure)")
 	}
 	cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
@@ -114,7 +150,7 @@ func e2eDynamicClient(t *testing.T) dynamic.Interface {
 	return dyn
 }
 
-func applyRGDForTest(t *testing.T, dyn dynamic.Interface, rgd *unstructured.Unstructured) {
+func applyRGD(t *testing.T, dyn dynamic.Interface, rgd *unstructured.Unstructured) {
 	t.Helper()
 	ctx := context.Background()
 	existing, err := dyn.Resource(rgdGVR).Get(ctx, rgd.GetName(), metav1.GetOptions{})
@@ -133,39 +169,118 @@ func applyRGDForTest(t *testing.T, dyn dynamic.Interface, rgd *unstructured.Unst
 	}
 }
 
-func deleteRGDForTest(dyn dynamic.Interface, name string) {
-	_ = dyn.Resource(rgdGVR).Delete(context.Background(), name, metav1.DeleteOptions{})
+// e2eInstance builds a sample instance: the template's sampleValues when present
+// (the curated working example), otherwise a minimal valid spec.
+func e2eInstance(t *testing.T, tmpl *infrav1alpha1.Template) *unstructured.Unstructured {
+	t.Helper()
+	spec := map[string]any{}
+	if sv := tmpl.Spec.SampleValues; sv != nil && len(sv.Raw) > 0 {
+		if err := json.Unmarshal(sv.Raw, &spec); err != nil {
+			t.Fatalf("template %q: decode sampleValues: %v", tmpl.Name, err)
+		}
+	} else if min, ok := e2eMinimalSpecs[tmpl.Name]; ok {
+		spec = min
+	} else {
+		t.Fatalf("template %q has no sampleValues and no e2eMinimalSpecs entry — add one", tmpl.Name)
+	}
+	name, _ := spec["name"].(string)
+	if name == "" {
+		name = "e2e-" + tmpl.Name
+		spec["name"] = name
+	}
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": tmpl.Spec.InstanceCRD.Group + "/" + tmpl.Spec.InstanceCRD.Version,
+		"kind":       tmpl.Spec.InstanceCRD.Kind,
+		"metadata":   map[string]any{"name": name},
+		"spec":       spec,
+	}}
+}
+
+// createInstance retries Create until kro has established + served the
+// per-template CRD (a fresh RGD takes a few seconds to register the kind).
+func createInstance(t *testing.T, dyn dynamic.Interface, gvr schema.GroupVersionResource, inst *unstructured.Unstructured) {
+	t.Helper()
+	deadline := time.Now().Add(e2eCRDWait)
+	for {
+		_, err := dyn.Resource(gvr).Create(context.Background(), inst, metav1.CreateOptions{})
+		if err == nil || apierrors.IsAlreadyExists(err) {
+			return
+		}
+		// "no matches for kind" / NotFound while the CRD is still registering.
+		if time.Now().After(deadline) {
+			t.Fatalf("create %s instance %q: CRD never became servable: %v", gvr.Resource, inst.GetName(), err)
+		}
+		time.Sleep(e2ePollEvery)
+	}
+}
+
+// waitInstanceApplied waits until kro has reconciled the instance and asserts it
+// applied its objects without an apply error. A readiness wait (images not
+// pulled in CI) is success — apply succeeded. An apply-error marker is failure.
+func waitInstanceApplied(t *testing.T, dyn dynamic.Interface, gvr schema.GroupVersionResource, name, tmplName string) {
+	t.Helper()
+	deadline := time.Now().Add(e2eInstanceWait)
+	var sawConditions bool
+	for time.Now().Before(deadline) {
+		obj, err := dyn.Resource(gvr).Get(context.Background(), name, metav1.GetOptions{})
+		if err != nil {
+			time.Sleep(e2ePollEvery)
+			continue
+		}
+		conds, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions")
+		for _, c := range conds {
+			cond, ok := c.(map[string]any)
+			if !ok {
+				continue
+			}
+			sawConditions = true
+			msg, _, _ := unstructured.NestedString(cond, "message")
+			for _, marker := range e2eApplyErrorMarkers {
+				if strings.Contains(msg, marker) {
+					ctype, _, _ := unstructured.NestedString(cond, "type")
+					t.Fatalf("template %q: kro failed to apply instance objects (%s): %s", tmplName, ctype, msg)
+				}
+			}
+		}
+		// kro reconciled it and recorded conditions, none of which are apply
+		// errors → the objects were applied. (Readiness is out of scope: the
+		// images may be unpullable in CI.)
+		if sawConditions {
+			return
+		}
+		time.Sleep(e2ePollEvery)
+	}
+	if !sawConditions {
+		t.Fatalf("template %q: kro never reconciled instance %q within %s", tmplName, name, e2eInstanceWait)
+	}
 }
 
-// waitGraphAccepted polls the RGD until its GraphAccepted condition resolves to
-// True or False (Unknown/AwaitingReconciliation keeps waiting), returning the
-// final status + message.
 func waitGraphAccepted(t *testing.T, dyn dynamic.Interface, name string) (string, string) {
 	t.Helper()
-	deadline := time.Now().Add(e2eAcceptWait)
+	deadline := time.Now().Add(e2eGraphWait)
 	var lastStatus, lastMsg string
 	for time.Now().Before(deadline) {
 		obj, err := dyn.Resource(rgdGVR).Get(context.Background(), name, metav1.GetOptions{})
 		if err == nil {
-			lastStatus, lastMsg = graphAcceptedCondition(obj)
+			lastStatus, lastMsg = conditionByType(obj, "GraphAccepted")
 			if lastStatus == "True" || lastStatus == "False" {
 				return lastStatus, lastMsg
 			}
 		}
 		time.Sleep(e2ePollEvery)
 	}
-	t.Fatalf("timed out after %s waiting for RGD %q GraphAccepted (last=%q: %s)", e2eAcceptWait, name, lastStatus, lastMsg)
+	t.Fatalf("timed out after %s waiting for RGD %q GraphAccepted (last=%q: %s)", e2eGraphWait, name, lastStatus, lastMsg)
 	return lastStatus, lastMsg
 }
 
-func graphAcceptedCondition(obj *unstructured.Unstructured) (string, string) {
+func conditionByType(obj *unstructured.Unstructured, condType string) (string, string) {
 	conds, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions")
 	for _, c := range conds {
 		cond, ok := c.(map[string]any)
 		if !ok {
 			continue
 		}
-		if t, _, _ := unstructured.NestedString(cond, "type"); t != "GraphAccepted" {
+		if tp, _, _ := unstructured.NestedString(cond, "type"); tp != condType {
 			continue
 		}
 		status, _, _ := unstructured.NestedString(cond, "status")