fix(infra): deterministic APIResourceSchema name hash (stop etcd-filling leak)

schemaPrefix hashed the CRD's OpenAPIV3Schema with fmt %v. OpenAPIV3Schema
is full of pointer fields (Default, Items, AdditionalProperties, …), and
fmt %v prints nested pointer fields by *address*, not value. Those
addresses change every reconcile, so the hash — and thus the immutable
APIResourceSchema name (tmpl<hash>.templates.infrastructure.kedge.faros.sh)
— was non-deterministic. Every reconcile minted a brand-new schema and
never deleted the old one.

In a long-lived environment this leaked ~4.5k APIResourceSchemas (~55MB in
/registry, mirrored in /cache) into a single logical cluster. On the next
control-plane restart, kcp's cold-start LIST of all schemas pushed etcd
past its memory limit into a permanent OOMKill loop, taking the root shard
(and the console) down.

The controller path already hashed via json.Marshal but kept a %v
fallback; the install path still used %v directly.

- install/apiexport.go: hash json.Marshal(schema); stable, address-free
  fallback instead of %v.
- controller/template/apiexport.go: drop the %v fallback for the same.
- install/apiexport_test.go: regression test — identical content
  (incl. Default pointer fields) must hash equal across allocations,
  and a real content change must still change the name.

Both callsites now frame the hash identically, so bootstrap-install and
the runtime controller produce the same name for the same schema.

Note: this stops new leaks. Recovering an already-affected cluster still
requires bumping etcd memory, GC-ing the orphaned schemas, and an etcd
defrag/compact.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: mjudeikis

providers/infrastructure/controller/template/apiexport.go

@@ -202,7 +202,11 @@ func schemaPrefix(crd *apiextensionsv1.CustomResourceDefinition) string {
 			// non-deterministic.
 			data, err := crdsJSONMarshal(v.Schema.OpenAPIV3Schema)
 			if err != nil {
-				_, _ = fmt.Fprintf(h, "%v\n", v.Schema.OpenAPIV3Schema)
+				// Stable, address-free fallback. NEVER use fmt %v here: it
+				// prints pointer-field addresses, which makes the schema name
+				// non-deterministic and leaks a new immutable APIResourceSchema
+				// every reconcile (fills etcd → OOM).
+				_, _ = fmt.Fprintf(h, "marshal-error:%s/%s\n", crd.Name, v.Name)
 				continue
 			}
 			_, _ = h.Write(data)

providers/infrastructure/install/apiexport.go

@@ -305,7 +305,23 @@ func schemaPrefix(crd *apiextensionsv1.CustomResourceDefinition) string {
 	for _, v := range crd.Spec.Versions {
 		fmt.Fprintln(h, v.Name)
 		if v.Schema != nil && v.Schema.OpenAPIV3Schema != nil {
-			fmt.Fprintf(h, "%v\n", v.Schema.OpenAPIV3Schema)
+			// Hash a deterministic JSON serialization. NEVER use fmt %v
+			// here: OpenAPIV3Schema is full of pointer fields (Default,
+			// Items, AdditionalProperties, …) and %v prints their memory
+			// addresses, which change every reconcile. That makes the
+			// APIResourceSchema name non-deterministic, so each reconcile
+			// mints a new immutable schema and leaks the old one — the leak
+			// that filled etcd and OOM-killed it. json.Marshal sorts object
+			// keys and renders pointers by value, so identical content always
+			// yields an identical name. Mirrors controller/template.
+			data, err := json.Marshal(v.Schema.OpenAPIV3Schema)
+			if err != nil {
+				// Should never happen for a JSONSchemaProps. Fall back to a
+				// stable, address-free key rather than %v.
+				fmt.Fprintf(h, "marshal-error:%s/%s\n", crd.Name, v.Name)
+				continue
+			}
+			h.Write(data)
 		}
 	}
 	return "tmpl" + hex.EncodeToString(h.Sum(nil))[:8]

providers/infrastructure/install/apiexport_test.go

@@ -0,0 +1,69 @@
+package install
+
+import (
+	"testing"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+)
+
+// schemaWithPointers builds a CRD whose OpenAPIV3Schema populates the
+// pointer-typed fields (Default, *bool flags) that fmt %v renders as
+// memory addresses. Each call allocates fresh, so two structurally
+// identical results live at different addresses — the precise condition
+// that made the old %v-based hash non-deterministic and leaked a new
+// immutable APIResourceSchema on every reconcile (eventually OOM-ing etcd).
+func schemaWithPointers() *apiextensionsv1.CustomResourceDefinition {
+	preserve := true
+	return &apiextensionsv1.CustomResourceDefinition{
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
+			Group: "infrastructure.kedge.faros.sh",
+			Names: apiextensionsv1.CustomResourceDefinitionNames{Kind: "Template", Plural: "templates"},
+			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{
+				Name: "v1alpha1",
+				Schema: &apiextensionsv1.CustomResourceValidation{
+					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
+						Type: "object",
+						Properties: map[string]apiextensionsv1.JSONSchemaProps{
+							"image": {
+								Type:    "string",
+								Default: &apiextensionsv1.JSON{Raw: []byte(`"registry.example/img:v1"`)},
+							},
+							"replicas": {
+								Type:    "integer",
+								Default: &apiextensionsv1.JSON{Raw: []byte(`1`)},
+							},
+						},
+						XPreserveUnknownFields: &preserve,
+					},
+				},
+			}},
+		},
+	}
+}
+
+// TestSchemaPrefixDeterministic locks the fix: identical schema content
+// must hash to the same name regardless of allocation, even when the
+// schema carries pointer fields. With the old fmt %v hash this failed
+// because %v printed pointer addresses.
+func TestSchemaPrefixDeterministic(t *testing.T) {
+	a := schemaPrefix(schemaWithPointers())
+	b := schemaPrefix(schemaWithPointers())
+	if a != b {
+		t.Fatalf("schemaPrefix must be deterministic for identical content; got %q vs %q", a, b)
+	}
+}
+
+// TestSchemaPrefixSensitiveToContent ensures a real content change still
+// produces a different name (so genuine schema updates get a new schema).
+func TestSchemaPrefixSensitiveToContent(t *testing.T) {
+	base := schemaWithPointers()
+	changed := schemaWithPointers()
+	changed.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["replicas"] =
+		apiextensionsv1.JSONSchemaProps{
+			Type:    "integer",
+			Default: &apiextensionsv1.JSON{Raw: []byte(`3`)}, // 1 -> 3
+		}
+	if schemaPrefix(base) == schemaPrefix(changed) {
+		t.Fatal("schemaPrefix must change when schema content changes")
+	}
+}