fix(infrastructure): operator recovers kro helm release stuck in pending-* (#342)

* fix(infrastructure): operator recovers kro helm release stuck in pending-*

An interrupted helm operation (typically the operator pod killed mid-upgrade
during an image rollout) leaves the kro release in a pending-upgrade /
pending-install / pending-rollback state. Helm then refuses every subsequent
operation with "another operation (install/upgrade/rollback) is in progress",
and since nothing clears pending state, the operator's 2-minute retries wedge
permanently (KroReleased=False, HelmFailed) until a human runs helm rollback.

EnsureKroRelease now clears a wedged release before upgrading:
- pending-upgrade / pending-rollback → roll back to the newest deployed/
  superseded revision (a known-good state exists);
- pending-install → uninstall the half-applied release (nothing to roll back
  to) so the upgrade reinstalls cleanly.

The decision logic is a pure, unit-tested function (planRecovery). Also routed
all helm invocations through a shared runHelm helper that honors the in-cluster
runtime (empty kubeconfig → no KUBECONFIG override), fixing DeleteKroRelease
which previously always set KUBECONFIG even for the in-cluster case.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* remove old api references

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mjudeikis

AGENTS.md

@@ -282,7 +282,6 @@ Each suite has a dedicated Make target. Most spin up their own hub on fixed port
 | `make e2e-external-kcp` | `external_kcp` | kcp via Helm in kind |
 | `make e2e-provider` | `provider` | Provider provisioning (quickstart) |
 | `make e2e-provider-flags` | `providerflags` | `--providers` flag mechanics (dep validation, filtering) |
-| `make e2e-infrastructure` | `infrastructure` | Infra provider cross-tenant isolation (stub mode, port 18082) |
 | `make e2e-tilt-cluster` | `tiltcluster` | Against a live `make tilt-cluster` multi-shard stack |
 | `make e2e-all` | all | Builds hub+agent images, runs everything (~30m) |
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: dev-edge-create dev-run-edge build test lint fix-lint codegen crds clean certs dev-setup run-dex run-hub run-hub-static run-hub-embedded run-hub-embedded-static run-hub-standalone run-hub-embedded-graphql run-kcp dev-login dev-login-static dev-create-workload dev dev-infra dev-run-kcp path boilerplate verify-boilerplate verify-codegen ldflags tools docker-build docker-build-hub docker-build-agent docker-build-dex docker-push-dex verify help-dev dev-status dev-clean-hooks helm-build-local helm-push-local helm-clean build-quickstart-provider build-quickstart-provider-portal build-kuery-provider build-kuery-provider-portal run-provider-kuery kuery-db-up kuery-db-down install-provider-kuery init-provider-kuery uninstall-provider-kuery run-provider-quickstart install-provider-quickstart init-provider-quickstart uninstall-provider-quickstart build-infrastructure-provider build-infrastructure-provider-portal codegen-infrastructure-provider run-provider-infrastructure install-provider-infrastructure init-provider-infrastructure uninstall-provider-infrastructure build-app-studio-provider build-app-studio-provider-portal codegen-app-studio-provider app-studio-db-up app-studio-db-down run-provider-app-studio install-provider-app-studio init-provider-app-studio uninstall-provider-app-studio build-code-provider build-code-provider-portal codegen-code-provider run-provider-code install-provider-code init-provider-code uninstall-provider-code dev-kro-up dev-kro-down dev-kro-seed dev-kro-register-self e2e-infrastructure portal-provider-symlinks build-mcp-provider-portal build-kubernetes-edges-provider-portal build-server-edges-provider-portal e2e-provider e2e-provider-flags e2e-provider-all
+.PHONY: dev-edge-create dev-run-edge build test lint fix-lint codegen crds clean certs dev-setup run-dex run-hub run-hub-static run-hub-embedded run-hub-embedded-static run-hub-standalone run-hub-embedded-graphql run-kcp dev-login dev-login-static dev-create-workload dev dev-infra dev-run-kcp path boilerplate verify-boilerplate verify-codegen ldflags tools docker-build docker-build-hub docker-build-agent docker-build-dex docker-push-dex verify help-dev dev-status dev-clean-hooks helm-build-local helm-push-local helm-clean build-quickstart-provider build-quickstart-provider-portal build-kuery-provider build-kuery-provider-portal run-provider-kuery kuery-db-up kuery-db-down install-provider-kuery init-provider-kuery uninstall-provider-kuery run-provider-quickstart install-provider-quickstart init-provider-quickstart uninstall-provider-quickstart build-infrastructure-provider build-infrastructure-provider-portal codegen-infrastructure-provider run-provider-infrastructure install-provider-infrastructure init-provider-infrastructure uninstall-provider-infrastructure build-app-studio-provider build-app-studio-provider-portal codegen-app-studio-provider app-studio-db-up app-studio-db-down run-provider-app-studio install-provider-app-studio init-provider-app-studio uninstall-provider-app-studio build-code-provider build-code-provider-portal codegen-code-provider run-provider-code install-provider-code init-provider-code uninstall-provider-code dev-kro-up dev-kro-down dev-kro-seed dev-kro-register-self portal-provider-symlinks build-mcp-provider-portal build-kubernetes-edges-provider-portal build-server-edges-provider-portal e2e-provider e2e-provider-flags e2e-provider-all
 
 BINDIR ?= bin
 GOFLAGS ?=
@@ -582,19 +582,6 @@ e2e-provider-flags: build-hub ## Run --providers flag mechanics suite
 ## Run both provider suites back-to-back (sequential — they share port 2380).
 e2e-provider-all: e2e-provider e2e-provider-flags ## Run provider + provider-flags suites sequentially
 
-## Provider-specific isolation suite for infrastructure. Spawns only
-## the infrastructure-provider binary on :18082 in stub mode (no kro
-## cluster, no hub, no kcp) and asserts cross-tenant reads / writes /
-## deletes are properly scoped. Independent of e2e-provider — they
-## don't share ports, so they could in principle run in parallel.
-E2E_KROMC_TIMEOUT ?= 5m
-e2e-infrastructure: build-infrastructure-provider ## Run infrastructure tenant-isolation e2e
-	@test -z "$$(lsof -ti :18082 2>/dev/null)" || { \
-		echo "port 18082 is in use; pkill infrastructure-provider and retry"; \
-		exit 1; \
-	}
-	go test ./test/e2e/suites/infrastructure/... -v -timeout $(E2E_KROMC_TIMEOUT) $(if $(E2E_FLAGS),-args $(E2E_FLAGS))
-
 ## Tilt-cluster suite: runs against an ALREADY-RUNNING operator-deployed,
 ## multi-shard Tilt stack (start it in another terminal with `make tilt-cluster`).
 ## Unlike the other e2e suites it does NOT spawn its own processes — it connects

docs/infrastructure-architecture.md

@@ -418,7 +418,7 @@ Note what tenants and the portal know about: `Template`, `Redis`. Nothing in thi
 | **B** | CachedResource publishing Templates | A tenant workspace with the APIBinding sees Templates as a read-only resource. `kubectl get templates -A` from a tenant returns the catalog. |
 | **C** | kro backend + APIExport VW wiring | Backend interface implemented for kro. The platform binary starts the kro multicluster-runtime pointed at the provider's APIExport VW. A tenant applies a Redis CR; kro reconciles it in the management cluster within 10s; status syncs back; delete propagates. Orphan sweeper for deleted tenant workspaces in scope. |
 | **D** | UI + MCP migration | Portal main app and dashboard tile read Templates + per-template CRs via GraphQL. MCP tools (`infrastructure__list_templates`, `__provision`, …) become kcp API calls. Old REST endpoints get a deprecation banner. |
-| **E** | Cleanup | REST handlers gone. `tenantHash`, `kedge-tenants-*` convention, per-request header identity all gone. `make e2e-infrastructure` exercises only the kcp path. |
+| **E** | Cleanup | REST handlers gone. `tenantHash`, `kedge-tenants-*` convention, per-request header identity all gone. The REST-surface e2e suite (`make e2e-infrastructure`) is removed; isolation is exercised via the kcp path. |
 
 A through C land the new platform + the first backend (~2.5 weeks). D adds another week (mostly portal rewrites). E is a day of deletions.
 

providers/infrastructure/README.md

@@ -130,7 +130,7 @@ helm CLI.
 
 | Surface | Where |
 |---|---|
-| HTTP REST | `server/` — `/api/templates`, `/api/instances` |
+| HTTP server | `server/` — `/healthz`, portal SPA, `/mcp` |
 | MCP transport | `mcpserver/` — `/mcp`, `/mcp/sse` (6 `kro_*` tools) |
 | Central kro client | `kro/` — `ResourceGraphDefinition` discovery + instance lifecycle |
 | Tenant kcp client | `tenant/` — per-tenant `cloud-credentials` Secret resolution |
@@ -236,20 +236,12 @@ npm --prefix portal run build
 go run .
 # → listening on :8081 (kro=*kro.stubClient tenant=false mcp=true)
 
-# 3. Smoke tests:
+# 3. Smoke test: liveness.
 curl -s localhost:8081/healthz
-curl -s localhost:8081/api/templates | jq '.items[].name'
-curl -s localhost:8081/api/templates/postgres | jq '.template.inputsSchema'
-
-# 4. Provision flow (dev mode lets ?tenant= replace the X-Kedge-Tenant header).
-export KEDGE_DEV_ALLOW_TENANT_QUERY=true
-curl -s -X POST 'localhost:8081/api/instances?tenant=dev&user=alice' \
-  -H 'Content-Type: application/json' \
-  -d '{"templateName":"postgres","name":"foo","values":{"name":"foo","size":"medium"}}'
-curl -s 'localhost:8081/api/instances?tenant=dev' | jq
-curl -s -X DELETE 'localhost:8081/api/instances/foo?tenant=dev' -i
-
-# 5. MCP tools/list (note: SSE response — pipe through `tail`).
+
+# 4. MCP tools/list (note: SSE response — pipe through `head`). Templates
+#    and instances are NOT served as REST — they are kro_* MCP tools and,
+#    in a real cluster, CRDs read/written directly against kcp.
 curl -s -X POST -H 'Content-Type: application/json' \
   -H 'Accept: application/json, text/event-stream' \
   -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \

providers/infrastructure/examples/rgds/README.md

@@ -14,7 +14,7 @@ kro/templates.go`) looks for these keys verbatim:
 |---|---|
 | `kedge.faros.sh/expose=true` (label) | gates visibility — required |
 | `kedge.faros.sh/template-name` (label) | catalog slug; defaults to `metadata.name` |
-| `kedge.faros.sh/template-version` (label) | required by `POST /api/instances` for safety |
+| `kedge.faros.sh/template-version` (label) | required when provisioning, for safety |
 | `kedge.faros.sh/category` (label) | filter chip in the catalog grid |
 | `kedge.faros.sh/cloud` (label) | filter chip + maps credential schema |
 | `kedge.faros.sh/display-name` (annotation) | human-readable name |

providers/infrastructure/examples/rgds/simple-webapp.yaml

@@ -7,7 +7,7 @@
 #
 #   kedge.faros.sh/expose=true            → mandatory; gates visibility
 #   kedge.faros.sh/template-name          → catalog slug (the URL segment)
-#   kedge.faros.sh/template-version       → required by POST /api/instances
+#   kedge.faros.sh/template-version       → required when provisioning
 #   kedge.faros.sh/category, /cloud       → optional facet chips in UI
 #   kedge.faros.sh/display-name           → human-readable name
 #   kedge.faros.sh/description            → short blurb

providers/infrastructure/kro/client.go

@@ -51,7 +51,7 @@ type Client interface {
 	DeleteInstance(ctx context.Context, tenantPath, name string) error
 }
 
-// TemplateFilter is the GET /api/templates query, normalized.
+// TemplateFilter narrows a template listing by category and cloud.
 type TemplateFilter struct {
 	Category string
 	Cloud    string

providers/infrastructure/kro/types.go

@@ -32,7 +32,7 @@ import (
 // JSON-schema-shaped InputsSchema the DynamicForm consumes.
 type Template struct {
 	// Name is the RGD's metadata.name; it doubles as the template
-	// identifier used in API URLs (POST /api/instances {templateName}).
+	// identifier callers reference when provisioning an instance.
 	Name string `json:"name"`
 	// DisplayName, Description, Category, Cloud are pulled from the
 	// RGD's labels/annotations (see kedge.faros.sh/* convention in
@@ -43,7 +43,7 @@ type Template struct {
 	Category    string `json:"category,omitempty"`
 	Cloud       string `json:"cloud,omitempty"`
 	// Version is read from the kedge.faros.sh/template-version label.
-	// Required by POST /api/instances so we never silently provision
+	// Required when provisioning so we never silently provision
 	// against a different RGD generation than the user previewed.
 	Version string `json:"version,omitempty"`
 	// IconURL is an optional asset URL stored in the
@@ -134,7 +134,7 @@ const (
 	// you want consumers' bookmarks to keep working.
 	LabelTemplateName = "kedge.faros.sh/template-name"
 	// LabelTemplateVersion pins a semver to the RGD revision. Required
-	// in POST /api/instances so a chart bump doesn't silently change
+	// when provisioning so a chart bump doesn't silently change
 	// what gets provisioned.
 	LabelTemplateVersion = "kedge.faros.sh/template-version"
 	// LabelCategory lets the catalog UI render filter chips.

providers/infrastructure/operator/helm.go

@@ -12,9 +12,11 @@ package operator
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 
 	"github.com/faroshq/provider-infrastructure/apis/v1alpha1"
 	"github.com/faroshq/provider-infrastructure/install"
@@ -26,6 +28,14 @@ import (
 // into this module. The kcp-kubeconfig Secret kro mounts is seeded separately
 // (install.SeedKroClusterFromKubeconfig) before this runs.
 func EnsureKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1alpha1.KroSpec) error {
+	// Clear any release wedged in a pending-* state by an interrupted prior helm
+	// operation before attempting the upgrade — otherwise helm refuses with
+	// "another operation (install/upgrade/rollback) is in progress" and the
+	// operator's retries loop forever (nothing else clears pending state).
+	if err := recoverPendingRelease(ctx, runtimeKubeconfigPath, kro); err != nil {
+		return err
+	}
+
 	args := []string{
 		"upgrade", "--install", kro.ReleaseName, kro.Chart,
 		"--version", kro.Version,
@@ -51,13 +61,7 @@ func EnsureKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1a
 		args = append(args, "--set", k+"="+v)
 	}
 
-	cmd := exec.CommandContext(ctx, "helm", args...)
-	// An empty path means "in-cluster runtime": run helm with no KUBECONFIG
-	// override so it uses the pod's in-cluster service account.
-	if runtimeKubeconfigPath != "" {
-		cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)
-	}
-	if out, err := cmd.CombinedOutput(); err != nil {
+	if out, err := runHelm(ctx, runtimeKubeconfigPath, args...); err != nil {
 		return fmt.Errorf("helm upgrade --install %s: %w\n%s", kro.ReleaseName, err, string(out))
 	}
 	return nil
@@ -66,11 +70,132 @@ func EnsureKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1a
 // DeleteKroRelease uninstalls the kro Helm release (best-effort; a missing
 // release is not an error).
 func DeleteKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1alpha1.KroSpec) error {
-	cmd := exec.CommandContext(ctx, "helm", "uninstall", kro.ReleaseName,
-		"--namespace", kro.Namespace, "--ignore-not-found")
-	cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)
-	if out, err := cmd.CombinedOutput(); err != nil {
+	if out, err := runHelm(ctx, runtimeKubeconfigPath, "uninstall", kro.ReleaseName,
+		"--namespace", kro.Namespace, "--ignore-not-found"); err != nil {
 		return fmt.Errorf("helm uninstall %s: %w\n%s", kro.ReleaseName, err, string(out))
 	}
 	return nil
 }
+
+// recoverPendingRelease detects a kro release left in a pending-* state by an
+// interrupted helm operation (most commonly the operator pod killed mid-upgrade
+// during a rollout) and clears it so the subsequent upgrade can proceed. Helm
+// stores the in-flight state in the release record and refuses any new operation
+// until it resolves; nothing clears it automatically, so the operator's 2-minute
+// retries would otherwise wedge permanently.
+//
+// Strategy (see planRecovery):
+//   - pending-upgrade / pending-rollback → roll back to the newest non-pending
+//     revision (a known-good state exists);
+//   - pending-install → the first install never completed, there is nothing to
+//     roll back to, so uninstall the half-applied release and let the upgrade
+//     reinstall it.
+func recoverPendingRelease(ctx context.Context, kubeconfigPath string, kro v1alpha1.KroSpec) error {
+	status, err := helmReleaseStatus(ctx, kubeconfigPath, kro)
+	if err != nil {
+		// Release not found (fresh install) or status unreadable — nothing to
+		// recover; let the upgrade --install proceed and surface any real error.
+		return nil
+	}
+
+	rev, hasDeployed := lastDeployedRevision(ctx, kubeconfigPath, kro)
+	switch action, target := planRecovery(status, rev, hasDeployed); action {
+	case recoverRollback:
+		if out, rerr := runHelm(ctx, kubeconfigPath, "rollback", kro.ReleaseName,
+			strconv.Itoa(target), "--namespace", kro.Namespace); rerr != nil {
+			return fmt.Errorf("recover pending %q: helm rollback %s to %d: %w\n%s",
+				status, kro.ReleaseName, target, rerr, string(out))
+		}
+	case recoverUninstall:
+		if out, rerr := runHelm(ctx, kubeconfigPath, "uninstall", kro.ReleaseName,
+			"--namespace", kro.Namespace, "--ignore-not-found"); rerr != nil {
+			return fmt.Errorf("recover pending %q: helm uninstall %s: %w\n%s",
+				status, kro.ReleaseName, rerr, string(out))
+		}
+	case recoverNone:
+		// Healthy (deployed/failed) — helm upgrade --install handles it directly.
+	}
+	return nil
+}
+
+// recoveryAction is the remediation planRecovery selects for a release status.
+type recoveryAction int
+
+const (
+	recoverNone recoveryAction = iota
+	recoverRollback
+	recoverUninstall
+)
+
+// planRecovery maps a helm release status to the remediation needed before a
+// fresh upgrade can run. Pure (no I/O) so the decision logic is unit-testable;
+// the revision/hasDeployed inputs come from lastDeployedRevision.
+func planRecovery(status string, lastDeployedRev int, hasDeployed bool) (recoveryAction, int) {
+	switch status {
+	case "pending-upgrade", "pending-rollback":
+		if hasDeployed {
+			return recoverRollback, lastDeployedRev
+		}
+		// No good revision to fall back to — treat like a failed first install.
+		return recoverUninstall, 0
+	case "pending-install":
+		return recoverUninstall, 0
+	default:
+		return recoverNone, 0
+	}
+}
+
+// helmReleaseStatus returns the release's current status (e.g. "deployed",
+// "pending-upgrade"). A non-nil error means the release was not found or its
+// status could not be read — callers treat that as "nothing to recover".
+func helmReleaseStatus(ctx context.Context, kubeconfigPath string, kro v1alpha1.KroSpec) (string, error) {
+	out, err := runHelm(ctx, kubeconfigPath, "status", kro.ReleaseName,
+		"--namespace", kro.Namespace, "-o", "json")
+	if err != nil {
+		return "", fmt.Errorf("helm status %s: %w\n%s", kro.ReleaseName, err, string(out))
+	}
+	var s struct {
+		Info struct {
+			Status string `json:"status"`
+		} `json:"info"`
+	}
+	if err := json.Unmarshal(out, &s); err != nil {
+		return "", fmt.Errorf("decode helm status %s: %w", kro.ReleaseName, err)
+	}
+	return s.Info.Status, nil
+}
+
+// lastDeployedRevision returns the highest revision whose status is a known-good
+// rollback target (deployed or superseded), and whether such a revision exists.
+func lastDeployedRevision(ctx context.Context, kubeconfigPath string, kro v1alpha1.KroSpec) (int, bool) {
+	out, err := runHelm(ctx, kubeconfigPath, "history", kro.ReleaseName,
+		"--namespace", kro.Namespace, "-o", "json")
+	if err != nil {
+		return 0, false
+	}
+	var hist []struct {
+		Revision int    `json:"revision"`
+		Status   string `json:"status"`
+	}
+	if err := json.Unmarshal(out, &hist); err != nil {
+		return 0, false
+	}
+	best, found := 0, false
+	for _, h := range hist {
+		if (h.Status == "deployed" || h.Status == "superseded") && h.Revision > best {
+			best, found = h.Revision, true
+		}
+	}
+	return best, found
+}
+
+// runHelm runs the helm CLI, pointing it at the runtime cluster. An empty
+// kubeconfigPath means "in-cluster runtime": helm runs with no KUBECONFIG
+// override so it uses the pod's in-cluster service account.
+func runHelm(ctx context.Context, kubeconfigPath string, args ...string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, "helm", args...)
+	if kubeconfigPath != "" {
+		cmd.Env = append(os.Environ(), "KUBECONFIG="+kubeconfigPath)
+	}
+	return cmd.CombinedOutput()
+}