feat(infrastructure): CRD-driven operator (kro helm lifecycle + serve) (#331)

* feat(infrastructure): CRD-driven operator (kro helm lifecycle + serve)

Add an InfrastructureProvider operator: one CR declares the kro + provider
image versions and two kubeconfig Secret refs; the operator continuously
bootstraps the provider kcp workspace, lifecycles the kro Helm release via the
helm CLI (multicluster fork chart + image, kcp-apiexport mode, kind hostAliases
+ self-cluster patches behind env flags), seeds kro's kcp-kubeconfig, and owns
the provider serve Deployment.

- apis/v1alpha1: InfrastructureProvider CRD (+ generated deepcopy/CRD YAML)
- operator/: controller-runtime reconciler, helm exec, serve Deployment,
  bootstrap (shared with the env-driven operator subcommand), kind dev patches
- main.go: `operator` (env-driven) + `controller` (CRD) subcommands;
  serveWithConfig extracted; loadControllerConfig honors INFRASTRUCTURE_WORKSPACE_PATH
- install/operatorseed.go: seed kro from a provided kubeconfig (no SA mint)
- deploy/chart: operator Deployment + RBAC + CRD + Secrets + CR (operator.enabled),
  legacy serve/Service gated off in operator mode
- Dockerfile: bundle the helm CLI for the operator pod
- Makefile + Tiltfile.cluster: operator dev flow (controller host binary;
  replaces kro-mgmt-up + infrastructure-init), --validate=false on register
- README: operator deploy section

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* chore(kuery): bump provider to 788efce

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mjudeikis

Makefile

@@ -828,10 +828,62 @@ run-provider-infrastructure: build-infrastructure-provider ## Run the infrastruc
 	KEDGE_HUB_INSECURE=true \
 	KEDGE_PROVIDER_NAME=infrastructure \
 	KEDGE_DEV_ALLOW_TENANT_QUERY=true \
+	INFRASTRUCTURE_WORKSPACE_PATH=$${INFRASTRUCTURE_WORKSPACE_PATH:-$(INFRASTRUCTURE_WORKSPACE_PATH)} \
 	KRO_KUBECONFIG=$${KRO_KUBECONFIG:-$$( [ -f "$(KRO_KIND_KUBECONFIG)" ] && echo "$(KRO_KIND_KUBECONFIG)" )} \
 	INFRASTRUCTURE_KUBECONFIG=$${INFRASTRUCTURE_KUBECONFIG:-$$( [ -f "$(INFRASTRUCTURE_RUNTIME_KUBECONFIG)" ] && echo "$(INFRASTRUCTURE_RUNTIME_KUBECONFIG)" )} \
 		$(BINDIR)/infrastructure-provider
 
+run-provider-infrastructure-operator: build-infrastructure-provider ## Run the infrastructure provider in OPERATOR mode (bootstrap reconcile + serve from a provider + runtime kubeconfig)
+	@echo "Starting infrastructure provider (operator) on :$(KROMC_PORT)"
+	@echo "  hub:      $(KROMC_HUB_URL)"
+	@echo "  provider: $${INFRASTRUCTURE_PROVIDER_KUBECONFIG:-$(KROMC_KCP_KUBECONFIG)} (kcp)"
+	@echo "  runtime:  $${INFRASTRUCTURE_RUNTIME_KUBECONFIG:-$(KRO_KIND_KUBECONFIG)} (kro cluster)"
+	@echo "  ws:       $(INFRASTRUCTURE_WORKSPACE_PATH)"
+	@# The operator needs the provider workspace to already exist — run
+	@# `make install-provider-infrastructure` (admin-portal onboarding in prod)
+	@# first. It then reconciles the in-workspace bootstrap and seeds kro itself.
+	PORT=$(KROMC_PORT) \
+	KEDGE_HUB_URL=$(KROMC_HUB_URL) \
+	KEDGE_HUB_TOKEN=$(KROMC_TOKEN) \
+	KEDGE_HUB_INSECURE=true \
+	KEDGE_PROVIDER_NAME=infrastructure \
+	KEDGE_DEV_ALLOW_TENANT_QUERY=true \
+	INFRASTRUCTURE_WORKSPACE_PATH=$(INFRASTRUCTURE_WORKSPACE_PATH) \
+	INFRASTRUCTURE_PROVIDER_KUBECONFIG=$${INFRASTRUCTURE_PROVIDER_KUBECONFIG:-$(KROMC_KCP_KUBECONFIG)} \
+	INFRASTRUCTURE_RUNTIME_KUBECONFIG=$${INFRASTRUCTURE_RUNTIME_KUBECONFIG:-$$( [ -f "$(KRO_KIND_KUBECONFIG)" ] && echo "$(KRO_KIND_KUBECONFIG)" )} \
+		$(BINDIR)/infrastructure-provider operator
+
+# ── CRD-driven operator (controller) dev flow ───────────────────────────────
+# Replaces kro-mgmt-up + infrastructure-init: one host-binary controller that
+# bootstraps the workspace, helm-installs kro (with the kind hostAliases +
+# self-cluster patches), and (skip-serve in dev) leaves serve to the host binary.
+INFRA_OPERATOR_NS ?= kedge-infrastructure-operator
+INFRA_OPERATOR_PROVIDER_KC ?= $(KROMC_KCP_KUBECONFIG)
+INFRA_OPERATOR_RUNTIME_KC ?= $(KRO_KIND_KUBECONFIG)
+INFRA_OPERATOR_KIND_NAME ?= $(KRO_KIND_NAME)
+INFRA_OPERATOR_SELF_KC ?= $(KCP_DATA_DIR)/kro-self.kubeconfig
+INFRA_OPERATOR_HOSTALIASES_IP ?= 10.96.2.2
+INFRA_OPERATOR_HOSTALIASES_NAMES ?= kcp.localhost,root.kcp.localhost,theseus.kcp.localhost
+INFRA_OPERATOR_CRD ?= providers/infrastructure/config/crds/infrastructure.kedge.faros.sh_infrastructureproviders.yaml
+
+run-provider-infrastructure-controller: build-infrastructure-provider ## Apply the operator CRD/Secrets/CR into the runtime cluster and run the controller (dev)
+	@echo "Applying operator CRD + Secrets + CR into runtime cluster ($(INFRA_OPERATOR_RUNTIME_KC))"
+	KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl apply -f $(INFRA_OPERATOR_CRD)
+	KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl create namespace $(INFRA_OPERATOR_NS) --dry-run=client -o yaml | KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl apply -f -
+	KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl -n $(INFRA_OPERATOR_NS) create secret generic provider-kubeconfig --from-file=kubeconfig=$(INFRA_OPERATOR_PROVIDER_KC) --dry-run=client -o yaml | KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl apply -f -
+	KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl -n $(INFRA_OPERATOR_NS) create secret generic runtime-kubeconfig --from-file=kubeconfig=$(INFRA_OPERATOR_RUNTIME_KC) --dry-run=client -o yaml | KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl apply -f -
+	@# kind-internal kubeconfig for the kro local-runtime self-member.
+	kind get kubeconfig --internal --name $(INFRA_OPERATOR_KIND_NAME) > $(INFRA_OPERATOR_SELF_KC)
+	@printf 'apiVersion: infrastructure.kedge.faros.sh/v1alpha1\nkind: InfrastructureProvider\nmetadata:\n  name: infrastructure\n  namespace: %s\nspec:\n  providerWorkspace: %s\n  providerKubeconfigSecret:\n    name: provider-kubeconfig\n  runtimeKubeconfigSecret:\n    name: runtime-kubeconfig\n  kro:\n    chart: %s\n    version: %s\n    image:\n      repository: %s\n      tag: %s\n  provider:\n    image:\n      repository: ghcr.io/faroshq/kedge-infrastructure-provider\n      tag: dev\n' "$(INFRA_OPERATOR_NS)" "$(INFRASTRUCTURE_WORKSPACE_PATH)" "$(KRO_CHART)" "$(KRO_CHART_VERSION)" "$(KRO_IMAGE_REPO)" "$(KRO_IMAGE_TAG)" | KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) kubectl apply -f -
+	@echo "Running infrastructure operator controller (KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC), skip-serve)"
+	KUBECONFIG=$(INFRA_OPERATOR_RUNTIME_KC) \
+	INFRASTRUCTURE_WORKSPACE_PATH=$(INFRASTRUCTURE_WORKSPACE_PATH) \
+	INFRASTRUCTURE_OPERATOR_SKIP_SERVE=true \
+	INFRASTRUCTURE_KRO_HOSTALIASES_IP=$(INFRA_OPERATOR_HOSTALIASES_IP) \
+	INFRASTRUCTURE_KRO_HOSTALIASES_NAMES=$(INFRA_OPERATOR_HOSTALIASES_NAMES) \
+	INFRASTRUCTURE_KRO_SELF_CLUSTER_KUBECONFIG=$(INFRA_OPERATOR_SELF_KC) \
+		$(BINDIR)/infrastructure-provider controller
+
 ## Run the App Studio provider binary locally. Mirrors the other external
 ## providers so the heartbeat path is consistent and the UI runs on :8085.
 app-studio-db-up: ## Start/reuse local Postgres for App Studio message history (skips when APP_STUDIO_DATABASE_URL or in-memory mode is set)
@@ -985,7 +1037,7 @@ install-provider-infrastructure: ## Apply infrastructure Provider + CatalogEntry
 	kubectl --kubeconfig=$(KROMC_KCP_KUBECONFIG) \
 		--server=$(KROMC_KCP_SERVER)/clusters/root:kedge:system:providers \
 		--insecure-skip-tls-verify \
-		apply -f $(KROMC_PROVIDER_MANIFEST) -f $(KROMC_MANIFEST)
+		apply --validate=false -f $(KROMC_PROVIDER_MANIFEST) -f $(KROMC_MANIFEST)
 
 ## Delete the infrastructure CatalogEntry + Provider (Provider delete triggers
 ## full teardown of the sub-workspace via the controller's finalizer).

Tiltfile.cluster

@@ -639,6 +639,8 @@ local_resource(
 # kro reaches kcp via in-cluster Service DNS — no cross-kind networking.
 # The infrastructure provider on the host still reaches kro via the kind
 # cluster's host-published apiserver (KIND_HOST_KUBECONFIG).
+# kro-mgmt-up is now MANUAL: the infrastructure-operator helm-installs +
+# patches kro itself. Kept as a fallback / for poking kro without the operator.
 local_resource(
     'kro-mgmt-up',
     cmd=('KRO_TARGET_KUBECONFIG={kc} KRO_TARGET_KIND_NAME={name} ' +
@@ -647,6 +649,8 @@ local_resource(
     deps=[
         'providers/infrastructure/examples/rgds',
     ],
+    trigger_mode=TRIGGER_MODE_MANUAL,
+    auto_init=False,
     labels=['providers-kro'],
 )
 
@@ -662,12 +666,14 @@ local_resource(
 local_resource(
     'infrastructure',
     cmd='make build-infrastructure-provider',
-    # Point KRO_KUBECONFIG at the same kind cluster's host kubeconfig (where
-    # kro is now installed). The provider's make target already falls back to
-    # this when KRO_KUBECONFIG is set in the env, so the default kedge-kro
-    # path remains untouched for the embedded-kcp Tiltfile.
-    serve_cmd='KRO_KUBECONFIG={kc} make run-provider-infrastructure'.format(
-        kc=KIND_HOST_KUBECONFIG),
+    # Serve (host binary). The operator does bootstrap + kro; serve just runs
+    # with the admin kubeconfig retargeted to the provider workspace
+    # (INFRASTRUCTURE_WORKSPACE_PATH), so no init-minted kubeconfig is needed.
+    # KRO_KUBECONFIG points at the kind cluster's host kubeconfig (where kro is).
+    serve_cmd=('KRO_KUBECONFIG={kro} INFRASTRUCTURE_KUBECONFIG={kc} ' +
+               'INFRASTRUCTURE_WORKSPACE_PATH=root:kedge:providers:infrastructure ' +
+               'make run-provider-infrastructure').format(
+                   kro=KIND_HOST_KUBECONFIG, kc=KCP_ADMIN_KUBECONFIG),
     deps=[
         'providers/infrastructure/main.go',
         'providers/infrastructure/heartbeat.go',
@@ -685,19 +691,10 @@ local_resource(
         'providers/infrastructure/portal/package.json',
         'providers/infrastructure/go.mod',
         'providers/infrastructure/go.sum',
-        # Restart whenever init writes/updates the runtime kubeconfig:
-        # the controller manager only starts when INFRASTRUCTURE_KUBECONFIG
-        # resolves to a real file (see the Makefile target). Without this
-        # watch, the provider that booted before `infrastructure-init`
-        # keeps running with controllers disabled, and per-template
-        # APIResourceSchemas never get added to the APIExport.
-        '.kcp/infrastructure-runtime.kubeconfig',
     ],
-    # kedge-hub for CatalogEntry registration target, kro-mgmt-up for the
-    # backend cluster the catalog reads from. Both must be green before
-    # the provider starts; otherwise it boots in stub mode which is fine
-    # but confusing for the dev who just ran `tilt up`.
-    resource_deps=['kedge-hub', 'kro-mgmt-up'],
+    # kedge-hub for the CatalogEntry registration target; infrastructure-operator
+    # for the workspace bootstrap + kro install the serve process depends on.
+    resource_deps=['kedge-hub', 'infrastructure-operator'],
     readiness_probe=probe(
         period_secs=5,
         http_get=http_get_action(port=8082, path='/healthz'),
@@ -752,6 +749,35 @@ local_resource(
     labels=['providers-kro'],
 )
 
+# --- infrastructure-operator (CRD-driven) ---
+# One controller that REPLACES kro-mgmt-up + infrastructure-init: it applies the
+# InfrastructureProvider CRD/Secrets/CR into the kind cluster, then reconciles —
+# bootstrapping the provider workspace, helm-installing kro (with the kind
+# hostAliases + self-cluster patches), and seeding kro's kcp-kubeconfig. Serve
+# still runs as the host-binary `infrastructure` resource (skip-serve) for fast
+# iteration. Order: infrastructure-register (creates the workspace) →
+# infrastructure-operator → infrastructure (serve).
+local_resource(
+    'infrastructure-operator',
+    cmd='make build-infrastructure-provider',
+    serve_cmd=('INFRA_OPERATOR_PROVIDER_KC={kc} INFRA_OPERATOR_RUNTIME_KC={kro} ' +
+               'INFRA_OPERATOR_KIND_NAME={name} ' +
+               'make run-provider-infrastructure-controller').format(
+                   kc=KCP_ADMIN_KUBECONFIG, kro=KIND_HOST_KUBECONFIG, name=cluster_name),
+    deps=[
+        'providers/infrastructure/main.go',
+        'providers/infrastructure/operator.go',
+        'providers/infrastructure/operator',
+        'providers/infrastructure/apis',
+        'providers/infrastructure/init_cmd.go',
+        'providers/infrastructure/install',
+        'providers/infrastructure/go.mod',
+        'providers/infrastructure/go.sum',
+    ],
+    resource_deps=['kedge-hub', 'infrastructure-register'],
+    labels=['providers-kro'],
+)
+
 # --- providers-code (git repository management) ---
 # Host binary on :8083, same shape as quickstart/infrastructure. register/init/
 # unregister override KROMC_KCP_KUBECONFIG / KROMC_KCP_SERVER to the in-cluster

providers/infrastructure/Dockerfile

@@ -19,10 +19,26 @@ COPY . ./
 COPY --from=portal /portal/dist ./portal/dist
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/infrastructure-provider .
 
+# 2b. Fetch the helm CLI. The operator (`controller` subcommand) shells out to
+#     helm to install/upgrade the kro release, so the runtime image needs it.
+FROM alpine:3.20 AS helm
+ARG TARGETARCH
+ARG HELM_VERSION=v3.16.4
+RUN apk add --no-cache curl tar && \
+    curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" | tar -xz && \
+    install -m 0755 "linux-${TARGETARCH}/helm" /helm
+
 # 3. Minimal runtime image.
 FROM gcr.io/distroless/static:nonroot
 COPY --from=build /out/infrastructure-provider /infrastructure-provider
+COPY --from=helm /helm /usr/local/bin/helm
 EXPOSE 8081
 ENV PORT=8081
+# helm needs writable cache/config/data dirs; point them at the world-writable
+# /tmp so the operator can run helm as nonroot. The operator also writes the
+# runtime kubeconfig to a /tmp temp file (os.CreateTemp).
+ENV HELM_CACHE_HOME=/tmp/helm/cache \
+    HELM_CONFIG_HOME=/tmp/helm/config \
+    HELM_DATA_HOME=/tmp/helm/data
 USER nonroot:nonroot
 ENTRYPOINT ["/infrastructure-provider"]