feat(infrastructure): operator defaults runtime to in-cluster + grants SA RBAC (#334)

mjudeikis · claude · web-flow · commit 22ac514a4681 · 2026-06-21T17:46:47.000+03:00
Make spec.runtimeKubeconfigSecret optional: when omitted the operator uses its
own cluster (in-cluster config) as the runtime — kro + the serve Deployment land
in the cluster the operator runs in. With it set, behavior is unchanged.

- CRD: runtimeKubeconfigSecret +optional (regenerated CRD YAML + chart copy)
- reconciler: fall back to the manager's rest.Config when no runtime Secret;
  run helm with no KUBECONFIG override (in-cluster creds) in that case
- serve: in-cluster runtime drops the runtime-kubeconfig mount/KRO_KUBECONFIG and
  runs the pod under a ServiceAccount bound to cluster-admin (ensureServeRBAC)
- controller_manager: kro backend falls back to in-cluster when KRO_KUBECONFIG
  is unset and running in a pod
- chart: operator SA bound to cluster-admin (operator.clusterAdmin, default true)
  so it can helm-install kro (ClusterRoles/CRDs) + manage runtime workloads;
  operator-config CR omits runtimeKubeconfigSecret when none is provided

Co-authored-by: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/providers/infrastructure/apis/v1alpha1/types_infrastructureprovider.go b/providers/infrastructure/apis/v1alpha1/types_infrastructureprovider.go
@@ -61,8 +61,12 @@ type InfrastructureProviderSpec struct {
 	ProviderKubeconfigSecret SecretKeyRef `json:"providerKubeconfigSecret"`
 
 	// RuntimeKubeconfigSecret references a Secret holding the kubeconfig of the
-	// cluster where kro and the provider serve Deployment run.
-	RuntimeKubeconfigSecret SecretKeyRef `json:"runtimeKubeconfigSecret"`
+	// cluster where kro and the provider serve Deployment run. Optional: when
+	// omitted the operator uses its own cluster (in-cluster config) as the
+	// runtime — i.e. kro + the serve Deployment land in the cluster the operator
+	// runs in.
+	// +optional
+	RuntimeKubeconfigSecret SecretKeyRef `json:"runtimeKubeconfigSecret,omitempty"`
 
 	// Hub configures the provider's heartbeat target. Optional.
 	// +optional
diff --git a/providers/infrastructure/config/crds/infrastructure.kedge.faros.sh_infrastructureproviders.yaml b/providers/infrastructure/config/crds/infrastructure.kedge.faros.sh_infrastructureproviders.yaml
@@ -193,7 +193,10 @@ spec:
               runtimeKubeconfigSecret:
                 description: |-
                   RuntimeKubeconfigSecret references a Secret holding the kubeconfig of the
-                  cluster where kro and the provider serve Deployment run.
+                  cluster where kro and the provider serve Deployment run. Optional: when
+                  omitted the operator uses its own cluster (in-cluster config) as the
+                  runtime — i.e. kro + the serve Deployment land in the cluster the operator
+                  runs in.
                 properties:
                   key:
                     default: kubeconfig
@@ -210,7 +213,6 @@ spec:
             - provider
             - providerKubeconfigSecret
             - providerWorkspace
-            - runtimeKubeconfigSecret
             type: object
           status:
             description: InfrastructureProviderStatus reports the operator's reconcile
diff --git a/providers/infrastructure/controller_manager.go b/providers/infrastructure/controller_manager.go
@@ -107,21 +107,32 @@ func startControllerManager(ctx context.Context, config *rest.Config) error {
 	// kcp workspace. It needs a separate client; KRO_KUBECONFIG points at
 	// that cluster (the same kubeconfig the legacy kro broker reads). When
 	// unset we run stub-only so dev/REST-only flows still boot.
+	// Resolve the kro runtime cluster: explicit KRO_KUBECONFIG, else the pod's
+	// in-cluster config (the operator's in-cluster-runtime mode — serve runs in
+	// the runtime cluster and authors RGDs against it via its pod SA). Falls
+	// back to stub-only when neither is available (dev/REST-only).
+	var kroCfg *rest.Config
+	var kroSrc string
 	if p := os.Getenv("KRO_KUBECONFIG"); p != "" {
-		kroCfg, err := clientcmd.BuildConfigFromFlags("", p)
+		c, err := clientcmd.BuildConfigFromFlags("", p)
 		if err != nil {
 			return fmt.Errorf("loading KRO_KUBECONFIG for kro backend: %w", err)
 		}
+		kroCfg, kroSrc = c, "KRO_KUBECONFIG="+p
+	} else if c, err := rest.InClusterConfig(); err == nil {
+		kroCfg, kroSrc = c, "in-cluster"
+	}
+	if kroCfg != nil {
 		kroDyn, err := dynamic.NewForConfig(kroCfg)
 		if err != nil {
 			return fmt.Errorf("kro backend dynamic client: %w", err)
 		}
 		if err := registry.Register(krobackend.New(kroDyn)); err != nil {
 			return fmt.Errorf("register kro backend: %w", err)
 		}
-		log.Printf("controller manager: kro backend registered (RGD runtime cluster from KRO_KUBECONFIG=%s)", p)
+		log.Printf("controller manager: kro backend registered (RGD runtime cluster: %s)", kroSrc)
 	} else {
-		log.Printf("controller manager: KRO_KUBECONFIG unset — kro backend not registered (stub-only)")
+		log.Printf("controller manager: no kro runtime config (KRO_KUBECONFIG unset, not in a pod) — kro backend not registered (stub-only)")
 	}
 
 	dyn, err := dynamic.NewForConfig(config)
diff --git a/providers/infrastructure/deploy/chart/crds/infrastructure.kedge.faros.sh_infrastructureproviders.yaml b/providers/infrastructure/deploy/chart/crds/infrastructure.kedge.faros.sh_infrastructureproviders.yaml
@@ -193,7 +193,10 @@ spec:
               runtimeKubeconfigSecret:
                 description: |-
                   RuntimeKubeconfigSecret references a Secret holding the kubeconfig of the
-                  cluster where kro and the provider serve Deployment run.
+                  cluster where kro and the provider serve Deployment run. Optional: when
+                  omitted the operator uses its own cluster (in-cluster config) as the
+                  runtime — i.e. kro + the serve Deployment land in the cluster the operator
+                  runs in.
                 properties:
                   key:
                     default: kubeconfig
@@ -210,7 +213,6 @@ spec:
             - provider
             - providerKubeconfigSecret
             - providerWorkspace
-            - runtimeKubeconfigSecret
             type: object
           status:
             description: InfrastructureProviderStatus reports the operator's reconcile
diff --git a/providers/infrastructure/deploy/chart/templates/operator-config.yaml b/providers/infrastructure/deploy/chart/templates/operator-config.yaml
@@ -44,9 +44,14 @@ spec:
   providerKubeconfigSecret:
     name: {{ $providerSecret }}
     key: {{ .Values.operator.providerKubeconfigSecret.key | default "kubeconfig" }}
+  {{- if or .Values.operator.runtimeKubeconfig .Values.operator.runtimeKubeconfigSecret.name }}
   runtimeKubeconfigSecret:
     name: {{ $runtimeSecret }}
     key: {{ .Values.operator.runtimeKubeconfigSecret.key | default "kubeconfig" }}
+  {{- else }}
+  # No runtimeKubeconfigSecret → the operator uses its own (in-cluster) cluster
+  # as the runtime (kro + serve land in the cluster the operator runs in).
+  {{- end }}
   {{- with .Values.hub }}
   hub:
     url: {{ .url | quote }}
diff --git a/providers/infrastructure/deploy/chart/templates/operator.yaml b/providers/infrastructure/deploy/chart/templates/operator.yaml
@@ -44,6 +44,28 @@ subjects:
   - kind: ServiceAccount
     name: {{ include "infrastructure.fullname" . }}-operator
     namespace: {{ .Release.Namespace }}
+{{- if .Values.operator.clusterAdmin }}
+---
+# The operator helm-installs the kro chart (which creates ClusterRoles, CRDs and
+# cluster-scoped objects) and, for the in-cluster runtime, manages namespaces,
+# Secrets, Deployments + the serve SA/binding on its own cluster — so its SA
+# needs cluster-admin. Set operator.clusterAdmin=false to bind a narrower role
+# yourself (e.g. when an explicit runtimeKubeconfig with its own creds is used).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "infrastructure.fullname" . }}-operator-admin
+  labels:
+    {{- include "infrastructure.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "infrastructure.fullname" . }}-operator
+    namespace: {{ .Release.Namespace }}
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
diff --git a/providers/infrastructure/deploy/chart/values.yaml b/providers/infrastructure/deploy/chart/values.yaml
@@ -138,6 +138,12 @@ affinity: {}
 operator:
   enabled: false
 
+  # Bind the operator ServiceAccount to cluster-admin. Required for the operator
+  # to helm-install the kro chart (which creates ClusterRoles/CRDs) and to manage
+  # runtime workloads in its own cluster (in-cluster runtime). Set false to wire
+  # a narrower role yourself.
+  clusterAdmin: true
+
   # Operator (controller) image. Defaults to the provider image above.
   image:
     repository: ""
diff --git a/providers/infrastructure/operator/controller.go b/providers/infrastructure/operator/controller.go
@@ -44,6 +44,10 @@ type Reconciler struct {
 	// Client reads CRs + referenced Secrets from the cluster the operator runs
 	// in (where the CRs live).
 	Client client.Client
+	// RestConfig is the operator's own cluster config (what the manager was
+	// built with). Used as the runtime cluster when a CR omits
+	// spec.runtimeKubeconfigSecret — i.e. "use the current context".
+	RestConfig *rest.Config
 }
 
 // Reconcile drives one CR to its desired state.
@@ -60,10 +64,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if err != nil {
 		return r.fail(ctx, &cr, v1alpha1.ConditionBootstrapped, "ProviderKubeconfigMissing", err)
 	}
-	runtimeKC, err := r.secretValue(ctx, cr.Namespace, cr.Spec.RuntimeKubeconfigSecret)
-	if err != nil {
-		return r.fail(ctx, &cr, v1alpha1.ConditionBootstrapped, "RuntimeKubeconfigMissing", err)
-	}
 	var hubToken []byte
 	if cr.Spec.Hub.TokenSecret != nil {
 		if hubToken, err = r.secretValue(ctx, cr.Namespace, *cr.Spec.Hub.TokenSecret); err != nil {
@@ -75,9 +75,30 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if err != nil {
 		return r.fail(ctx, &cr, v1alpha1.ConditionBootstrapped, "ProviderKubeconfigInvalid", err)
 	}
-	runtimeCfg, err := clientcmd.RESTConfigFromKubeConfig(runtimeKC)
-	if err != nil {
-		return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigInvalid", err)
+
+	// Runtime cluster: an explicit kubeconfig Secret, or — when omitted — the
+	// operator's own cluster (in-cluster / current context). In the in-cluster
+	// case runtimeKC stays nil: helm runs without a KUBECONFIG override (using
+	// its in-cluster credentials) and the serve Deployment uses its pod SA.
+	var (
+		runtimeCfg *rest.Config
+		runtimeKC  []byte
+	)
+	if cr.Spec.RuntimeKubeconfigSecret.Name != "" {
+		runtimeKC, err = r.secretValue(ctx, cr.Namespace, cr.Spec.RuntimeKubeconfigSecret)
+		if err != nil {
+			return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigMissing", err)
+		}
+		runtimeCfg, err = clientcmd.RESTConfigFromKubeConfig(runtimeKC)
+		if err != nil {
+			return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigInvalid", err)
+		}
+	} else {
+		if r.RestConfig == nil {
+			return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigMissing",
+				fmt.Errorf("no runtimeKubeconfigSecret and operator has no in-cluster config"))
+		}
+		runtimeCfg = r.RestConfig
 	}
 
 	// 1. Bootstrap the provider workspace.
@@ -100,12 +121,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if err := install.SeedKroClusterFromKubeconfig(ctx, runtimeCfg, providerKC, cr.Spec.ProviderWorkspace); err != nil {
 		return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "KroSeedFailed", err)
 	}
-	tmp, cleanup, err := writeTempKubeconfig(runtimeKC)
-	if err != nil {
-		return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigWriteFailed", err)
+	// helm needs a KUBECONFIG file only for an explicit runtime; for the
+	// in-cluster runtime (runtimeKC nil) we run helm with no override so it uses
+	// its in-cluster service account.
+	helmKubeconfig := ""
+	if runtimeKC != nil {
+		tmp, cleanup, werr := writeTempKubeconfig(runtimeKC)
+		if werr != nil {
+			return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "RuntimeKubeconfigWriteFailed", werr)
+		}
+		defer cleanup()
+		helmKubeconfig = tmp
 	}
-	defer cleanup()
-	if err := EnsureKroRelease(ctx, tmp, cr.Spec.Kro); err != nil {
+	if err := EnsureKroRelease(ctx, helmKubeconfig, cr.Spec.Kro); err != nil {
 		return r.fail(ctx, &cr, v1alpha1.ConditionKroReleased, "HelmFailed", err)
 	}
 	// Dev-only kind networking patches (no-op in prod; gated by env). Lets the
@@ -190,7 +218,7 @@ func Run(ctx context.Context, cfg *rest.Config) error {
 	if err != nil {
 		return fmt.Errorf("manager.New: %w", err)
 	}
-	if err := (&Reconciler{Client: mgr.GetClient()}).SetupWithManager(mgr); err != nil {
+	if err := (&Reconciler{Client: mgr.GetClient(), RestConfig: cfg}).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("setup reconciler: %w", err)
 	}
 	klog.FromContext(ctx).Info("infrastructure operator manager starting")
diff --git a/providers/infrastructure/operator/helm.go b/providers/infrastructure/operator/helm.go
@@ -52,7 +52,11 @@ func EnsureKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1a
 	}
 
 	cmd := exec.CommandContext(ctx, "helm", args...)
-	cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)
+	// An empty path means "in-cluster runtime": run helm with no KUBECONFIG
+	// override so it uses the pod's in-cluster service account.
+	if runtimeKubeconfigPath != "" {
+		cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)
+	}
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("helm upgrade --install %s: %w\n%s", kro.ReleaseName, err, string(out))
 	}
diff --git a/providers/infrastructure/operator/serve.go b/providers/infrastructure/operator/serve.go
@@ -16,6 +16,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -52,13 +53,14 @@ func EnsureProviderServe(
 
 	name := cr.Name
 	providerSecret := name + "-provider-kubeconfig"
-	runtimeSecret := name + "-runtime-kubeconfig"
 	if err := upsertOpaqueSecret(ctx, cs, ServeNamespace, providerSecret, "kubeconfig", providerKubeconfig); err != nil {
 		return fmt.Errorf("replicate provider kubeconfig: %w", err)
 	}
-	if err := upsertOpaqueSecret(ctx, cs, ServeNamespace, runtimeSecret, "kubeconfig", runtimeKubeconfig); err != nil {
-		return fmt.Errorf("replicate runtime kubeconfig: %w", err)
-	}
+
+	// inCluster: the runtime is the operator's own cluster (no runtime
+	// kubeconfig). The serve pod then runs the kro backend with its pod
+	// ServiceAccount (in-cluster) instead of a mounted runtime kubeconfig.
+	inCluster := len(runtimeKubeconfig) == 0
 
 	port := cr.Spec.Provider.Port
 	if port == 0 {
@@ -73,7 +75,6 @@ func EnsureProviderServe(
 		{Name: "PORT", Value: fmt.Sprintf("%d", port)},
 		{Name: "KEDGE_PROVIDER_NAME", Value: "infrastructure"},
 		{Name: "INFRASTRUCTURE_KUBECONFIG", Value: providerKubeconfigMount},
-		{Name: "KRO_KUBECONFIG", Value: runtimeKubeconfigMount},
 	}
 	if cr.Spec.Hub.URL != "" {
 		env = append(env, corev1.EnvVar{Name: "KEDGE_HUB_URL", Value: cr.Spec.Hub.URL})
@@ -83,11 +84,30 @@ func EnsureProviderServe(
 	}
 	volMounts := []corev1.VolumeMount{
 		{Name: "provider-kubeconfig", MountPath: "/var/run/secrets/kedge/provider", ReadOnly: true},
-		{Name: "runtime-kubeconfig", MountPath: "/var/run/secrets/kedge/runtime", ReadOnly: true},
 	}
 	volumes := []corev1.Volume{
 		{Name: "provider-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: providerSecret}}},
-		{Name: "runtime-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: runtimeSecret}}},
+	}
+	// Serve's kro backend reaches the runtime cluster either via a mounted
+	// runtime kubeconfig (explicit runtime) or its in-cluster SA (in-cluster
+	// runtime — KRO_KUBECONFIG left unset; controller_manager falls back to
+	// in-cluster).
+	serveSA := ""
+	if !inCluster {
+		runtimeSecret := name + "-runtime-kubeconfig"
+		if err := upsertOpaqueSecret(ctx, cs, ServeNamespace, runtimeSecret, "kubeconfig", runtimeKubeconfig); err != nil {
+			return fmt.Errorf("replicate runtime kubeconfig: %w", err)
+		}
+		env = append(env, corev1.EnvVar{Name: "KRO_KUBECONFIG", Value: runtimeKubeconfigMount})
+		volMounts = append(volMounts, corev1.VolumeMount{Name: "runtime-kubeconfig", MountPath: "/var/run/secrets/kedge/runtime", ReadOnly: true})
+		volumes = append(volumes, corev1.Volume{Name: "runtime-kubeconfig", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: runtimeSecret}}})
+	} else {
+		// Give the serve pod an SA bound to the access its kro backend needs on
+		// the (operator's own) runtime cluster.
+		serveSA = name
+		if err := ensureServeRBAC(ctx, cs, serveSA); err != nil {
+			return fmt.Errorf("serve RBAC: %w", err)
+		}
 	}
 	if cr.Spec.Hub.TokenSecret != nil && len(hubToken) > 0 {
 		hubSecret := name + "-hub-token"
@@ -117,6 +137,7 @@ func EnsureProviderServe(
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{Labels: labels},
 				Spec: corev1.PodSpec{
+					ServiceAccountName: serveSA,
 					Containers: []corev1.Container{{
 						Name:         "provider",
 						Image:        image,
@@ -166,6 +187,37 @@ func EnsureProviderServe(
 	return ensureServeService(ctx, cs, name, labels, port)
 }
 
+// ensureServeRBAC creates the serve pod's ServiceAccount (in ServeNamespace)
+// and binds it to cluster-admin so its in-cluster kro backend can author
+// RGD-defined instances, namespaces, and secrets on the runtime cluster. Used
+// only for the in-cluster runtime (no runtime kubeconfig). Scope down for
+// least privilege in hardened environments.
+func ensureServeRBAC(ctx context.Context, cs kubernetes.Interface, saName string) error {
+	sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName, Namespace: ServeNamespace}}
+	if _, err := cs.CoreV1().ServiceAccounts(ServeNamespace).Get(ctx, saName, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		if _, cerr := cs.CoreV1().ServiceAccounts(ServeNamespace).Create(ctx, sa, metav1.CreateOptions{}); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
+			return fmt.Errorf("create serve ServiceAccount: %w", cerr)
+		}
+	} else if err != nil {
+		return fmt.Errorf("get serve ServiceAccount: %w", err)
+	}
+
+	crbName := "kedge-infrastructure-serve-" + saName
+	crb := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{Name: crbName},
+		RoleRef:    rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"},
+		Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: ServeNamespace}},
+	}
+	if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, crbName, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		if _, cerr := cs.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
+			return fmt.Errorf("create serve ClusterRoleBinding: %w", cerr)
+		}
+	} else if err != nil {
+		return fmt.Errorf("get serve ClusterRoleBinding: %w", err)
+	}
+	return nil
+}
+
 func ensureServeService(ctx context.Context, cs kubernetes.Interface, name string, labels map[string]string, port int32) error {
 	want := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ServeNamespace, Labels: labels},

Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,11 @@ func EnsureKroRelease(ctx context.Context, runtimeKubeconfigPath string, kro v1a`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`cmd := exec.CommandContext(ctx, "helm", args...)`
`55`		`- cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)`
	`55`	`+ // An empty path means "in-cluster runtime": run helm with no KUBECONFIG`
	`56`	`+ // override so it uses the pod's in-cluster service account.`
	`57`	`+ if runtimeKubeconfigPath != "" {`
	`58`	`+ cmd.Env = append(os.Environ(), "KUBECONFIG="+runtimeKubeconfigPath)`
	`59`	`+ }`
`56`	`60`	`if out, err := cmd.CombinedOutput(); err != nil {`
`57`	`61`	`return fmt.Errorf("helm upgrade --install %s: %w\n%s", kro.ReleaseName, err, string(out))`
`58`	`62`	`}`