Skip to content

Commit c3bec03

Browse files
mjudeikis-botOpenClaw Botmjudeikis
authored
feat: persist hub kubeconfig to Secret for in-cluster agents (#123)
When kedge-agent runs in a Kubernetes Deployment (uid 65534 / nobody, readOnlyRootFilesystem) the bootstrap join-token flow fails because it cannot write ~/.kedge/agent-<name>.kubeconfig on a read-only root FS. Solution: Secret-based kubeconfig persistence. 1. cmd/kedge-agent/incluster.go – new file with helpers: - isInCluster(): detects Kubernetes Pod via KUBERNETES_SERVICE_HOST - loadKubeconfigFromSecret(): reads hub kubeconfig from a namespaced Secret - writeKubeconfigToTempFile(): writes kubeconfig to /tmp (writable) 2. cmd/kedge-agent/main.go – on startup, when a bootstrap token is set and the process is in-cluster, attempt to load a previously saved kubeconfig from the Secret. If found, swap token → kubeconfig path so the agent reconnects without a new token exchange. 3. pkg/agent/incluster.go – new file with exported helpers for the agent package: IsInCluster(), SaveKubeconfigToSecret(). 4. pkg/agent/agent.go – onAgentToken callbacks (kubernetes mode and server mode) now additionally call SaveKubeconfigToSecret when in-cluster. After a successful save the process calls os.Exit(1) to force a pod restart; the new pod finds the populated Secret and runs normally without the bootstrap token. 5. pkg/cli/cmd/install.go – installKubernetes manifest template now creates per-edge ServiceAccount, empty kubeconfig Secret, Role, and RoleBinding so the agent pod can read/update only its own Secret. Deployment and ServiceAccount names are now scoped per-edge (kedge-agent-<edgeName>). 6. deploy/charts/kedge-agent/templates/secret-agent-kubeconfig.yaml – Helm template for the empty placeholder Secret (only when token mode). 7. deploy/charts/kedge-agent/templates/role-kubeconfig.yaml – Helm template for the namespaced Role + RoleBinding granting the agent's ServiceAccount get/update/patch on only its kubeconfig Secret. Co-authored-by: OpenClaw Bot <bot@openclaw.ai> Co-authored-by: Mangirdas Judeikis <mangirdas@judeikis.lt>
1 parent 8e98d13 commit c3bec03

7 files changed

Lines changed: 350 additions & 3 deletions

File tree

cmd/kedge-agent/incluster.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
Copyright 2026 The Faros Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
24+
apierrors "k8s.io/apimachinery/pkg/api/errors"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/client-go/kubernetes"
27+
"k8s.io/client-go/rest"
28+
)
29+
30+
const (
31+
// inClusterNamespace is the namespace where the agent Secret is stored.
32+
inClusterNamespace = "kedge-system"
33+
// kubeconfigSecretKey is the data key in the Secret that holds the kubeconfig.
34+
kubeconfigSecretKey = "kubeconfig"
35+
)
36+
37+
// isInCluster returns true when the process is running inside a Kubernetes Pod
38+
// (detected by the presence of the KUBERNETES_SERVICE_HOST environment variable).
39+
func isInCluster() bool {
40+
return os.Getenv("KUBERNETES_SERVICE_HOST") != ""
41+
}
42+
43+
// agentKubeconfigSecretName returns the name of the Secret used to persist
44+
// the hub kubeconfig for the given edge.
45+
func agentKubeconfigSecretName(edgeName string) string {
46+
return "kedge-agent-" + edgeName + "-kubeconfig"
47+
}
48+
49+
// newInClusterKubernetesClient builds a Kubernetes clientset using the
50+
// in-cluster service account credentials.
51+
func newInClusterKubernetesClient() (*kubernetes.Clientset, error) {
52+
cfg, err := rest.InClusterConfig()
53+
if err != nil {
54+
return nil, fmt.Errorf("building in-cluster config: %w", err)
55+
}
56+
cs, err := kubernetes.NewForConfig(cfg)
57+
if err != nil {
58+
return nil, fmt.Errorf("creating in-cluster kubernetes client: %w", err)
59+
}
60+
return cs, nil
61+
}
62+
63+
// loadKubeconfigFromSecret fetches the hub kubeconfig from the in-cluster Secret.
64+
// Returns ("", nil) when the Secret exists but the kubeconfig key is empty or absent.
65+
func loadKubeconfigFromSecret(edgeName string) (string, error) {
66+
cs, err := newInClusterKubernetesClient()
67+
if err != nil {
68+
return "", err
69+
}
70+
secret, err := cs.CoreV1().Secrets(inClusterNamespace).Get(
71+
context.Background(),
72+
agentKubeconfigSecretName(edgeName),
73+
metav1.GetOptions{},
74+
)
75+
if apierrors.IsNotFound(err) {
76+
return "", nil
77+
}
78+
if err != nil {
79+
return "", fmt.Errorf("getting kubeconfig secret: %w", err)
80+
}
81+
return string(secret.Data[kubeconfigSecretKey]), nil
82+
}
83+
84+
// writeKubeconfigToTempFile writes kubeconfig content to a temp file under /tmp
85+
// and returns the path. The file persists for the lifetime of the process.
86+
func writeKubeconfigToTempFile(data string) (string, error) {
87+
f, err := os.CreateTemp("/tmp", "kedge-agent-kubeconfig-*.yaml")
88+
if err != nil {
89+
return "", fmt.Errorf("creating temp kubeconfig file: %w", err)
90+
}
91+
name := f.Name()
92+
_, writeErr := f.WriteString(data)
93+
closeErr := f.Close()
94+
if writeErr != nil {
95+
return "", fmt.Errorf("writing temp kubeconfig file: %w", writeErr)
96+
}
97+
if closeErr != nil {
98+
return "", fmt.Errorf("closing temp kubeconfig file: %w", closeErr)
99+
}
100+
return name, nil
101+
}

cmd/kedge-agent/main.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,27 @@ func main() {
4242

4343
logger := klog.FromContext(ctx)
4444

45+
// In-cluster kubeconfig recovery: when running inside a Kubernetes Pod
46+
// with a bootstrap token, check whether a prior run already saved the
47+
// hub kubeconfig to the well-known Secret. If so, use it directly
48+
// instead of performing a new token exchange.
49+
if opts.Token != "" && opts.EdgeName != "" && isInCluster() {
50+
kc, err := loadKubeconfigFromSecret(opts.EdgeName)
51+
if err != nil {
52+
logger.Info("Could not check in-cluster kubeconfig Secret", "err", err)
53+
} else if kc != "" {
54+
tmpPath, err := writeKubeconfigToTempFile(kc)
55+
if err != nil {
56+
logger.Error(err, "failed to write in-cluster kubeconfig to temp file")
57+
} else {
58+
logger.Info("Using kubeconfig from in-cluster Secret (previous registration)", "edgeName", opts.EdgeName, "path", tmpPath)
59+
opts.HubKubeconfig = tmpPath
60+
opts.Token = ""
61+
opts.UsingSavedKubeconfig = true
62+
}
63+
}
64+
}
65+
4566
// Token-exchange: if a saved kubeconfig exists from a previous
4667
// join-token registration, use it instead of the bootstrap token.
4768
if opts.HubKubeconfig == "" && opts.EdgeName != "" {
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{{- if .Values.agent.hub.token }}
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: Role
4+
metadata:
5+
name: {{ include "kedge-agent.fullname" . }}-kubeconfig
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "kedge-agent.labels" . | nindent 4 }}
9+
rules:
10+
- apiGroups: [""]
11+
resources: ["secrets"]
12+
resourceNames: ["kedge-agent-{{ .Values.agent.edgeName }}-kubeconfig"]
13+
verbs: ["get", "update", "patch"]
14+
---
15+
apiVersion: rbac.authorization.k8s.io/v1
16+
kind: RoleBinding
17+
metadata:
18+
name: {{ include "kedge-agent.fullname" . }}-kubeconfig
19+
namespace: {{ .Release.Namespace }}
20+
labels:
21+
{{- include "kedge-agent.labels" . | nindent 4 }}
22+
subjects:
23+
- kind: ServiceAccount
24+
name: {{ include "kedge-agent.serviceAccountName" . }}
25+
namespace: {{ .Release.Namespace }}
26+
roleRef:
27+
kind: Role
28+
name: {{ include "kedge-agent.fullname" . }}-kubeconfig
29+
apiGroup: rbac.authorization.k8s.io
30+
{{- end }}
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{{- if .Values.agent.hub.token }}
2+
apiVersion: v1
3+
kind: Secret
4+
metadata:
5+
name: kedge-agent-{{ .Values.agent.edgeName }}-kubeconfig
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "kedge-agent.labels" . | nindent 4 }}
9+
type: Opaque
10+
data: {}
11+
{{- end }}

pkg/agent/agent.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,19 @@ func (a *Agent) runKubernetesMode(ctx context.Context, logger klog.Logger, hubCl
392392
if err := SaveAgentKubeconfig(a.opts.EdgeName, kubeconfigB64); err != nil {
393393
logger.Error(err, "failed to save agent kubeconfig from hub")
394394
}
395+
// In-cluster mode: also persist to Secret so it survives pod restarts,
396+
// then force a restart so the agent re-launches with the saved kubeconfig.
397+
if IsInCluster() {
398+
kubeconfigData, decErr := decodeKubeconfigB64(kubeconfigB64)
399+
if decErr != nil {
400+
logger.Error(decErr, "failed to decode kubeconfig for in-cluster Secret save")
401+
} else if saveErr := SaveKubeconfigToSecret(a.opts.EdgeName, kubeconfigData); saveErr != nil {
402+
logger.Error(saveErr, "failed to save kubeconfig to in-cluster Secret")
403+
} else {
404+
logger.Info("Saved kubeconfig to in-cluster Secret; restarting pod to activate", "edgeName", a.opts.EdgeName)
405+
os.Exit(1)
406+
}
407+
}
395408
}
396409
go tunnel.StartProxyTunnel(ctx, tunnelURL, a.hubConfig.BearerToken, a.opts.EdgeName, "edges", a.downstreamConfig, a.hubTLSConfig, tunnelState, a.opts.SSHProxyPort, clusterName, onAgentToken, nil)
397410

@@ -482,6 +495,18 @@ func (a *Agent) runServerMode(ctx context.Context, logger klog.Logger, hubClient
482495
if err := SaveAgentKubeconfig(a.opts.EdgeName, kubeconfigB64); err != nil {
483496
logger.Error(err, "failed to save agent kubeconfig from hub")
484497
}
498+
// In-cluster mode: also persist to Secret and restart.
499+
if IsInCluster() {
500+
kubeconfigData, decErr := decodeKubeconfigB64(kubeconfigB64)
501+
if decErr != nil {
502+
logger.Error(decErr, "failed to decode kubeconfig for in-cluster Secret save")
503+
} else if saveErr := SaveKubeconfigToSecret(a.opts.EdgeName, kubeconfigData); saveErr != nil {
504+
logger.Error(saveErr, "failed to save kubeconfig to in-cluster Secret")
505+
} else {
506+
logger.Info("Saved kubeconfig to in-cluster Secret; restarting pod to activate", "edgeName", a.opts.EdgeName)
507+
os.Exit(1)
508+
}
509+
}
485510
}
486511

487512
// In join-token mode, pass SSH credentials as WebSocket headers so the hub

pkg/agent/incluster.go

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
Copyright 2026 The Faros Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package agent
18+
19+
import (
20+
"context"
21+
"encoding/base64"
22+
"fmt"
23+
"os"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
apierrors "k8s.io/apimachinery/pkg/api/errors"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/client-go/kubernetes"
29+
"k8s.io/client-go/rest"
30+
)
31+
32+
const (
33+
// inClusterNamespace is the namespace where the agent kubeconfig Secret is stored.
34+
inClusterNamespace = "kedge-system"
35+
// kubeconfigSecretKey is the data key within the Secret.
36+
kubeconfigSecretKey = "kubeconfig"
37+
)
38+
39+
// IsInCluster returns true when the process is running inside a Kubernetes Pod.
40+
func IsInCluster() bool {
41+
return os.Getenv("KUBERNETES_SERVICE_HOST") != ""
42+
}
43+
44+
// AgentKubeconfigSecretName returns the name of the Secret used to persist
45+
// the hub kubeconfig for the given edge when running in-cluster.
46+
func AgentKubeconfigSecretName(edgeName string) string {
47+
return "kedge-agent-" + edgeName + "-kubeconfig"
48+
}
49+
50+
// newInClusterKubernetesClient builds a Kubernetes clientset using in-cluster
51+
// service-account credentials.
52+
func newInClusterKubernetesClient() (*kubernetes.Clientset, error) {
53+
cfg, err := rest.InClusterConfig()
54+
if err != nil {
55+
return nil, fmt.Errorf("building in-cluster config: %w", err)
56+
}
57+
cs, err := kubernetes.NewForConfig(cfg)
58+
if err != nil {
59+
return nil, fmt.Errorf("creating in-cluster kubernetes client: %w", err)
60+
}
61+
return cs, nil
62+
}
63+
64+
// decodeKubeconfigB64 decodes a base64-encoded kubeconfig string as returned
65+
// by the hub's token-exchange header.
66+
func decodeKubeconfigB64(kubeconfigB64 string) (string, error) {
67+
b, err := base64.StdEncoding.DecodeString(kubeconfigB64)
68+
if err != nil {
69+
return "", fmt.Errorf("decoding base64 kubeconfig: %w", err)
70+
}
71+
return string(b), nil
72+
}
73+
74+
// SaveKubeconfigToSecret writes the hub kubeconfig to the in-cluster Secret so
75+
// that it survives a pod restart.
76+
func SaveKubeconfigToSecret(edgeName, kubeconfigData string) error {
77+
cs, err := newInClusterKubernetesClient()
78+
if err != nil {
79+
return err
80+
}
81+
secretName := AgentKubeconfigSecretName(edgeName)
82+
existing, err := cs.CoreV1().Secrets(inClusterNamespace).Get(
83+
context.Background(),
84+
secretName,
85+
metav1.GetOptions{},
86+
)
87+
if apierrors.IsNotFound(err) {
88+
_, err = cs.CoreV1().Secrets(inClusterNamespace).Create(
89+
context.Background(),
90+
&corev1.Secret{
91+
ObjectMeta: metav1.ObjectMeta{
92+
Name: secretName,
93+
Namespace: inClusterNamespace,
94+
},
95+
Type: corev1.SecretTypeOpaque,
96+
Data: map[string][]byte{
97+
kubeconfigSecretKey: []byte(kubeconfigData),
98+
},
99+
},
100+
metav1.CreateOptions{},
101+
)
102+
if err != nil {
103+
return fmt.Errorf("creating kubeconfig secret: %w", err)
104+
}
105+
return nil
106+
}
107+
if err != nil {
108+
return fmt.Errorf("getting kubeconfig secret: %w", err)
109+
}
110+
if existing.Data == nil {
111+
existing.Data = make(map[string][]byte)
112+
}
113+
existing.Data[kubeconfigSecretKey] = []byte(kubeconfigData)
114+
_, err = cs.CoreV1().Secrets(inClusterNamespace).Update(
115+
context.Background(),
116+
existing,
117+
metav1.UpdateOptions{},
118+
)
119+
if err != nil {
120+
return fmt.Errorf("updating kubeconfig secret: %w", err)
121+
}
122+
return nil
123+
}

0 commit comments

Comments
 (0)