Support image pull secrets for Fleet controllers and agents (#5201)

weyfonk · web-flow · commit 6992451140bc · 2026-06-05T17:13:24.000+02:00
* Support image pull secrets for Fleet controllers and jobs

Fleet controller images can now be sourced from private registries
requiring authentication.
This can be used as follows:
1. Create a secret pointing to a private registry containing a Fleet
   controller image:
```
kubectl create secret docker-registry registry-creds \
    -n cattle-fleet-system \
    --docker-username=fleet-ci \
    --docker-password=foo \
    --docker-server=[my-server]:[my-port]
```
2. Install Fleet with:
```
--set image.repository=[my-server]:[my-port]/fleet
--set-json global.cattle.imagePullSecrets='[{"name": "registry-creds"}]'
```

All Fleet controllers (`fleet-controller`, `gitjob`, `helmops` and
created jobs (migrations, cleanups) will now source the controller image
from that registry.

* Propagate image pull secrets to agents

Agent deployments now support image pull secrets, which are copied into
the agent's namespace before the deployment is created.

The existing feature for automated copy of resources to downstream
clusters is not usable in this case, since it requires an already
running agent to copy the resource, in this case an image pull secret,
to the downstream cluster.

* Propagate controller image pull secrets to git job

This ensures that git jobs are able to run from images fetched from
private registries if needed.

Image pull secrets are copied into the git job's namespace, and owned by
the GitRepo.
diff --git a/charts/fleet/templates/configmap.yaml b/charts/fleet/templates/configmap.yaml
@@ -19,6 +19,9 @@ data:
       {{ if .Values.garbageCollectionInterval }}
       "garbageCollectionInterval": "{{.Values.garbageCollectionInterval}}",
       {{ end }}
+      {{ if .Values.global.cattle.imagePullSecrets }}
+      "imagePullSecrets": {{toJson .Values.global.cattle.imagePullSecrets}},
+      {{ end }}
       "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}},
       "bootstrap": {
         "paths": "{{.Values.bootstrap.paths}}",
diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
@@ -35,6 +35,10 @@ spec:
 {{ toYaml $.Values.extraAnnotations.fleetController | indent 8 }}
 {{- end }}
     spec:
+{{- if $.Values.global.cattle.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml $.Values.global.cattle.imagePullSecrets | indent 8 }}
+{{- end }}
       containers:
       - env:
         - name: NAMESPACE
diff --git a/charts/fleet/templates/deployment_gitjob.yaml b/charts/fleet/templates/deployment_gitjob.yaml
@@ -37,6 +37,10 @@ spec:
 {{- end }}
     spec:
       serviceAccountName: gitjob
+{{- if $.Values.global.cattle.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml $.Values.global.cattle.imagePullSecrets | indent 8 }}
+{{- end }}
       containers:
         - image: "{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}"
           name: gitjob
diff --git a/charts/fleet/templates/deployment_helmops.yaml b/charts/fleet/templates/deployment_helmops.yaml
@@ -37,6 +37,10 @@ spec:
 {{- end }}
     spec:
       serviceAccountName: helmops
+{{- if $.Values.global.cattle.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml $.Values.global.cattle.imagePullSecrets | indent 8 }}
+{{- end }}
       containers:
         - image: "{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}"
           name: helmops
diff --git a/charts/fleet/templates/job_cleanup_clusterregistrations.yaml b/charts/fleet/templates/job_cleanup_clusterregistrations.yaml
@@ -13,6 +13,10 @@ spec:
       labels:
         app: fleet-job
     spec:
+{{- if .Values.global.cattle.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.global.cattle.imagePullSecrets | indent 8 }}
+{{- end }}
       serviceAccountName: fleet-controller
       restartPolicy: Never
       securityContext:
diff --git a/charts/fleet/templates/job_cleanup_gitrepojobs.yaml b/charts/fleet/templates/job_cleanup_gitrepojobs.yaml
@@ -16,6 +16,10 @@ spec:
           labels:
             app: fleet-job
         spec:
+{{- if .Values.global.cattle.imagePullSecrets }}
+          imagePullSecrets:
+{{ toYaml .Values.global.cattle.imagePullSecrets | indent 12 }}
+{{- end }}
           serviceAccountName: gitjob
           restartPolicy: Never
           securityContext:
diff --git a/charts/fleet/templates/job_migrate_gitrepo_helmrepourl.yaml b/charts/fleet/templates/job_migrate_gitrepo_helmrepourl.yaml
@@ -13,6 +13,10 @@ spec:
       labels:
         app: fleet-job
     spec:
+{{- if .Values.global.cattle.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml .Values.global.cattle.imagePullSecrets | indent 8 }}
+{{- end }}
       serviceAccountName: fleet-controller
       restartPolicy: Never
       securityContext:
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
@@ -72,6 +72,7 @@ bootstrap:
 
 global:
   cattle:
+    imagePullSecrets: []
     systemDefaultRegistry: ""
 
 ## Node labels for pod assignment
diff --git a/internal/cmd/controller/agentmanagement/agent/agent.go b/internal/cmd/controller/agentmanagement/agent/agent.go
@@ -57,6 +57,26 @@ func AgentWithConfig(ctx context.Context, agentNamespace, controllerNamespace, a
 
 	objs = append(objs, secret)
 
+	// Copy image pull secrets, if any, to the agent namespace.
+	// This ensures that those secrets are available at agent deployment time.
+	for _, ips := range opts.ImagePullSecrets {
+		source, err := client.Core.Secret().Get(controllerNamespace, ips.Name, metav1.GetOptions{})
+		if err != nil {
+			return objs, fmt.Errorf("failed to get image pull secret %q from controller namespace: %w", ips.Name, err)
+		}
+
+		dest := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      ips.Name,
+				Namespace: agentNamespace,
+			},
+			Type: source.Type,
+			Data: source.Data,
+		}
+
+		objs = append(objs, dest)
+	}
+
 	agentConfig, err := agentConfig(ctx, agentNamespace, controllerNamespace, cg, &opts.ConfigOptions)
 	if err != nil {
 		return objs, err
@@ -73,6 +93,7 @@ func AgentWithConfig(ctx context.Context, agentNamespace, controllerNamespace, a
 	// keep in sync with manageagent.go
 	mo := opts.ManifestOptions
 	mo.AgentImage = cfg.AgentImage
+	mo.ImagePullSecrets = cfg.ImagePullSecrets
 	mo.AgentImagePullPolicy = cfg.AgentImagePullPolicy
 	mo.CheckinInterval = cfg.AgentCheckinInterval.Duration.String()
 	mo.SystemDefaultRegistry = cfg.SystemDefaultRegistry
diff --git a/internal/cmd/controller/agentmanagement/agent/manifest.go b/internal/cmd/controller/agentmanagement/agent/manifest.go
@@ -46,6 +46,7 @@ type ManifestOptions struct {
 	DriftWorkers            string
 	cmd.LeaderElectionOptions
 	PriorityClassName string
+	ImagePullSecrets  []corev1.LocalObjectReference
 }
 
 // Manifest builds and returns a deployment manifest for the fleet-agent with a
@@ -167,6 +168,7 @@ func agentApp(namespace string, agentScope string, opts ManifestOptions) *appsv1
 					},
 				},
 				Spec: corev1.PodSpec{
+					ImagePullSecrets:   opts.ImagePullSecrets,
 					PriorityClassName:  opts.PriorityClassName,
 					ServiceAccountName: serviceAccount,
 					Containers: []corev1.Container{
diff --git a/internal/cmd/controller/agentmanagement/controllers/cluster/import.go b/internal/cmd/controller/agentmanagement/controllers/cluster/import.go
@@ -459,6 +459,7 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
 				HostNetwork:       *cmp.Or(cluster.Spec.HostNetwork, new(false)),
 				AgentReplicas:     agentReplicas,
 				PriorityClassName: priorityClassName,
+				ImagePullSecrets:  cfg.ImagePullSecrets,
 			},
 		})
 	objs = append(objs, agentObjs...)
diff --git a/internal/cmd/controller/agentmanagement/controllers/manageagent/manageagent.go b/internal/cmd/controller/agentmanagement/controllers/manageagent/manageagent.go
@@ -350,6 +350,7 @@ func (h *handler) newAgentBundle(ns string, cluster *fleet.Cluster) (runtime.Obj
 			// keep in sync with agent/agent.go
 			AgentImage:              cfg.AgentImage,
 			AgentImagePullPolicy:    cfg.AgentImagePullPolicy,
+			ImagePullSecrets:        cfg.ImagePullSecrets,
 			CheckinInterval:         cfg.AgentCheckinInterval.Duration.String(),
 			SystemDefaultRegistry:   cfg.SystemDefaultRegistry,
 			BundleDeploymentWorkers: cfg.AgentWorkers.BundleDeployment,
@@ -364,6 +365,16 @@ func (h *handler) newAgentBundle(ns string, cluster *fleet.Cluster) (runtime.Obj
 		return nil, err
 	}
 
+	imagePullSecrets := make([]fleet.DownstreamResource, len(cfg.ImagePullSecrets))
+	for i, ips := range cfg.ImagePullSecrets {
+		// Gets can be skipped at manifest build time, will be handled by the bundle controller anyway.
+
+		imagePullSecrets[i] = fleet.DownstreamResource{
+			Kind: "secret",
+			Name: ips.Name,
+		}
+	}
+
 	return &fleet.Bundle{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      names.SafeConcatName(AgentBundleName, cluster.Name),
@@ -375,6 +386,9 @@ func (h *handler) newAgentBundle(ns string, cluster *fleet.Cluster) (runtime.Obj
 				Helm: &fleet.HelmOptions{
 					TakeOwnership: true,
 				},
+				// XXX: this may only be useful when updating an existing agent's image pull secrets (as an agent
+				// deployment would then already be running)
+				//DownstreamResources: imagePullSecrets,
 			},
 			Resources: []fleet.BundleResource{
 				{
diff --git a/internal/cmd/controller/gitops/reconciler/gitjob.go b/internal/cmd/controller/gitops/reconciler/gitjob.go
@@ -223,6 +223,44 @@ func (r *GitJobReconciler) newGitJob(ctx context.Context, obj *v1alpha1.GitRepo)
 	}
 	maps.Copy(jobSpec.Template.Spec.NodeSelector, fleetControllerDeployment.Spec.Template.Spec.NodeSelector)
 
+	jobSpec.Template.Spec.ImagePullSecrets = fleetControllerDeployment.Spec.Template.Spec.ImagePullSecrets
+
+	for _, ips := range fleetControllerDeployment.Spec.Template.Spec.ImagePullSecrets {
+		var secret corev1.Secret
+		err := r.Get(ctx, types.NamespacedName{
+			//Namespace: obj.Namespace,
+			Namespace: fleetControllerDeployment.Namespace,
+			Name:      ips.Name,
+		}, &secret)
+
+		if err != nil {
+			return nil, fmt.Errorf("could not get image pull secret %q: %w", ips.Name, err)
+		}
+
+		gitjobSecret := corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      ips.Name,
+				Namespace: obj.Namespace,
+			},
+			Data: secret.Data,
+			Type: secret.Type,
+		}
+
+		_, err = controllerutil.CreateOrUpdate(ctx, r.Client, &gitjobSecret, func() error {
+			gitjobSecret.Data = secret.Data
+			gitjobSecret.Type = secret.Type
+
+			if err := controllerutil.SetControllerReference(obj, &gitjobSecret, r.Scheme); err != nil {
+				return fmt.Errorf("failed to set gitrepo ownership on image pull secret %q: %w", ips.Name, err)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, fmt.Errorf("could not create or update image pull secret %q in git job namespace: %w", ips.Name, err)
+		}
+	}
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{
@@ -238,6 +276,7 @@ func (r *GitJobReconciler) newGitJob(ctx context.Context, obj *v1alpha1.GitRepo)
 		},
 		Spec: *jobSpec,
 	}
+
 	// if the repo references a shard, add the same label to the job
 	// this avoids a call to Reconcile for controllers that do not match
 	// the shard-id
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -156,6 +156,10 @@ type Config struct {
 
 	// AgentWorkers specifies the maximum number of workers for each agent reconciler.
 	AgentWorkers AgentWorkers `json:"agentWorkers,omitzero"`
+
+	// ImagePullSecrets references secrets located in the same namespace as the Fleet controller deployment, to be
+	// propagated to downstream clusters and used as image pull secrets for Fleet agent images.
+	ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 }
 
 type AgentWorkers struct {
