fix: derive kubelet clusterDNS for a custom serviceSubnet (C2)

internal/kubeadm/action/action.go

@@ -138,7 +138,21 @@ func (e *KubeadmExecutor) runInit(ctx context.Context) error {
 	}}
 	initCfg.CertificateKey = certKey
 
-	content, err := kubeadmconfig.Marshal(cluster, initCfg)
+	// C2: when the operator set a custom serviceSubnet, pin clusterDNS to the IP
+	// derived from it so the kubelet's DNS matches the service CIDR. The init-time
+	// KubeletConfiguration is uploaded to the kubelet-config ConfigMap by kubeadm,
+	// so joining workers inherit the same clusterDNS. Omitted for the default
+	// subnet (kubeadm derives it) so the default path is byte-for-byte unchanged.
+	kubeletCfg, err := kubeadmconfig.BuildKubeletConfiguration(e.Input)
+	if err != nil {
+		return err
+	}
+	docs := []any{cluster, initCfg}
+	if len(kubeletCfg.ClusterDNS) > 0 {
+		docs = append(docs, kubeletCfg)
+	}
+
+	content, err := kubeadmconfig.Marshal(docs...)
 	if err != nil {
 		return err
 	}
@@ -299,7 +313,19 @@ func (e *KubeadmExecutor) runUpgradeApply(ctx context.Context) error {
 func (e *KubeadmExecutor) runRepairKubeletConfig(ctx context.Context) error {
 	cluster := kubeadmconfig.BuildClusterConfiguration(e.Input)
 	initCfg := kubeadmconfig.BuildInitConfiguration(e.Input)
-	content, err := kubeadmconfig.Marshal(cluster, initCfg)
+	// C2/ADR-17: this regenerates /var/lib/kubelet/config.yaml, so it must pin the
+	// SAME clusterDNS we own at init for a custom serviceSubnet -- otherwise the
+	// repair would re-derive it implicitly and could disagree with the init-time
+	// value. Omitted (kubeadm derives) for the default subnet, as in runInit.
+	kubeletCfg, err := kubeadmconfig.BuildKubeletConfiguration(e.Input)
+	if err != nil {
+		return err
+	}
+	docs := []any{cluster, initCfg}
+	if len(kubeletCfg.ClusterDNS) > 0 {
+		docs = append(docs, kubeletCfg)
+	}
+	content, err := kubeadmconfig.Marshal(docs...)
 	if err != nil {
 		return err
 	}

internal/kubeadm/action/action_test.go

@@ -387,3 +387,82 @@ func TestRepairKubeletConfig_BoundedWait(t *testing.T) {
 		t.Fatal("expected a bounded error when the local API never returns")
 	}
 }
+
+// C2: with a custom serviceSubnet, runInit emits a KubeletConfiguration document
+// pinning clusterDNS to the derived DNS IP. With the default (empty) subnet it
+// emits no KubeletConfiguration (kubeadm derives clusterDNS itself).
+func TestRunInitEmitsKubeletClusterDNSForCustomSubnet(t *testing.T) {
+	capture := func(in kubeadmconfig.Input) string {
+		t.Helper()
+		var content string
+		fr := &fakeRunner{respond: func(args []string) (kubeadm.Result, error) {
+			switch {
+			case args[0] == "token" && args[1] == "generate":
+				return kubeadm.Result{Stdout: "abcdef.0123456789abcdef\n"}, nil
+			case args[0] == "certs" && args[1] == "certificate-key":
+				return kubeadm.Result{Stdout: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n"}, nil
+			case args[0] == "init":
+				b, _ := os.ReadFile(configArg(args))
+				content = string(b)
+			}
+			return kubeadm.Result{}, nil
+		}}
+		e := &KubeadmExecutor{
+			Runner: fr, Minter: credential.Minter{Runner: fr, RootPath: "/"},
+			RootPath: "/", RunDir: t.TempDir(), Role: actualstate.RoleInit,
+			Input: in, TokenTTL: time.Hour,
+		}
+		if err := e.Execute(context.Background(), reconcile.ActionRunInit); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		return content
+	}
+
+	custom := capture(kubeadmconfig.Input{
+		KubernetesVersion:    "v1.34.0",
+		ControlPlaneEndpoint: "10.0.0.1:6443",
+		ServiceSubnet:        "172.20.0.0/16",
+	})
+	if !strings.Contains(custom, "kind: KubeletConfiguration") {
+		t.Fatalf("custom subnet init config missing KubeletConfiguration:\n%s", custom)
+	}
+	if !strings.Contains(custom, "172.20.0.10") {
+		t.Fatalf("custom subnet init config missing derived clusterDNS 172.20.0.10:\n%s", custom)
+	}
+
+	def := capture(kubeadmconfig.Input{
+		KubernetesVersion:    "v1.34.0",
+		ControlPlaneEndpoint: "10.0.0.1:6443",
+	})
+	if strings.Contains(def, "kind: KubeletConfiguration") {
+		t.Fatalf("default subnet must NOT emit a KubeletConfiguration:\n%s", def)
+	}
+}
+
+// C2/ADR-17: the join path must NOT emit a KubeletConfiguration even with a custom
+// serviceSubnet -- joining nodes inherit clusterDNS from the kubelet-config
+// ConfigMap kubeadm uploaded at init. Locks the init-emits/join-omits split.
+func TestRunJoinOmitsKubeletConfiguration(t *testing.T) {
+	var content string
+	fr := &fakeRunner{respond: func(args []string) (kubeadm.Result, error) {
+		if args[0] == "join" {
+			b, _ := os.ReadFile(configArg(args))
+			content = string(b)
+		}
+		return kubeadm.Result{}, nil
+	}}
+	e := &KubeadmExecutor{
+		Runner: fr, RootPath: "/", RunDir: t.TempDir(), Role: actualstate.RoleWorker,
+		Input: kubeadmconfig.Input{ControlPlaneEndpoint: "10.0.0.1:6443", ServiceSubnet: "172.20.0.0/16"},
+		Join: &credential.JoinMaterial{
+			Token:        "abcdef.0123456789abcdef",
+			CACertHashes: []string{"sha256:deadbeef"},
+		},
+	}
+	if err := e.Execute(context.Background(), reconcile.ActionRunJoin); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if strings.Contains(content, "kind: KubeletConfiguration") {
+		t.Fatalf("join config must NOT contain a KubeletConfiguration:\n%s", content)
+	}
+}

internal/kubeadmconfig/build.go

@@ -51,6 +51,26 @@ func BuildClusterConfiguration(in Input) ClusterConfiguration {
 	return c
 }
 
+// BuildKubeletConfiguration builds a KubeletConfiguration that pins clusterDNS to
+// the IP derived from a CUSTOM serviceSubnet (pitfall C2). When ServiceSubnet is
+// empty it returns a config with NO clusterDNS so kubeadm derives it from its own
+// default service CIDR -- preserving today's default-path behavior exactly. We
+// only override clusterDNS when the operator customized serviceSubnet. Callers
+// should emit the document only when ClusterDNS is non-empty (see runInit); this
+// document is credential-free.
+func BuildKubeletConfiguration(in Input) (KubeletConfiguration, error) {
+	k := NewKubeletConfiguration()
+	if strings.TrimSpace(in.ServiceSubnet) == "" {
+		return k, nil
+	}
+	dns, err := DeriveDNSIP(in.ServiceSubnet)
+	if err != nil {
+		return KubeletConfiguration{}, fmt.Errorf("derive clusterDNS: %w", err)
+	}
+	k.ClusterDNS = []string{dns}
+	return k, nil
+}
+
 // BuildInitConfiguration builds the non-credential parts of an InitConfiguration.
 // BootstrapTokens and CertificateKey are populated later by the credential layer.
 func BuildInitConfiguration(in Input) InitConfiguration {

internal/kubeadmconfig/build_test.go

@@ -50,3 +50,32 @@ func TestBuildInitConfigurationHasNoCredentials(t *testing.T) {
 		t.Fatalf("non-credential builder must not set bootstrap tokens or certificate key")
 	}
 }
+
+func TestBuildKubeletConfiguration(t *testing.T) {
+	// Custom serviceSubnet -> clusterDNS pinned to the derived 10th host IP, with
+	// kubelet v1beta1 TypeMeta. clusterDNS is a single-element list.
+	k, err := BuildKubeletConfiguration(Input{ServiceSubnet: "172.20.0.0/16"})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if k.APIVersion != KubeletAPIVersion || k.Kind != KindKubeletConfiguration {
+		t.Fatalf("wrong TypeMeta: %+v", k.TypeMeta)
+	}
+	if len(k.ClusterDNS) != 1 || k.ClusterDNS[0] != "172.20.0.10" {
+		t.Fatalf("clusterDNS = %v, want [172.20.0.10]", k.ClusterDNS)
+	}
+
+	// Empty serviceSubnet -> no clusterDNS (default path: kubeadm derives it).
+	k2, err := BuildKubeletConfiguration(Input{ServiceSubnet: ""})
+	if err != nil {
+		t.Fatalf("unexpected error on empty subnet: %v", err)
+	}
+	if len(k2.ClusterDNS) != 0 {
+		t.Fatalf("expected no clusterDNS for empty serviceSubnet, got %v", k2.ClusterDNS)
+	}
+
+	// Malformed serviceSubnet -> error (fail fast, do not silently default).
+	if _, err := BuildKubeletConfiguration(Input{ServiceSubnet: "garbage"}); err == nil {
+		t.Fatal("expected error for malformed serviceSubnet")
+	}
+}

internal/kubeadmconfig/dns.go

@@ -0,0 +1,81 @@
+package kubeadmconfig
+
+import (
+	"fmt"
+	"net"
+	"strings"
+)
+
+// dnsIPOffset is the host index within the service CIDR that kubeadm assigns to
+// the cluster DNS (kube-dns/coredns) Service, mirroring kubeadm's GetDNSIP, which
+// is GetIndexedIP(serviceCIDR, 10). We compute it one layer earlier so clusterDNS
+// becomes a pure, hardware-free-testable function of the configured serviceSubnet
+// (pitfall C2) and so the provider explicitly OWNS the value rather than relying
+// on kubeadm's implicit derivation (which a future kubeadm change could shift).
+const dnsIPOffset = 10
+
+// DeriveDNSIP returns the cluster DNS service IP for the given serviceSubnet,
+// matching kubeadm's derivation: the dnsIPOffset-th (10th) host IP of the service
+// CIDR. For a dual-stack serviceSubnet ("v4cidr,v6cidr"), the DNS IP is derived
+// from the FIRST (primary) CIDR, as kubeadm does. It fails fast on an empty,
+// malformed, or too-small subnet rather than guessing or wrapping around.
+func DeriveDNSIP(serviceSubnet string) (string, error) {
+	s := strings.TrimSpace(serviceSubnet)
+	if s == "" {
+		return "", fmt.Errorf("serviceSubnet is empty")
+	}
+	cidrs := strings.Split(s, ",")
+	if len(cidrs) > 2 {
+		return "", fmt.Errorf("serviceSubnet %q has more than two CIDRs", serviceSubnet)
+	}
+	primary := strings.TrimSpace(cidrs[0])
+	if primary == "" {
+		return "", fmt.Errorf("serviceSubnet %q has an empty primary CIDR", serviceSubnet)
+	}
+	// Reject a dual-stack value with an empty secondary element (e.g. "cidr," or
+	// "cidr, "): kubeadm validation rejects it, so fail fast here rather than
+	// emit a config that kubeadm later refuses with a less obvious error.
+	if len(cidrs) == 2 && strings.TrimSpace(cidrs[1]) == "" {
+		return "", fmt.Errorf("serviceSubnet %q has an empty secondary CIDR", serviceSubnet)
+	}
+	_, ipnet, err := net.ParseCIDR(primary)
+	if err != nil {
+		return "", fmt.Errorf("parse serviceSubnet %q: %w", primary, err)
+	}
+	ip, err := indexedIP(ipnet, dnsIPOffset)
+	if err != nil {
+		return "", fmt.Errorf("serviceSubnet %q: %w", primary, err)
+	}
+	return ip.String(), nil
+}
+
+// indexedIP returns the IP at the given offset within ipnet, computed by
+// big-endian addition on the network base address so it is correct for both IPv4
+// (4-byte) and IPv6 (16-byte) and for non-zero-aligned bases (e.g. 10.96.0.64/26
+// -> +10 -> 10.96.0.74). It errors when the offset falls outside the network,
+// i.e. the CIDR is too small to contain that host.
+func indexedIP(ipnet *net.IPNet, offset int) (net.IP, error) {
+	base := ipnet.IP
+	if v4 := base.To4(); v4 != nil {
+		base = v4
+	} else {
+		base = base.To16()
+	}
+	if base == nil {
+		return nil, fmt.Errorf("unsupported IP form")
+	}
+	out := make(net.IP, len(base))
+	copy(out, base)
+	for i, carry := len(out)-1, offset; i >= 0 && carry > 0; i-- {
+		sum := int(out[i]) + carry
+		out[i] = byte(sum & 0xff)
+		carry = sum >> 8
+		if i == 0 && carry > 0 {
+			return nil, fmt.Errorf("offset %d overflows the address space", offset)
+		}
+	}
+	if !ipnet.Contains(out) {
+		return nil, fmt.Errorf("offset %d is outside the subnet (CIDR too small for the DNS IP)", offset)
+	}
+	return out, nil
+}

internal/kubeadmconfig/dns_test.go

@@ -0,0 +1,59 @@
+package kubeadmconfig
+
+import "testing"
+
+func TestDeriveDNSIP(t *testing.T) {
+	tests := []struct {
+		name    string
+		subnet  string
+		want    string
+		wantErr bool
+	}{
+		// kubeadm default-equivalent and custom IPv4 subnets -> 10th host IP.
+		{"default /12", "10.96.0.0/12", "10.96.0.10", false},
+		{"custom /16", "10.0.0.0/16", "10.0.0.10", false},
+		{"custom non-default base /16", "172.20.0.0/16", "172.20.0.10", false},
+		{"custom /24", "10.96.0.0/24", "10.96.0.10", false},
+		// Boundary: /28 (16 hosts) -- index 10 is the last comfortably-usable host.
+		{"boundary /28", "10.96.0.0/28", "10.96.0.10", false},
+		// Non-zero-aligned base: offset adds to the network address (regression
+		// guard for byte-wise addition vs. naive last-octet set).
+		{"non-zero-aligned /20", "192.168.128.0/20", "192.168.128.10", false},
+		{"non-zero-aligned /26", "10.96.0.64/26", "10.96.0.74", false},
+		// IPv6.
+		{"ipv6 /108", "fd00::/108", "fd00::a", false},
+		// Dual-stack: derive from the primary (first) CIDR.
+		{"dual-stack v4 primary", "10.96.0.0/12,fd00::/108", "10.96.0.10", false},
+		{"dual-stack v6 primary", "fd00::/108,10.96.0.0/12", "fd00::a", false},
+		// Tolerate whitespace around the dual-stack split.
+		{"dual-stack with spaces", "10.96.0.0/12 , fd00::/108", "10.96.0.10", false},
+		// Errors: fail fast, never guess.
+		{"empty", "", "", true},
+		{"whitespace only", "   ", "", true},
+		{"garbage", "not-a-cidr", "", true},
+		{"bare ip no mask", "10.96.0.0", "", true},
+		{"too small ipv4 /29", "10.96.0.0/29", "", true},
+		{"too small ipv6 /125", "fd00::/125", "", true},
+		{"three cidrs", "10.96.0.0/12,fd00::/108,10.1.0.0/16", "", true},
+		{"empty primary", ",fd00::/108", "", true},
+		{"trailing comma empty secondary", "10.96.0.0/12,", "", true},
+		{"empty secondary with space", "10.96.0.0/12, ", "", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DeriveDNSIP(tt.subnet)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("DeriveDNSIP(%q) = %q, want error", tt.subnet, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("DeriveDNSIP(%q) unexpected error: %v", tt.subnet, err)
+			}
+			if got != tt.want {
+				t.Fatalf("DeriveDNSIP(%q) = %q, want %q", tt.subnet, got, tt.want)
+			}
+		})
+	}
+}

internal/kubeadmconfig/types.go

@@ -133,6 +133,21 @@ type JoinControlPlane struct {
 	CertificateKey   string      `json:"certificateKey,omitempty"`
 }
 
+// KubeletConfiguration is the kubelet component config (kubelet.config.k8s.io/
+// v1beta1). We emit it only to pin clusterDNS to the value derived from a CUSTOM
+// serviceSubnet (pitfall C2); kubeadm otherwise derives clusterDNS implicitly from
+// its own service CIDR. clusterDNS is a LIST in the kubelet API, not a scalar.
+// This document carries no credentials.
+type KubeletConfiguration struct {
+	TypeMeta   `json:",inline"`
+	ClusterDNS []string `json:"clusterDNS,omitempty"`
+}
+
+// NewKubeletConfiguration returns a KubeletConfiguration with TypeMeta set.
+func NewKubeletConfiguration() KubeletConfiguration {
+	return KubeletConfiguration{TypeMeta: TypeMeta{APIVersion: KubeletAPIVersion, Kind: KindKubeletConfiguration}}
+}
+
 // NewClusterConfiguration returns a ClusterConfiguration with TypeMeta set.
 func NewClusterConfiguration() ClusterConfiguration {
 	return ClusterConfiguration{TypeMeta: TypeMeta{APIVersion: KubeadmAPIVersion, Kind: KindClusterConfiguration}}