Got a working cluster with split control plane.

Author: cheftako

channels/pkg/cmd/apply_channel.go

@@ -43,9 +43,10 @@ import (
 )
 
 type ApplyChannelOptions struct {
-	Yes      bool
-	Interval time.Duration
-	NodeName string
+	Yes       bool
+	Interval  time.Duration
+	NodeName  string
+	NodeLabel string
 }
 
 func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
@@ -67,6 +68,7 @@ func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
 	cmd.Flags().BoolVar(&options.Yes, "yes", false, "Apply update")
 	cmd.Flags().DurationVar(&options.Interval, "interval", 0, "If non-zero, re-apply the channel on this interval until interrupted (e.g. 60s)")
 	cmd.Flags().StringVar(&options.NodeName, "node-name", "", "If set, patch the named node with the mandatory control-plane labels each iteration; typically supplied via the downward API.")
+	cmd.Flags().StringVar(&options.NodeLabel, "node-label", "", "If set, patch the named node with this label each iteration; typically supplied via the downward API.")
 
 	return cmd
 }
@@ -80,7 +82,7 @@ func runApplyChannelIteration(ctx context.Context, f *ChannelsFactory, out io.Wr
 		labelerClient, err := f.KubernetesClient()
 		if err != nil {
 			merr = multierr.Append(merr, fmt.Errorf("building kubernetes client for node labeler: %w", err))
-		} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName); err != nil {
+		} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName, options.NodeLabel); err != nil {
 			merr = multierr.Append(merr, fmt.Errorf("bootstrapping node labels: %w", err))
 		}
 	}

channels/pkg/nodelabeler/labeler.go

@@ -38,7 +38,7 @@ type nodePatchMetadata struct {
 
 // BootstrapControlPlaneNodeLabels applies labels to the current node so that it acts as a control-plane.
 // Safe to call repeatedly: the patch is skipped when the labels already match.
-func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string) error {
+func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string, nodeLabel string) error {
 	if nodeName == "" {
 		return fmt.Errorf("node name is required")
 	}
@@ -49,7 +49,7 @@ func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Inte
 		return fmt.Errorf("querying node %q: %w", nodeName, err)
 	}
 
-	labels := nodelabels.BuildMandatoryControlPlaneLabels()
+	labels := nodelabels.BuildMandatoryControlPlaneLabels(nodeLabel)
 
 	shouldPatch := false
 	for k, v := range labels {

nodeup/pkg/model/channels.go

@@ -46,7 +46,7 @@ type ChannelsBuilder struct {
 var _ fi.NodeupModelBuilder = &ChannelsBuilder{}
 
 func (b *ChannelsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster && !b.HasAPIServer {
+	if !b.IsMaster && !b.HasAPIServer { // TODO Fix me, only include API Server if not control plane.
 		return nil
 	}
 

pkg/model/components/channels/model.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/nodelabels"
 	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/pkg/wellknownusers"
 	"k8s.io/kops/upup/pkg/fi"
@@ -100,6 +101,7 @@ func (b *ChannelsBuilder) channelList() ([]string, error) {
 
 func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 	image := b.AssetBuilder.RemapImage("registry.k8s.io/kops/channels:" + kopsroot.KopsVersionImageTag())
+	// image = "gcr.io/wfender-dev-20240625/kops/channels:1.37.0-alpha.1"
 
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
@@ -121,11 +123,24 @@ func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 		},
 	}
 
+	// Kops Clusters with a ControlPlane nodes will run kops-channel there
+	// For those clusters we should label the cluster with the control plane label
+	// For Split Control Plane clusters we run kops-channel on the APIServer node.
+	// For those clusters we should label the cluster with the KOPS apiserver label
+	nodeLabel := nodelabels.RoleLabelKopsAPIServer
+	for _, ig := range b.AllInstanceGroups {
+		if ig.IsControlPlane() {
+			nodeLabel = nodelabels.RoleLabelControlPlane20
+			break
+		}
+	}
+
 	args := []string{
 		"apply", "channel",
 		"--v=4",
 		"--yes",
 		"--interval=" + channelsInterval.String(),
+		"--node-label=" + nodeLabel,
 		"--node-name=$(NODE_NAME)",
 	}
 	args = append(args, channels...)

pkg/nodelabels/builder.go

@@ -26,10 +26,15 @@ import (
 
 const (
 	RoleLabelAPIServer16           = "node-role.kubernetes.io/api-server"
+	RoleLabelKopsAPIServer         = "kops.k8s.io/node-api-server"
 	RoleLabelNode16                = "node-role.kubernetes.io/node"
+	RoleLabelKopsNode              = "kops.k8s.io/node-node"
 	RoleLabelEtcd                  = "node-role.kubernetes.io/etcd"
+	RoleLabelKopsEtcd              = "kops.k8s.io/node-etcd"
 	RoleLabelScheduler             = "node-role.kubernetes.io/scheduler"
+	RoleLabelKopsScheduler         = "kops.k8s.io/node-scheduler"
 	RoleLabelKubeControllerManager = "node-role.kubernetes.io/kube-controller-manager"
+	RoleLabelKopsKCM               = "kops.k8s.io/node-kube-controller-manager"
 
 	RoleLabelControlPlane20 = "node-role.kubernetes.io/control-plane"
 )
@@ -64,7 +69,7 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 
 	// Merge KubeletConfig for NodeLabels
 	c := &api.KubeletConfigSpec{}
-	if isControlPlane {
+	if instanceGroup.Spec.Role.IsControlPlaneType() {
 		reflectutils.JSONMergeStruct(c, cluster.Spec.ControlPlaneKubelet)
 	} else {
 		reflectutils.JSONMergeStruct(c, cluster.Spec.Kubelet)
@@ -84,8 +89,9 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 		// We keep the featureflag as a placeholder to change the logic;
 		// when we drop the featureflag we should just always include the label, even for
 		// full control-plane nodes.
-		if isAPIServer || featureflag.APIServerNodes.Enabled() {
+		if isAPIServer && featureflag.APIServerNodes.Enabled() {
 			nodeLabels[RoleLabelAPIServer16] = ""
+			nodeLabels[RoleLabelKopsAPIServer] = ""
 			nodeLabels["kops.k8s.io/kops-controller-pki"] = ""
 		}
 	}
@@ -95,37 +101,38 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelNode16] = ""
+		nodeLabels[RoleLabelKopsNode] = ""
 	}
 
 	if isEtcd {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelEtcd] = ""
-		nodeLabels[RoleLabelControlPlane20] = ""
+		nodeLabels[RoleLabelKopsEtcd] = ""
 	}
 
 	if isScheduler {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelScheduler] = ""
-		nodeLabels[RoleLabelControlPlane20] = ""
+		nodeLabels[RoleLabelKopsScheduler] = ""
 	}
 
 	if isKubeControllerManager {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelKubeControllerManager] = ""
-		nodeLabels[RoleLabelControlPlane20] = ""
+		nodeLabels[RoleLabelKopsKCM] = ""
 	}
 
 	if isControlPlane {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
-		for label, value := range BuildMandatoryControlPlaneLabels() {
+		for label, value := range BuildMandatoryControlPlaneLabels(RoleLabelControlPlane20) {
 			nodeLabels[label] = value
 		}
 	}
@@ -141,9 +148,13 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 }
 
 // BuildMandatoryControlPlaneLabels returns the list of labels all CP nodes must have
-func BuildMandatoryControlPlaneLabels() map[string]string {
+func BuildMandatoryControlPlaneLabels(nodeLabel string) map[string]string {
 	nodeLabels := make(map[string]string)
-	nodeLabels[RoleLabelControlPlane20] = ""
+	if nodeLabel == "" {
+		nodeLabels[RoleLabelControlPlane20] = ""
+	} else {
+		nodeLabels[nodeLabel] = ""
+	}
 	nodeLabels["kops.k8s.io/kops-controller-pki"] = ""
 	nodeLabels["node.kubernetes.io/exclude-from-external-load-balancers"] = ""
 	return nodeLabels

upup/models/cloudup/resources/addons/gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml.template

@@ -23,6 +23,9 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
+            - matchExpressions:
+              - key: kops.k8s.io/node-api-server
+                operator: Exists
             - matchExpressions:
               - key: node-role.kubernetes.io/control-plane
                 operator: Exists

upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template

@@ -39,6 +39,9 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
+            - matchExpressions:
+              - key: kops.k8s.io/node-api-server
+                operator: Exists
             - matchExpressions:
               - key: node-role.kubernetes.io/control-plane
                 operator: Exists

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -42,6 +42,9 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
+            - matchExpressions:
+              - key: kops.k8s.io/node-api-server
+                operator: Exists
             - matchExpressions:
               - key: node-role.kubernetes.io/control-plane
                 operator: Exists