Switched new kops role to be about workload not destination.

Fixed image rewrite to work for all of control plane.
Fixed docs to reflect changes.
Fixed role direct equality check.

Author: cheftako

channels/pkg/cmd/apply_channel.go

@@ -43,10 +43,10 @@ import (
 )
 
 type ApplyChannelOptions struct {
-	Yes       bool
-	Interval  time.Duration
-	NodeName  string
-	NodeLabel string
+	Yes        bool
+	Interval   time.Duration
+	NodeName   string
+	NodeLabels string
 }
 
 func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
@@ -68,7 +68,7 @@ func NewCmdApplyChannel(f *ChannelsFactory, out io.Writer) *cobra.Command {
 	cmd.Flags().BoolVar(&options.Yes, "yes", false, "Apply update")
 	cmd.Flags().DurationVar(&options.Interval, "interval", 0, "If non-zero, re-apply the channel on this interval until interrupted (e.g. 60s)")
 	cmd.Flags().StringVar(&options.NodeName, "node-name", "", "If set, patch the named node with the mandatory control-plane labels each iteration; typically supplied via the downward API.")
-	cmd.Flags().StringVar(&options.NodeLabel, "node-label", "", "If set, patch the named node with this label each iteration; typically supplied via the downward API.")
+	cmd.Flags().StringVar(&options.NodeLabels, "node-labels", "", "If set, patch the named node with each of the labels each iteration; typically supplied via the downward API.")
 
 	return cmd
 }
@@ -82,7 +82,7 @@ func runApplyChannelIteration(ctx context.Context, f *ChannelsFactory, out io.Wr
 		labelerClient, err := f.KubernetesClient()
 		if err != nil {
 			merr = multierr.Append(merr, fmt.Errorf("building kubernetes client for node labeler: %w", err))
-		} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName, options.NodeLabel); err != nil {
+		} else if err := nodelabeler.BootstrapControlPlaneNodeLabels(ctx, labelerClient, options.NodeName, options.NodeLabels); err != nil {
 			merr = multierr.Append(merr, fmt.Errorf("bootstrapping node labels: %w", err))
 		}
 	}

channels/pkg/nodelabeler/labeler.go

@@ -38,7 +38,7 @@ type nodePatchMetadata struct {
 
 // BootstrapControlPlaneNodeLabels applies labels to the current node so that it acts as a control-plane.
 // Safe to call repeatedly: the patch is skipped when the labels already match.
-func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string, nodeLabel string) error {
+func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Interface, nodeName string, nodeLabels string) error {
 	if nodeName == "" {
 		return fmt.Errorf("node name is required")
 	}
@@ -49,7 +49,7 @@ func BootstrapControlPlaneNodeLabels(ctx context.Context, client kubernetes.Inte
 		return fmt.Errorf("querying node %q: %w", nodeName, err)
 	}
 
-	labels := nodelabels.BuildMandatoryControlPlaneLabels(nodeLabel)
+	labels := nodelabels.BuildMandatoryControlPlaneLabels(nodeLabels)
 
 	shouldPatch := false
 	for k, v := range labels {

docs/cli/kops_rolling-update_cluster.md

@@ -333,7 +333,7 @@ func validateKarpenterInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster
 	if cluster.GetCloudProvider() != kops.CloudProviderAWS {
 		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "manager"), "Karpenter InstanceGroups are only supported on AWS"))
 	}
-	if g.Spec.Role != kops.InstanceGroupRoleNode {
+	if g.Spec.Role.HasNode() {
 		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Karpenter InstanceGroups must have role Node"))
 	}
 	if g.Spec.MaxSize != nil && *g.Spec.MaxSize <= 0 {

docs/cli/kops_update_cluster.md

@@ -127,7 +127,7 @@ func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 	// For those clusters we should label the cluster with the control plane label
 	// For Split Control Plane clusters we run kops-channel on the APIServer node.
 	// For those clusters we should label the cluster with the KOPS apiserver label
-	nodeLabel := nodelabels.RoleLabelKopsAPIServer
+	nodeLabel := nodelabels.RoleLabelKopsCCM + "," + nodelabels.RoleLabelKopsChannel
 	for _, ig := range b.AllInstanceGroups {
 		if ig.IsControlPlane() {
 			nodeLabel = nodelabels.RoleLabelControlPlane20
@@ -140,7 +140,7 @@ func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 		"--v=4",
 		"--yes",
 		"--interval=" + channelsInterval.String(),
-		"--node-label=" + nodeLabel,
+		"--node-labels=" + nodeLabel,
 		"--node-name=$(NODE_NAME)",
 	}
 	args = append(args, channels...)

pkg/apis/kops/validation/instancegroup.go

@@ -18,6 +18,7 @@ package nodelabels
 
 import (
 	"fmt"
+	"strings"
 
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/featureflag"
@@ -26,15 +27,13 @@ import (
 
 const (
 	RoleLabelAPIServer16           = "node-role.kubernetes.io/api-server"
-	RoleLabelKopsAPIServer         = "kops.k8s.io/node-api-server"
 	RoleLabelNode16                = "node-role.kubernetes.io/node"
-	RoleLabelKopsNode              = "kops.k8s.io/node-node"
 	RoleLabelEtcd                  = "node-role.kubernetes.io/etcd"
-	RoleLabelKopsEtcd              = "kops.k8s.io/node-etcd"
 	RoleLabelScheduler             = "node-role.kubernetes.io/scheduler"
-	RoleLabelKopsScheduler         = "kops.k8s.io/node-scheduler"
 	RoleLabelKubeControllerManager = "node-role.kubernetes.io/kube-controller-manager"
-	RoleLabelKopsKCM               = "kops.k8s.io/node-kube-controller-manager"
+
+	RoleLabelKopsCCM     = "kops.k8s.io/cloud-controller-manager"
+	RoleLabelKopsChannel = "kops.k8s.io/kops-channel"
 
 	RoleLabelControlPlane20 = "node-role.kubernetes.io/control-plane"
 )
@@ -91,7 +90,6 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 		// full control-plane nodes.
 		if isAPIServer && featureflag.APIServerNodes.Enabled() {
 			nodeLabels[RoleLabelAPIServer16] = ""
-			nodeLabels[RoleLabelKopsAPIServer] = ""
 			nodeLabels["kops.k8s.io/kops-controller-pki"] = ""
 		}
 	}
@@ -101,31 +99,27 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelNode16] = ""
-		nodeLabels[RoleLabelKopsNode] = ""
 	}
 
 	if isEtcd {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelEtcd] = ""
-		nodeLabels[RoleLabelKopsEtcd] = ""
 	}
 
 	if isScheduler {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelScheduler] = ""
-		nodeLabels[RoleLabelKopsScheduler] = ""
 	}
 
 	if isKubeControllerManager {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelKubeControllerManager] = ""
-		nodeLabels[RoleLabelKopsKCM] = ""
 	}
 
 	if isControlPlane {
@@ -148,12 +142,15 @@ func BuildNodeLabels(cluster *api.Cluster, instanceGroup *api.InstanceGroup) (ma
 }
 
 // BuildMandatoryControlPlaneLabels returns the list of labels all CP nodes must have
-func BuildMandatoryControlPlaneLabels(nodeLabel string) map[string]string {
+func BuildMandatoryControlPlaneLabels(baseLabels string) map[string]string {
 	nodeLabels := make(map[string]string)
-	if nodeLabel == "" {
+	if baseLabels == "" {
 		nodeLabels[RoleLabelControlPlane20] = ""
 	} else {
-		nodeLabels[nodeLabel] = ""
+		labels := strings.Split(baseLabels, ",")
+		for _, label := range labels {
+			nodeLabels[label] = ""
+		}
 	}
 	nodeLabels["kops.k8s.io/kops-controller-pki"] = ""
 	nodeLabels["node.kubernetes.io/exclude-from-external-load-balancers"] = ""

pkg/model/components/channels/model.go

@@ -74,6 +74,7 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 	}
 
 	for _, role := range kops.AllInstanceGroupRoles {
+		isControlPlaneType := role.IsControlPlaneType()
 		isMaster := role.HasControlPlane()
 		isAPIServer := role.HasAPIServer()
 		isEtcd := role.HasEtcd()
@@ -119,7 +120,7 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 
 		// `docker load` our images when using a KOPS_BASE_URL, so we
 		// don't need to push/pull from a registry
-		if os.Getenv("KOPS_BASE_URL") != "" && isMaster {
+		if os.Getenv("KOPS_BASE_URL") != "" && isControlPlaneType {
 			for _, arch := range architectures.GetSupported() {
 				for _, name := range []string{"kops-utils-cp", "kops-controller", "kops-channels", "dns-controller", "kube-apiserver-healthcheck"} {
 					baseURL, err := url.Parse(os.Getenv("KOPS_BASE_URL"))

pkg/nodelabels/builder.go

@@ -195,7 +195,7 @@ func (v *clusterValidatorImpl) Validate(ctx context.Context) (*ValidationCluster
 	if v.maxUnreadyNodes > 0 {
 		var notReadyWorkerNodes []string
 		for _, cloudGroup := range cloudGroups {
-			if cloudGroup.InstanceGroup != nil && cloudGroup.InstanceGroup.Spec.Role == kops.InstanceGroupRoleNode {
+			if cloudGroup.InstanceGroup != nil && cloudGroup.InstanceGroup.Spec.Role.HasNode() {
 				var allMembers []*cloudinstances.CloudInstance
 				allMembers = append(allMembers, cloudGroup.Ready...)
 				allMembers = append(allMembers, cloudGroup.NeedUpdate...)

pkg/nodemodel/nodeupconfigbuilder.go

@@ -24,7 +24,7 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
-              - key: kops.k8s.io/node-api-server
+              - key: kops.k8s.io/cloud-controller-manager
                 operator: Exists
             - matchExpressions:
               - key: node-role.kubernetes.io/control-plane