WIP azure: serve cluster assets from a kOps-managed container registry

cmd/kops/update_cluster.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 	apisutil "k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/assets/assetcopy"
 	"k8s.io/kops/pkg/commands/commandutils"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/pkg/predicates"
@@ -379,6 +380,7 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Up
 		GetAssets:                  c.GetAssets,
 		DeletionProcessing:         deletionProcessing,
 		ControlPlaneRunningVersion: minControlPlaneRunningVersion,
+		PushAssets:                 assetcopy.Copy,
 	}
 
 	applyResults, err := applyCmd.Run(ctx)

docs/cluster_spec.md

@@ -1500,6 +1500,41 @@ spec:
     containerProxy: proxy.example.com
 ```
 
+### fileRepository with an OCI registry (Azure only)
+
+On Azure, `fileRepository` may be an `oci://` URL pointing at an Azure Container Registry.
+File assets (including nodeup) are then stored in the registry as OCI artifacts and downloaded
+by nodes using their managed identity, so no public file server is needed. Combined with `containerRegistry` pointing at the same registry, all of
+the cluster's assets are served from one private registry.
+
+With `managed: true`, kOps creates and manages the registry (Basic SKU) in the cluster's
+resource group, grants the instances `AcrPull`, and pushes the file and image assets during
+`kops update cluster`. `kops delete cluster` deletes the registry with the rest of the cluster
+resources. Without `managed`, the registry must already exist and instances must be granted
+pull access.
+
+```yaml
+spec:
+  assets:
+    managed: true
+```
+
+When `fileRepository` and `containerRegistry` are not set, they default to a registry name
+derived from the subscription ID and the cluster name (registry names are global, so the
+cluster name alone would not be unique). The derived values are visible with
+`kops get cluster -o yaml --full`. Both locations can also be set explicitly:
+
+```yaml
+spec:
+  assets:
+    containerRegistry: myregistry.azurecr.io
+    fileRepository: oci://myregistry.azurecr.io/assets
+    managed: true
+```
+
+With the `terraform` target the assets are not pushed during `kops update cluster`; run
+`kops get assets --copy` after `terraform apply` instead.
+
 ## sysctlParameters
 {{ kops_feature_table(kops_added_default='1.17') }}
 

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1

go.sum

@@ -35,6 +35,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2/go.mod h1:jVRrRDLCOuif95HDYC23ADTMlvahB7tMdl519m9Iyjc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 h1:/Di3vB4sNeQ+7A8efjUVENvyB945Wruvstucqp7ZArg=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0/go.mod h1:gM3K25LQlsET3QR+4V74zxCsFAy0r6xMNN9n80SZn+4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0/go.mod h1:ceIuwmxDWptoW3eCqSXlnPsZFKh4X+R38dWPv7GS9Vs=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=

k8s/crds/kops.k8s.io_clusters.yaml

@@ -176,6 +176,11 @@ spec:
                     description: FileRepository is the url for a private file serving
                       repository
                     type: string
+                  managed:
+                    description: |-
+                      Managed requests that kOps create and manage the registry holding the
+                      cluster's file and image assets, as part of the cluster's cloud resources.
+                    type: boolean
                 type: object
               authentication:
                 description: Authentication field controls how the cluster is configured

nodeup/pkg/model/containerd.go

@@ -17,6 +17,7 @@ limitations under the License.
 package model
 
 import (
+	"context"
 	_ "embed"
 	"encoding/csv"
 	"encoding/json"
@@ -104,6 +105,20 @@ func (b *ContainerdBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		}
 	}
 
+	// containerd pulls the sandbox image itself, without kubelet's image credential
+	// providers; when the sandbox image is in the cluster's Azure Container Registry,
+	// pre-pull it with the instance's managed identity.
+	if b.NodeupConfig.UseACRCredentialProvider && b.NodeupConfig.ContainerdConfig != nil {
+		if image := fi.ValueOf(b.NodeupConfig.ContainerdConfig.SandboxImage); strings.Contains(image, ".azurecr.io/") {
+			registry, _, _ := strings.Cut(image, "/")
+			task := &nodetasks.PullImageTask{Name: image}
+			task.SetAuth(func(ctx context.Context) (string, string, error) {
+				return fi.ACRDockerCredentials(ctx, registry)
+			})
+			c.AddTask(task)
+		}
+	}
+
 	return nil
 }
 

nodeup/pkg/model/context.go

@@ -343,6 +343,8 @@ func (c *NodeupModelContext) UseExternalKubeletCredentialProvider() bool {
 		return true
 	case kops.CloudProviderAWS:
 		return true
+	case kops.CloudProviderAzure:
+		return c.NodeupConfig.UseACRCredentialProvider
 	default:
 		return false
 	}

nodeup/pkg/model/kubelet.go

@@ -224,6 +224,10 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 			if err := b.addECRCredentialProvider(c); err != nil {
 				return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
 			}
+		case kops.CloudProviderAzure:
+			if err := b.addACRCredentialProvider(c); err != nil {
+				return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
+			}
 		}
 	}
 
@@ -443,6 +447,12 @@ func (b *KubeletBuilder) getGCPCredentialProviderPath() string {
 	return b.binaryPath() + "/gcp-credential-provider"
 }
 
+// getACRCredentialProviderPath returns the path of the ACR Credentials Provider based on distro and archiecture
+func (b *KubeletBuilder) getACRCredentialProviderPath() string {
+	// The binary name must match the provider name in the CredentialProviderConfig.
+	return b.binaryPath() + "/acr-credential-provider"
+}
+
 // buildManifestDirectory creates the directory where kubelet expects static manifests to reside
 func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
 	if kubeletConfig.PodManifestPath == "" {
@@ -731,6 +741,74 @@ providers:
 	return nil
 }
 
+// addACRCredentialProvider installs the Azure Container Registry Kubelet Credential Provider
+func (b *KubeletBuilder) addACRCredentialProvider(c *fi.NodeupModelBuilderContext) error {
+	azureConfigFilePath := "/etc/kubernetes/azure.json"
+
+	{
+		assetName := "azure-acr-credential-provider-linux-" + string(b.Architecture)
+		assetPath := ""
+		asset, err := b.Assets.Find(assetName, assetPath)
+		if err != nil {
+			return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
+		}
+		if asset == nil {
+			return fmt.Errorf("unable to locate asset %q", assetName)
+		}
+
+		t := &nodetasks.File{
+			Path:     b.getACRCredentialProviderPath(),
+			Contents: asset,
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0755"),
+		}
+		c.AddTask(t)
+	}
+
+	{
+		// The credential provider authenticates with the instance's managed identity;
+		// kOps Azure nodes have no other cloud config file, so write a minimal one.
+		azureConfig := `{
+  "cloud": "AzurePublicCloud",
+  "useManagedIdentityExtension": true
+}
+`
+		t := &nodetasks.File{
+			Path:     azureConfigFilePath,
+			Contents: fi.NewStringResource(azureConfig),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0644"),
+		}
+		c.AddTask(t)
+	}
+
+	{
+		configContent := `apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - apiVersion: credentialprovider.kubelet.k8s.io/v1
+    name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    args:
+      - ` + azureConfigFilePath + `
+`
+
+		t := &nodetasks.File{
+			Path:     credentialProviderConfigFilePath,
+			Contents: fi.NewStringResource(configContent),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0644"),
+		}
+		c.AddTask(t)
+	}
+	return nil
+}
+
 // NodeLabels are defined in the InstanceGroup, but set flags on the kubelet config.
 // We have a conflict here: on the one hand we want an easy to use abstract specification
 // for the cluster, on the other hand we don't want two fields that do the same thing.

pkg/apis/kops/cluster.go

@@ -18,6 +18,8 @@ package kops
 
 import (
 	"fmt"
+	"net/url"
+	"strings"
 
 	"github.com/blang/semver/v4"
 	corev1 "k8s.io/api/core/v1"
@@ -377,6 +379,9 @@ type AssetsSpec struct {
 	FileRepository *string `json:"fileRepository,omitempty"`
 	// ContainerProxy is a url for a pull-through proxy of a container registry.
 	ContainerProxy *string `json:"containerProxy,omitempty"`
+	// Managed requests that kOps create and manage the registry holding the
+	// cluster's file and image assets, as part of the cluster's cloud resources.
+	Managed *bool `json:"managed,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources
@@ -917,6 +922,33 @@ func (c *Cluster) AzureResourceGroupName() string {
 	return c.Name
 }
 
+// OCIAssetRegistryHost returns the registry host of the assets fileRepository
+// when it is an oci:// URL, and "" otherwise.
+func (c *ClusterSpec) OCIAssetRegistryHost() string {
+	if c.Assets == nil || c.Assets.FileRepository == nil {
+		return ""
+	}
+	u, err := url.Parse(*c.Assets.FileRepository)
+	if err != nil || u.Scheme != "oci" {
+		return ""
+	}
+	return u.Host
+}
+
+// AzureManagedContainerRegistryName returns the name of the Azure Container Registry
+// that kOps creates and manages for the cluster's assets, or "" when kOps does not
+// manage one (assets.managed not set or fileRepository not an oci:// URL).
+func (c *Cluster) AzureManagedContainerRegistryName() string {
+	if c.Spec.Assets == nil || c.Spec.Assets.Managed == nil || !*c.Spec.Assets.Managed {
+		return ""
+	}
+	registryName, ok := strings.CutSuffix(c.Spec.OCIAssetRegistryHost(), ".azurecr.io")
+	if !ok {
+		return ""
+	}
+	return registryName
+}
+
 // IsSharedAzureRouteTable returns true if the route table is shared.
 func (c *Cluster) IsSharedAzureRouteTable() bool {
 	return c.Spec.CloudProvider.Azure.RouteTableName != ""

pkg/apis/kops/v1alpha2/cluster.go

@@ -365,6 +365,9 @@ type AssetsSpec struct {
 	FileRepository *string `json:"fileRepository,omitempty"`
 	// ContainerProxy is a url for a pull-through proxy of a docker registry
 	ContainerProxy *string `json:"containerProxy,omitempty"`
+	// Managed requests that kOps create and manage the registry holding the
+	// cluster's file and image assets, as part of the cluster's cloud resources.
+	Managed *bool `json:"managed,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources

pkg/apis/kops/v1alpha3/cluster.go

@@ -341,6 +341,9 @@ type AssetsSpec struct {
 	FileRepository *string `json:"fileRepository,omitempty"`
 	// ContainerProxy is a url for a pull-through proxy of a docker registry
 	ContainerProxy *string `json:"containerProxy,omitempty"`
+	// Managed requests that kOps create and manage the registry holding the
+	// cluster's file and image assets, as part of the cluster's cloud resources.
+	Managed *bool `json:"managed,omitempty"`
 }
 
 // IAMSpec adds control over the IAM security policies applied to resources