azure: download assets from the OCI registry on nodes

The bootstrap script and nodeup's asset store fetch OCI blobs by digest,
authenticating with the instance's managed identity. Pod and sandbox
images are pulled through the acr-credential-provider kubelet plugin.

Author: hakman

nodeup/pkg/model/context.go

@@ -343,6 +343,8 @@ func (c *NodeupModelContext) UseExternalKubeletCredentialProvider() bool {
 		return true
 	case kops.CloudProviderAWS:
 		return true
+	case kops.CloudProviderAzure:
+		return c.NodeupConfig.UseACRCredentialProvider
 	default:
 		return false
 	}

nodeup/pkg/model/kubelet.go

@@ -224,6 +224,10 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 			if err := b.addECRCredentialProvider(c); err != nil {
 				return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
 			}
+		case kops.CloudProviderAzure:
+			if err := b.addACRCredentialProvider(c); err != nil {
+				return fmt.Errorf("failed to add the %s kubelet credential provider: %w", b.CloudProvider(), err)
+			}
 		}
 	}
 
@@ -443,6 +447,12 @@ func (b *KubeletBuilder) getGCPCredentialProviderPath() string {
 	return b.binaryPath() + "/gcp-credential-provider"
 }
 
+// getACRCredentialProviderPath returns the path of the ACR Credentials Provider based on distro and archiecture
+func (b *KubeletBuilder) getACRCredentialProviderPath() string {
+	// The binary name must match the provider name in the CredentialProviderConfig.
+	return b.binaryPath() + "/acr-credential-provider"
+}
+
 // buildManifestDirectory creates the directory where kubelet expects static manifests to reside
 func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
 	if kubeletConfig.PodManifestPath == "" {
@@ -731,6 +741,74 @@ providers:
 	return nil
 }
 
+// addACRCredentialProvider installs the Azure Container Registry Kubelet Credential Provider
+func (b *KubeletBuilder) addACRCredentialProvider(c *fi.NodeupModelBuilderContext) error {
+	azureConfigFilePath := "/etc/kubernetes/azure.json"
+
+	{
+		assetName := "azure-acr-credential-provider-linux-" + string(b.Architecture)
+		assetPath := ""
+		asset, err := b.Assets.Find(assetName, assetPath)
+		if err != nil {
+			return fmt.Errorf("trying to locate asset %q: %v", assetName, err)
+		}
+		if asset == nil {
+			return fmt.Errorf("unable to locate asset %q", assetName)
+		}
+
+		t := &nodetasks.File{
+			Path:     b.getACRCredentialProviderPath(),
+			Contents: asset,
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0755"),
+		}
+		c.AddTask(t)
+	}
+
+	{
+		// The credential provider authenticates with the instance's managed identity;
+		// kOps Azure nodes have no other cloud config file, so write a minimal one.
+		azureConfig := `{
+  "cloud": "AzurePublicCloud",
+  "useManagedIdentityExtension": true
+}
+`
+		t := &nodetasks.File{
+			Path:     azureConfigFilePath,
+			Contents: fi.NewStringResource(azureConfig),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0644"),
+		}
+		c.AddTask(t)
+	}
+
+	{
+		configContent := `apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - apiVersion: credentialprovider.kubelet.k8s.io/v1
+    name: acr-credential-provider
+    matchImages:
+      - "*.azurecr.io"
+      - "*.azurecr.cn"
+      - "*.azurecr.de"
+      - "*.azurecr.us"
+    defaultCacheDuration: "10m"
+    args:
+      - ` + azureConfigFilePath + `
+`
+
+		t := &nodetasks.File{
+			Path:     credentialProviderConfigFilePath,
+			Contents: fi.NewStringResource(configContent),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0644"),
+		}
+		c.AddTask(t)
+	}
+	return nil
+}
+
 // NodeLabels are defined in the InstanceGroup, but set flags on the kubelet config.
 // We have a conflict here: on the one hand we want an easy to use abstract specification
 // for the cluster, on the other hand we don't want two fields that do the same thing.

pkg/model/resources/nodeup.go

@@ -107,6 +107,8 @@ validate-hash() {
   fi
 }
 
+{{- OCIDownloadFunctions }}
+
 function download-release() {
   case "$(uname -m)" in
   x86_64*|i?86_64*|amd64*)
@@ -169,6 +171,19 @@ type NodeUpScript struct {
 	EnvironmentVariables func() (string, error)
 }
 
+// usesOCIAssetRegistry is true when nodeup is downloaded from an OCI registry
+// (an oci:// assets.fileRepository), authenticating with the instance identity.
+func (b *NodeUpScript) usesOCIAssetRegistry() bool {
+	for _, asset := range b.NodeUpAssets {
+		for _, location := range asset.Locations {
+			if strings.HasPrefix(location, "oci://") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func funcEmptyString() (string, error) {
 	return "", nil
 }
@@ -233,11 +248,54 @@ func (b *NodeUpScript) Build() (fi.Resource, error) {
 		"ProxyEnv":             b.ProxyEnv,
 		"EnvironmentVariables": b.EnvironmentVariables,
 		"CopyCheckUrlBlock":    b.copyCheckUrlBlock,
+		"OCIDownloadFunctions": b.ociDownloadFunctions,
 	}
 
 	return newTemplateResource("nodeup", nodeUpTemplate, functions, nil)
 }
 
+// ociDownloadFunctions returns a shell function that downloads a blob from an OCI
+// registry, authenticating with the instance identity. Assets stored in OCI
+// registries are addressed by digest, which is the sha256 hash of their content.
+func (b *NodeUpScript) ociDownloadFunctions() (string, error) {
+	if !b.usesOCIAssetRegistry() {
+		return "", nil
+	}
+	if b.CloudProvider != string(kops.CloudProviderAzure) {
+		return "", fmt.Errorf("OCI asset registry is not supported on cloud provider %q", b.CloudProvider)
+	}
+
+	// Azure Container Registry: exchange a managed-identity token from the instance
+	// metadata service for a registry refresh token, then for a pull-scoped access token.
+	return `
+# Download an OCI blob by digest. args: file, hash, url (oci://<registry>/<repository>)
+download-oci() {
+  local -r file="$1"
+  local -r hash="$2"
+  local -r url="$3"
+
+  local -r stripped="${url#oci://}"
+  local -r registry="${stripped%%/*}"
+  local -r repository="${stripped#*/}"
+  local aad_token refresh_token registry_token
+
+  if ! aad_token=$(curl -fsS -H "Metadata: true" "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fmanagement.azure.com%2F" | sed -e 's/.*"access_token":"//' -e 's/".*//'); then
+    echo "== Failed to get an identity token from the instance metadata service =="
+    return 1
+  fi
+  if ! refresh_token=$(curl -fsS "https://${registry}/oauth2/exchange" --data-urlencode "grant_type=access_token" --data-urlencode "service=${registry}" --data-urlencode "access_token=${aad_token}" | sed -e 's/.*"refresh_token":"//' -e 's/".*//'); then
+    echo "== Failed to exchange the identity token for a registry refresh token =="
+    return 1
+  fi
+  if ! registry_token=$(curl -fsS "https://${registry}/oauth2/token" --data-urlencode "grant_type=refresh_token" --data-urlencode "service=${registry}" --data-urlencode "scope=repository:${repository}:pull" --data-urlencode "refresh_token=${refresh_token}" | sed -e 's/.*"access_token":"//' -e 's/".*//'); then
+    echo "== Failed to get a registry access token =="
+    return 1
+  fi
+
+  curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 -H "Authorization: Bearer ${registry_token}" "https://${registry}/v2/${repository}/blobs/sha256:${hash}"
+}`, nil
+}
+
 func (b *NodeUpScript) copyCheckUrlBlock() (string, error) {
 	if b.CloudProvider == string(kops.CloudProviderGCE) {
 		return `commands=(
@@ -261,6 +319,19 @@ func (b *NodeUpScript) copyCheckUrlBlock() (string, error) {
           return 0
         fi
       done`, nil
+	} else if b.usesOCIAssetRegistry() {
+		// All file assets are remapped to the OCI registry, so all urls are oci:// URLs.
+		return `if ! download-oci "${file}" "${hash}" "${url}"; then
+        echo "== Failed to download ${url} =="
+        continue
+      fi
+      if ! validate-hash "${file}" "${hash}"; then
+        echo "== Failed to validate hash for ${url} =="
+        rm -f "${file}"
+        continue
+      fi
+      echo "== Downloaded ${url} with hash ${hash} =="
+      return 0`, nil
 	} else {
 		return `commands=(
         "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"

pkg/model/resources/nodeup_test.go

@@ -19,6 +19,12 @@ package resources
 import (
 	"strings"
 	"testing"
+
+	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/architectures"
+	"k8s.io/kops/util/pkg/hashing"
 )
 
 func Test_NodeUpTabs(t *testing.T) {
@@ -28,3 +34,54 @@ func Test_NodeUpTabs(t *testing.T) {
 		}
 	}
 }
+
+func Test_NodeUpScriptOCIAssetRegistry(t *testing.T) {
+	renderScript := func(baseURL string) string {
+		script := &NodeUpScript{
+			NodeUpAssets: map[architectures.Architecture]*assets.MirroredAsset{
+				architectures.ArchitectureAmd64: {
+					Locations: []string{baseURL + "/binaries/kops/1.35.0/linux/amd64/nodeup"},
+					Hash:      hashing.MustFromString("833723369ad345a88dd85d61b1e77336d56e61b864557ded71b92b6e34158e6a"),
+				},
+				architectures.ArchitectureArm64: {
+					Locations: []string{baseURL + "/binaries/kops/1.35.0/linux/arm64/nodeup"},
+					Hash:      hashing.MustFromString("e525c28a65ff0ce4f95f9e730195b4e67fdcb15ceb1f36b5ad6921a8a4490c71"),
+				},
+			},
+			BootConfig:    &nodeup.BootConfig{},
+			CloudProvider: "azure",
+		}
+		resource, err := script.Build()
+		if err != nil {
+			t.Fatalf("building nodeup script: %v", err)
+		}
+		rendered, err := fi.ResourceAsString(resource)
+		if err != nil {
+			t.Fatalf("rendering nodeup script: %v", err)
+		}
+		return rendered
+	}
+
+	rendered := renderScript("oci://myregistry.azurecr.io/assets")
+	for _, expected := range []string{
+		"download-oci()",
+		`if ! download-oci "${file}" "${hash}" "${url}"; then`,
+		"/oauth2/exchange",
+		`"https://${registry}/v2/${repository}/blobs/sha256:${hash}"`,
+	} {
+		if !strings.Contains(rendered, expected) {
+			t.Errorf("expected the nodeup script to contain %q", expected)
+		}
+	}
+	// All file assets are remapped to the OCI registry; the generic download commands are not needed.
+	if strings.Contains(rendered, "commands=(") {
+		t.Errorf("expected the nodeup script to not contain the generic download commands")
+	}
+
+	rendered = renderScript("https://artifacts.k8s.io")
+	for _, unexpected := range []string{"download-oci"} {
+		if strings.Contains(rendered, unexpected) {
+			t.Errorf("expected the nodeup script to not contain %q", unexpected)
+		}
+	}
+}

pkg/nodemodel/fileassets.go

@@ -29,6 +29,10 @@ import (
 	"k8s.io/kops/util/pkg/architectures"
 )
 
+// acrCredentialProviderVersion is the cloud-provider-azure release providing the
+// azure-acr-credential-provider binary; hashes are pinned in pkg/assets/assetdata/acr.yaml.
+const acrCredentialProviderVersion = "v1.34.12"
+
 // KubernetesFileAssets are the assets for downloading Kubernetes binaries
 type KubernetesFileAssets struct {
 	// KubernetesFileAssets are the assets for downloading Kubernetes binaries
@@ -102,6 +106,20 @@ func BuildKubernetesFileAssets(ig model.InstanceGroup, assetBuilder *assets.Asse
 				return nil, err
 			}
 			kubernetesAssets[arch] = append(kubernetesAssets[arch], assets.BuildMirroredAsset(asset))
+		case kops.CloudProviderAzure:
+			// The ACR credential provider is only used when pulling from a private
+			// registry holding the cluster's assets.
+			if ig.RawClusterSpec().OCIAssetRegistryHost() != "" {
+				u, err := url.Parse(fmt.Sprintf("https://github.com/kubernetes-sigs/cloud-provider-azure/releases/download/%s/azure-acr-credential-provider-linux-%s", acrCredentialProviderVersion, arch))
+				if err != nil {
+					return nil, err
+				}
+				asset, err := assetBuilder.RemapFile(u, nil)
+				if err != nil {
+					return nil, err
+				}
+				kubernetesAssets[arch] = append(kubernetesAssets[arch], assets.BuildMirroredAsset(asset))
+			}
 		}
 
 		if ig.InstallCNIAssets() {

upup/pkg/fi/assetstore.go

@@ -211,6 +211,9 @@ func (a *AssetStore) Add(ctx context.Context, id string) error {
 	if i == -1 {
 		i = strings.Index(id, "@gs://")
 	}
+	if i == -1 {
+		i = strings.Index(id, "@oci://")
+	}
 	if i != -1 {
 		urls := strings.Split(id[i+1:], ",")
 		hash, err := hashing.FromString(id[:i])

upup/pkg/fi/http.go

@@ -88,13 +88,8 @@ func downloadURLToWriter(ctx context.Context, desturl string, dest io.Writer, ha
 	if err != nil {
 		return nil, fmt.Errorf("Invalud URL for file %q: %v", desturl, err)
 	}
-	if u.Scheme != "gs" {
-		reader, err = OpenURL(desturl)
-		if err != nil {
-			return nil, err
-		}
-		defer reader.Close()
-	} else {
+	switch u.Scheme {
+	case "gs":
 		bucketName := u.Host
 		objectName := strings.TrimPrefix(u.Path, "/")
 
@@ -109,6 +104,18 @@ func downloadURLToWriter(ctx context.Context, desturl string, dest io.Writer, ha
 			return nil, fmt.Errorf("Failed to open reader on object %q: %v", desturl, err)
 		}
 		defer reader.Close()
+	case "oci":
+		reader, err = openOCIBlob(ctx, u, hash)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+	default:
+		reader, err = OpenURL(desturl)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
 	}
 	start := time.Now()
 	defer func() {