feat: pre-bundle control-plane images for air-gap first boot (ADR-16)

Dockerfile

@@ -216,11 +216,61 @@ RUN git clone --depth 1 --branch "v${CONTAINERD_VERSION}" \
     fi
 WORKDIR /src
 # Lay the static binaries out under /bin/containerd/bin so the final stage can
-# COPY the whole bin/ tree in one shot.
+# COPY the whole bin/ tree in one shot. ctr is built too: the boot-time image
+# import (ADR-16) uses `ctr -n k8s.io images import`, and it must be static (musl).
 RUN set -eux; \
-    make STATIC=1 bin/containerd bin/containerd-shim-runc-v2; \
+    make STATIC=1 bin/containerd bin/containerd-shim-runc-v2 bin/ctr; \
     mkdir -p /bin/containerd/bin; \
-    cp bin/containerd bin/containerd-shim-runc-v2 /bin/containerd/bin/
+    cp bin/containerd bin/containerd-shim-runc-v2 bin/ctr /bin/containerd/bin/
+
+# ----------------------------------------------------------------------------
+# Stage: pre-bundle the control-plane container images (ADR-16). Resolve the EXACT
+# set from the bundled kubeadm (same source as the pause pin, so refs never drift)
+# and fetch each as a ctr-importable tarball with crane (daemonless, no docker/
+# containerd needed at build). The tarballs are embedded read-only in the final
+# image and imported into containerd at boot, so a first boot converges with NO
+# registry access (air-gap). CNI is intentionally NOT bundled (operator's choice).
+# ----------------------------------------------------------------------------
+FROM alpine:3.21 AS image-bundler
+ARG KUBERNETES_VERSION
+ARG TARGETARCH
+# Pinned; crane is a static Go binary that runs on musl.
+ARG CRANE_VERSION=v0.20.3
+RUN apk add --no-cache curl ca-certificates
+COPY --from=k8s-binaries /bin/kubeadm /usr/bin/kubeadm
+# Install crane (checksum-verified), then resolve the control-plane image set from
+# the bundled kubeadm (single source of truth, shared with the pause pin) and fetch
+# each as a ctr-importable tarball. NOTE: no inline '#' comments inside this RUN --
+# the backslash continuations join it into one logical line, where a '#' would
+# comment out everything after it.
+RUN set -eux; \
+    case "${TARGETARCH}" in \
+      amd64) arch="x86_64" ;; \
+      arm64) arch="arm64" ;; \
+      *) echo "unsupported TARGETARCH ${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    base="https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}"; \
+    asset="go-containerregistry_Linux_${arch}.tar.gz"; \
+    curl -fsSL -o "${asset}" "${base}/${asset}"; \
+    curl -fsSL -o checksums.txt "${base}/checksums.txt"; \
+    grep " ${asset}\$" checksums.txt | sha256sum -c -; \
+    tar -xzf "${asset}" crane; install -m0755 crane /usr/bin/crane; \
+    rm -f "${asset}" checksums.txt crane; \
+    /usr/bin/kubeadm config images list \
+      --kubernetes-version "${KUBERNETES_VERSION}" \
+      --image-repository registry.k8s.io > /tmp/imglist; \
+    test -s /tmp/imglist; \
+    grep -q '/pause:' /tmp/imglist; \
+    mkdir -p /images; \
+    while read -r ref; do \
+      [ -n "${ref}" ] || continue; \
+      f="/images/$(echo "${ref}" | tr '/:' '__').tar"; \
+      crane pull "${ref}" "${f}"; \
+      echo "bundled ${ref} -> $(basename "${f}")"; \
+    done < /tmp/imglist; \
+    n="$(ls /images/*.tar | wc -l)"; \
+    echo "bundled ${n} control-plane image tarball(s)"; \
+    [ "${n}" -ge 5 ]
 
 # ----------------------------------------------------------------------------
 # Final stage: a Kairos image with everything wired up.
@@ -259,6 +309,13 @@ COPY systemd/kubelet.service                /etc/systemd/system/kubelet.service
 COPY systemd/kubelet.service.d/10-kubeadm.conf /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
 COPY sysctl/k8s.conf                        /etc/sysctl.d/k8s.conf
 COPY modules-load/k8s.conf                  /etc/modules-load.d/k8s.conf
+COPY systemd/provider-kubernetes-image-import.service /etc/systemd/system/provider-kubernetes-image-import.service
+
+# --- Pre-bundled control-plane images (ADR-16) ------------------------------
+# Embed the control-plane image tarballs (fetched by the image-bundler stage)
+# read-only in the OS image; the import oneshot loads them into containerd at boot
+# so a first boot converges with no registry access (air-gap).
+COPY --from=image-bundler /images /opt/provider-kubernetes/images
 
 # Pin containerd's pod-sandbox (pause) image to the EXACT version the bundled
 # kubeadm expects for this Kubernetes minor, instead of a hardcoded tag. kubeadm
@@ -276,7 +333,7 @@ RUN set -eux; \
     echo "pinned containerd sandbox_image to ${pause}"
 
 # --- Boot-time setup: enable services; modules and sysctls load via /etc -----
-RUN systemctl enable containerd.service kubelet.service
+RUN systemctl enable containerd.service kubelet.service provider-kubernetes-image-import.service
 
 # Record the bundled Kubernetes version on the image (OS_VERSION style banner
 # kept short; the provider also detects/enforces the version at runtime).

internal/imageimport/imageimport.go

@@ -0,0 +1,70 @@
+// Package imageimport imports the control-plane container-image tarballs that the
+// image build pre-bundles (ADR-16) into containerd's "k8s.io" namespace, so that
+// kubeadm init finds the control-plane images locally and never pulls from a
+// registry -- enabling a first boot to converge fully air-gapped.
+//
+// It is invoked at boot by a systemd oneshot (ordered After=containerd.service,
+// Before=kubelet.service) via the `agent-provider-kubernetes import-images`
+// subcommand. The operation is idempotent (re-importing already-present images is
+// harmless), bounded by the caller's context, and a no-op when no tarballs are
+// present.
+package imageimport
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sort"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/kairos-io/provider-kubernetes/internal/kubeadm"
+)
+
+// DefaultDir is the read-only (immutable) path the image build embeds the
+// control-plane image tarballs at. Being in the OS image layer, they survive
+// reboots and are available with no network (air-gap).
+const DefaultDir = "/opt/provider-kubernetes/images"
+
+// Runner executes `ctr` via argv (no shell). It is satisfied by
+// kubeadm.ExecRunner{Path: "ctr"}; the interface keeps Import unit-testable.
+type Runner interface {
+	Run(ctx context.Context, args ...string) (kubeadm.Result, error)
+}
+
+// Import loads every *.tar under dir into containerd's k8s.io namespace via
+// `ctr -n k8s.io images import <tar>`. It returns nil (a logged no-op) when dir
+// has no tarballs -- so an image with nothing bundled still boots. It attempts
+// every tarball and returns the first error encountered (fail loud without
+// silently skipping the rest); a partial failure leaves the missing image to be
+// pulled by kubeadm under its own bounded budget rather than blocking boot.
+func Import(ctx context.Context, dir string, r Runner) error {
+	tars, err := filepath.Glob(filepath.Join(dir, "*.tar"))
+	if err != nil {
+		return fmt.Errorf("glob %q: %w", dir, err)
+	}
+	if len(tars) == 0 {
+		logrus.Infof("image-import: no tarballs under %s; nothing to import", dir)
+		return nil
+	}
+	sort.Strings(tars)
+
+	var firstErr error
+	for _, tar := range tars {
+		out, runErr := r.Run(ctx, "-n", "k8s.io", "images", "import", tar)
+		if runErr != nil {
+			logrus.Errorf("image-import: failed to import %s: %v\n%s",
+				filepath.Base(tar), runErr, kubeadm.Sanitize(out.Stderr))
+			if firstErr == nil {
+				firstErr = fmt.Errorf("import %s: %w", filepath.Base(tar), runErr)
+			}
+			continue
+		}
+		logrus.Infof("image-import: imported %s", filepath.Base(tar))
+	}
+	if firstErr != nil {
+		return fmt.Errorf("one or more image imports failed: %w", firstErr)
+	}
+	logrus.Infof("image-import: imported %d tarball(s) from %s", len(tars), dir)
+	return nil
+}

internal/imageimport/imageimport_test.go

@@ -0,0 +1,91 @@
+package imageimport
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/kairos-io/provider-kubernetes/internal/kubeadm"
+)
+
+// fakeRunner records the argv of every Run call and can be made to fail.
+type fakeRunner struct {
+	calls   [][]string
+	failOn  string // basename of a tar to fail on ("" = none)
+	failErr error
+}
+
+func (f *fakeRunner) Run(_ context.Context, args ...string) (kubeadm.Result, error) {
+	f.calls = append(f.calls, args)
+	if f.failOn != "" && len(args) > 0 && filepath.Base(args[len(args)-1]) == f.failOn {
+		return kubeadm.Result{Stderr: "boom"}, f.failErr
+	}
+	return kubeadm.Result{}, nil
+}
+
+func writeTar(t *testing.T, dir, name string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestImportRunsCtrPerTarball(t *testing.T) {
+	dir := t.TempDir()
+	writeTar(t, dir, "etcd.tar")
+	writeTar(t, dir, "kube-apiserver.tar")
+	writeTar(t, dir, "ignored.txt") // non-tar must be skipped
+
+	fr := &fakeRunner{}
+	if err := Import(context.Background(), dir, fr); err != nil {
+		t.Fatalf("Import: %v", err)
+	}
+	if len(fr.calls) != 2 {
+		t.Fatalf("want 2 ctr calls (one per .tar), got %d: %v", len(fr.calls), fr.calls)
+	}
+	// Exact argv: ctr -n k8s.io images import <tar> (sorted, so etcd first).
+	want := []string{"-n", "k8s.io", "images", "import", filepath.Join(dir, "etcd.tar")}
+	got := fr.calls[0]
+	if len(got) != len(want) {
+		t.Fatalf("argv = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("argv[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)
+		}
+	}
+}
+
+func TestImportNoopOnEmptyOrMissingDir(t *testing.T) {
+	fr := &fakeRunner{}
+	// Empty dir.
+	if err := Import(context.Background(), t.TempDir(), fr); err != nil {
+		t.Fatalf("empty dir: want nil, got %v", err)
+	}
+	// Missing dir.
+	if err := Import(context.Background(), filepath.Join(t.TempDir(), "nope"), fr); err != nil {
+		t.Fatalf("missing dir: want nil, got %v", err)
+	}
+	if len(fr.calls) != 0 {
+		t.Fatalf("want 0 ctr calls on empty/missing dir, got %d", len(fr.calls))
+	}
+}
+
+func TestImportAttemptsAllAndReturnsError(t *testing.T) {
+	dir := t.TempDir()
+	writeTar(t, dir, "a.tar")
+	writeTar(t, dir, "b.tar") // fails
+	writeTar(t, dir, "c.tar")
+
+	fr := &fakeRunner{failOn: "b.tar", failErr: errors.New("import failed")}
+	err := Import(context.Background(), dir, fr)
+	if err == nil {
+		t.Fatal("want error when a tarball fails to import, got nil")
+	}
+	// All three are still attempted (a bad tarball does not skip the rest).
+	if len(fr.calls) != 3 {
+		t.Fatalf("want all 3 tarballs attempted, got %d calls", len(fr.calls))
+	}
+}

internal/provider/options.go

@@ -18,6 +18,12 @@ const (
 	// stdout/stderr; surfacing failures to operators (OQ-4 logs-only for v1).
 	ReconcileLogPath = "/var/log/provider-kubernetes-reconcile.log"
 
+	// ImportLogPath is where the boot-time import-images step (ADR-16) writes its
+	// stdout/stderr. The import runs as a yip step ordered BEFORE reconcile so the
+	// pre-bundled control-plane images are in containerd before kubeadm init/join
+	// runs (a systemd oneshot alone races the kairos-agent reconcile stage).
+	ImportLogPath = "/var/log/provider-kubernetes-image-import.log"
+
 	// The Layer-1 structured status paths are defined in internal/status:
 	//   status.StatusRunPath = "/run/provider-kubernetes/status.yaml"  (tmpfs, 0640)
 	//   status.StatusLogPath = "/var/log/provider-kubernetes/status.yaml"  (persistent, 0640)

internal/provider/provider.go

@@ -66,6 +66,19 @@ func Provider(cluster clusterplugin.Cluster) yip.YipConfig {
 					Group:       0,
 					Content:     string(clusterYAML),
 				}},
+			}, {
+				Name: "provider-kubernetes: import pre-bundled images",
+				Commands: []string{
+					// ADR-16: import the pre-bundled control-plane images into
+					// containerd BEFORE reconcile runs kubeadm init/join, so init
+					// finds them locally (air-gap) and never pulls. yip runs stage
+					// steps in array order, so this deterministically precedes the
+					// reconcile step below -- the systemd oneshot races this stage
+					// and cannot be relied on for ordering. No-op when nothing is
+					// bundled; failures are logged but do not abort the boot (yip
+					// continues to reconcile, which surfaces any missing image).
+					ProviderBinaryPath + " import-images >>" + ImportLogPath + " 2>&1 || true",
+				},
 			}, {
 				Name: "provider-kubernetes: reconcile",
 				Commands: []string{

internal/provider/provider_test.go

@@ -139,11 +139,20 @@ func TestProviderEmitsBootStage(t *testing.T) {
 	if got.Role != clusterplugin.RoleInit {
 		t.Fatalf("cluster role round-trip failed: %q", got.Role)
 	}
-	// Second stage invokes the reconcile subcommand.
-	if len(stages[1].Commands) != 1 {
-		t.Fatalf("expected reconcile stage to have exactly one Command, got %+v", stages[1].Commands)
+	// Second stage imports the pre-bundled control-plane images (ADR-16). It MUST
+	// come before the reconcile stage so kubeadm init/join finds images locally.
+	if len(stages) < 3 {
+		t.Fatalf("expected three stages under network.after (write, import, reconcile), got %d", len(stages))
 	}
-	cmd := stages[1].Commands[0]
+	importCmd := stages[1].Commands[0]
+	if !strings.Contains(importCmd, ProviderBinaryPath+" import-images") {
+		t.Fatalf("expected import-images step before reconcile, got %q", importCmd)
+	}
+	// Third stage invokes the reconcile subcommand.
+	if len(stages[2].Commands) != 1 {
+		t.Fatalf("expected reconcile stage to have exactly one Command, got %+v", stages[2].Commands)
+	}
+	cmd := stages[2].Commands[0]
 	if !strings.Contains(cmd, ProviderBinaryPath+" reconcile") {
 		t.Fatalf("reconcile command shape wrong: %q", cmd)
 	}

internal/reconcile/budget_test.go

@@ -0,0 +1,63 @@
+package reconcile
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// TestDefaultBudgetTotalIsTheBindingCeiling locks the #4099-1 never-hang
+// invariant: Total is the hard wall-clock ceiling and is deliberately LESS than
+// PerAttempt*MaxAttempts, so the worst-case failure-detection time is bounded by
+// Total (8m), not the naive product (with 6m/2 that product is 12m). If someone
+// bumps PerAttempt/MaxAttempts without keeping Total the binding cap, this fails.
+func TestDefaultBudgetTotalIsTheBindingCeiling(t *testing.T) {
+	b := DefaultBudget()
+	if b.PerAttempt <= 0 || b.MaxAttempts <= 0 || b.Total <= 0 {
+		t.Fatalf("DefaultBudget has non-positive fields: %+v", b)
+	}
+	product := b.PerAttempt * time.Duration(b.MaxAttempts)
+	if b.Total >= product {
+		t.Fatalf("Total (%s) must be < PerAttempt*MaxAttempts (%s) so Total is the binding never-hang ceiling", b.Total, product)
+	}
+	// Sanity upper bound: a failing reconcile must surface well within a boot's
+	// patience; 10m is a generous ceiling for the current 8m value.
+	if b.Total > 10*time.Minute {
+		t.Fatalf("Total (%s) exceeds the 10m never-hang sanity bound", b.Total)
+	}
+}
+
+// blockingExec simulates a perpetually-stuck action (e.g. wait-for-control-plane
+// against an unreachable endpoint): it blocks until its context deadline fires.
+type blockingExec struct{ calls int }
+
+func (b *blockingExec) Execute(ctx context.Context, _ Action) error {
+	b.calls++
+	<-ctx.Done()
+	return ctx.Err()
+}
+
+// TestReconcilerBoundedByTotalNotProduct proves the run is bounded by Budget.Total
+// even when every attempt hangs to its deadline -- the failure surfaces at ~Total,
+// NOT PerAttempt*MaxAttempts. This is the runtime guarantee behind #4099-1.
+func TestReconcilerBoundedByTotalNotProduct(t *testing.T) {
+	exec := &blockingExec{}
+	r := Reconciler{
+		// product = 100ms*5 = 500ms; Total caps at 250ms.
+		Budget: Budget{PerAttempt: 100 * time.Millisecond, MaxAttempts: 5, Total: 250 * time.Millisecond},
+		Exec:   exec,
+		Sleep:  func(time.Duration) {}, // skip backoff; the ctx deadlines do the bounding
+	}
+
+	start := time.Now()
+	err := r.Run(context.Background(), []Action{ActionWaitForControlPlane})
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("expected a loud failure when the action never succeeds")
+	}
+	// Bounded by Total (250ms) plus slack, and clearly under the 500ms product.
+	if elapsed >= 450*time.Millisecond {
+		t.Fatalf("run took %s; must be bounded by ~Total (250ms), not PerAttempt*MaxAttempts (500ms)", elapsed)
+	}
+}