test: tiered e2e suite (kubeadm-in-container) + per-PR CI job (ADR-13)

.github/workflows/ci.yml

@@ -173,3 +173,105 @@ jobs:
           need "$uc_help" --config "init phase upload-certs"
 
           echo "OK: kubeadm ${{ matrix.minor }} exposes the upgrade-path subcommands/flags"
+
+  # ADR-13 Tier-1 e2e suite: real kubeadm in a privileged systemd node container.
+  # Runs all scenarios in test/e2e/ (init-converge, reset, init-clobber-refusal,
+  # upgrade-skew-refusal, worker-join) against the real provider binary.
+  #
+  # SECURITY NOTE (privileged containers, ADR-13 E3 gate): the e2e node container
+  # must run --privileged because kubeadm/kubelet/containerd/etcd require real
+  # cgroup v2 control, mount namespaces, netns creation, and /sys/fs/cgroup write
+  # access -- unavailable without privilege inside a container. This is the same
+  # constraint kind (Kubernetes-in-Docker) has. The only image that runs privileged
+  # is OUR OWN built kairos-kubeadm-e2e-node image, derived from the kairos-kubeadm
+  # base we build in this same job from the checkout source. No third-party image
+  # is ever pulled. Ephemeral per-run kubeadm bootstrap tokens (bounded TTL, never
+  # persisted) are the only secrets; no long-lived credentials are embedded. Teardown
+  # is guaranteed: t.Cleanup registers docker rm -fv per container, and the failure
+  # step below sweeps leaked containers by label.
+  #
+  # Image strategy: option (a) -- rebuild kairos-kubeadm + e2e-node in this job
+  # using the same stable-<minor>.txt patch-resolution as the image job. This is
+  # self-contained (no artifact upload/download between jobs), fault-isolated
+  # per-minor-leg, and avoids docker/actions-artifact size limits. The ~4 min
+  # build overhead is well within the 30 min per-leg timeout.
+  #
+  # GitHub-hosted ubuntu-latest supports --privileged containers, /boot/config-*,
+  # and /lib/modules. The harness mounts those read-only so kubeadm's
+  # SystemVerification preflight can read the kernel config without relaxing any
+  # preflight check.
+  e2e:
+    name: e2e (k8s ${{ matrix.minor }})
+    needs: discover
+    # SECURITY (ADR-13 E3, security-architect R1): this job runs a --privileged
+    # node container (real kubeadm/kubelet/containerd) on attacker-influenceable
+    # PR code. The load-bearing controls are: (1) GitHub-HOSTED ephemeral runner
+    # only -- NEVER a self-hosted runner; (2) NO secrets and an explicit
+    # read-only token (below) -- NEVER add secrets or packages:write; (3) the
+    # `pull_request` trigger (NOT pull_request_target), so fork PRs get no secrets.
+    # These four conditions, not the container flags, are what contain a hostile PR.
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 50
+    strategy:
+      fail-fast: false
+      matrix:
+        minor: ${{ fromJSON(needs.discover.outputs.minors) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+          check-latest: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Resolve Kubernetes patch + crictl versions
+        id: ver
+        run: |
+          set -euo pipefail
+          k8s="$(curl -fsSL "https://dl.k8s.io/release/stable-${{ matrix.minor }}.txt")"
+          case "$k8s" in
+            v${{ matrix.minor }}.*) : ;;
+            *) echo "unexpected patch ${k8s} for minor ${{ matrix.minor }}" >&2; exit 1 ;;
+          esac
+          crictl="v${{ matrix.minor }}.0"
+          echo "k8s=${k8s} crictl=${crictl}"
+          echo "k8s=${k8s}" >> "$GITHUB_OUTPUT"
+          echo "crictl=${crictl}" >> "$GITHUB_OUTPUT"
+
+      - name: Build kairos-kubeadm base image
+        run: |
+          make image \
+            VERSION=ci-${{ matrix.minor }} \
+            KUBERNETES_VERSION=${{ steps.ver.outputs.k8s }} \
+            CRICTL_VERSION=${{ steps.ver.outputs.crictl }} \
+            IMAGE=kairos-kubeadm:${{ steps.ver.outputs.k8s }}
+
+      - name: Build e2e node image
+        run: |
+          make e2e-node-image \
+            KUBERNETES_VERSION=${{ steps.ver.outputs.k8s }} \
+            BASE_IMAGE=kairos-kubeadm:${{ steps.ver.outputs.k8s }} \
+            E2E_NODE_IMAGE=kairos-kubeadm-e2e-node:${{ steps.ver.outputs.k8s }}
+
+      - name: Run e2e suite
+        run: |
+          E2E_NODE_IMAGE=kairos-kubeadm-e2e-node:${{ steps.ver.outputs.k8s }} \
+          E2E_KUBERNETES_VERSION=${{ steps.ver.outputs.k8s }} \
+          go test -tags e2e -count=1 -timeout 45m -v ./test/e2e/...
+
+      - name: Prune e2e container volumes and images on failure
+        if: failure()
+        run: |
+          docker ps -aq --filter 'label=provider-kubernetes-e2e=1' \
+            | xargs -r docker rm -fv || true
+          docker image rm -f \
+            "kairos-kubeadm-e2e-node:${{ steps.ver.outputs.k8s }}" \
+            "kairos-kubeadm:${{ steps.ver.outputs.k8s }}" || true
+          docker volume prune -f || true

Dockerfile.e2e-node

@@ -0,0 +1,76 @@
+# Dockerfile.e2e-node -- a kind-style "systemd-as-PID1 node container" for the
+# provider-kubernetes e2e harness (ADR-13, slice E2).
+#
+# It FROM-derives a built kairos-kubeadm image so it REUSES the
+# checksum-verified toolchain (kubeadm / kubelet / kubectl / containerd / runc /
+# CNI) already baked there -- nothing is re-downloaded here. We only add the
+# tweaks a real kubeadm+kubelet+containerd need to run inside a privileged
+# container on cgroup v2 with the systemd cgroup driver, mirroring the
+# upstream-blessed kind node-image pattern.
+#
+# Build (via the Makefile, which sets BASE_IMAGE for you):
+#   make e2e-node-image KUBERNETES_VERSION=v1.34.0
+# or directly:
+#   docker build \
+#     --build-arg BASE_IMAGE=kairos-kubeadm:v1.34.0 \
+#     -f Dockerfile.e2e-node -t kairos-kubeadm-e2e-node:v1.34.0 .
+#
+# Run (the harness does this for you; recorded here for reference):
+#   docker run -d --privileged \
+#     --cgroupns=private \
+#     --tmpfs /run --tmpfs /tmp \
+#     -v /var/lib/containerd -v /var/lib/kubelet -v /var/lib/etcd \
+#     --entrypoint /sbin/init \
+#     kairos-kubeadm-e2e-node:v1.34.0
+
+ARG BASE_IMAGE=kairos-kubeadm:v1.34.0
+FROM ${BASE_IMAGE}
+
+# kubeadm refuses to run with swap on by default; a container has no swap, but be
+# explicit and disable the kubelet's swap-detection nuisance the way kind does.
+ENV container=docker
+
+# --- kind systemd-in-container tweaks -------------------------------------
+# 1. Mask units that are meaningless or harmful inside a container (they try to
+#    touch real hardware / the host kernel and would fail or hang). This is the
+#    same set kind masks in its node image entrypoint.
+# 2. Ensure the journal goes to a writable location and systemd does not try to
+#    set up a separate cgroup hierarchy (we run cgroup v2 unified, delegated).
+# 3. Enable containerd + kubelet so systemd brings them up at boot (the base
+#    image already `systemctl enable`s them, but we re-assert idempotently).
+RUN set -eux; \
+    # Units that must not run in a container.
+    for unit in \
+      systemd-udevd.service \
+      systemd-udevd-kernel.socket \
+      systemd-udevd-control.socket \
+      systemd-modules-load.service \
+      sys-kernel-config.mount \
+      sys-kernel-debug.mount \
+      sys-kernel-tracing.mount \
+      systemd-journald-audit.socket ; do \
+        systemctl mask "$unit" || true; \
+    done; \
+    # Do not let getty/console units spin.
+    systemctl mask getty.target console-getty.service || true; \
+    # Make sure containerd + kubelet start under systemd.
+    systemctl enable containerd.service kubelet.service || true
+
+# The kubelet must not be gated on swap; kind passes failSwapOn=false via the
+# kubelet config. kubeadm 1.34 defaults to handling swap, but a privileged
+# container can momentarily see host swap, so we belt-and-braces the flag through
+# the kubeadm extra-args drop-in the unit already sources (/etc/default/kubelet).
+RUN set -eux; \
+    mkdir -p /etc/default; \
+    printf 'KUBELET_EXTRA_ARGS=--fail-swap-on=false\n' > /etc/default/kubelet
+
+# Declare the stateful directories as volumes so each container run gets a fresh,
+# isolated, writable layer for them (kind keeps these out of the image layer for
+# correctness and teardown). The harness also passes anonymous -v for these.
+VOLUME [ "/var/lib/containerd", "/var/lib/kubelet", "/var/lib/etcd" ]
+
+# systemd is PID 1. The kairos base image is systemd-based, so /sbin/init is
+# systemd. STOPSIGNAL SIGRTMIN+3 is systemd's clean-shutdown signal (kind uses
+# the same), so `docker stop` shuts the node down gracefully.
+STOPSIGNAL SIGRTMIN+3
+ENTRYPOINT [ "/sbin/init" ]

Makefile

@@ -18,7 +18,15 @@ RUNC_VERSION       ?= v1.3.0
 CNI_PLUGINS_VERSION ?= v1.8.0
 IMAGE              ?= kairos-kubeadm:$(VERSION)
 
-.PHONY: all build test vet fmt fmt-check lint tidy image clean
+# e2e (ADR-13). The node image FROM-derives the kairos-kubeadm base so it reuses
+# the checksum-verified toolchain. BASE_IMAGE defaults to the per-version
+# kairos-kubeadm tag; E2E_NODE_IMAGE is the derived systemd-node image the
+# harness runs. E2E_TIMEOUT is a never-hang backstop for the whole suite.
+BASE_IMAGE       ?= kairos-kubeadm:$(KUBERNETES_VERSION)
+E2E_NODE_IMAGE   ?= kairos-kubeadm-e2e-node:$(KUBERNETES_VERSION)
+E2E_TIMEOUT      ?= 40m
+
+.PHONY: all build test vet fmt fmt-check lint tidy image clean e2e-node-image e2e
 
 all: build
 
@@ -62,6 +70,30 @@ image:
 	  --build-arg PROVIDER_VERSION=$(VERSION) \
 	  -t $(IMAGE) .
 
+## e2e-node-image: build the systemd-as-PID1 node container image (ADR-13 E2),
+## FROM-deriving the kairos-kubeadm base. If the base image is absent, build it
+## first with: make image KUBERNETES_VERSION=$(KUBERNETES_VERSION) VERSION=$(KUBERNETES_VERSION) IMAGE=$(BASE_IMAGE)
+e2e-node-image:
+	@if ! docker image inspect $(BASE_IMAGE) >/dev/null 2>&1; then \
+	  echo "ERROR: base image $(BASE_IMAGE) not found."; \
+	  echo "Build it first:"; \
+	  echo "  make image KUBERNETES_VERSION=$(KUBERNETES_VERSION) VERSION=$(KUBERNETES_VERSION) IMAGE=$(BASE_IMAGE)"; \
+	  exit 1; \
+	fi
+	docker build \
+	  --build-arg BASE_IMAGE=$(BASE_IMAGE) \
+	  -f Dockerfile.e2e-node \
+	  -t $(E2E_NODE_IMAGE) .
+
+## e2e: run the tag-gated e2e suite (ADR-13 E1) against a real kubeadm node
+## container. Builds the node image first. Requires Docker with privileged
+## containers. The e2e files are behind //go:build e2e, so `make test` is
+## unaffected.
+e2e: e2e-node-image
+	E2E_NODE_IMAGE=$(E2E_NODE_IMAGE) \
+	E2E_KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
+	$(GO) test -tags e2e -timeout $(E2E_TIMEOUT) -v ./test/e2e/...
+
 ## clean: remove build artifacts
 clean:
 	rm -rf bin coverage.out

internal/reconcile/budget.go

@@ -18,12 +18,41 @@ type Budget struct {
 	Total time.Duration
 }
 
-// DefaultBudget returns conservative bounded defaults. These exist so that an
-// unbounded operation is impossible by construction; tune per-action as needed.
+// DefaultBudget returns the bounded default applied to every non-upgrade action
+// (run-init, run-join, wait-for-control-plane). Sizing reflects two facts:
+//
+//   - PerAttempt=6m: the production image bundles the kubeadm/kubelet binaries but
+//     does NOT pre-pull the control-plane container images (etcd, apiserver,
+//     controller-manager, scheduler), so a real first-boot `kubeadm init` PULLS
+//     those images and then waits for them to become healthy. On a slow node this
+//     can run 3-6 minutes; the old 2m deadline SIGKILL-ed kubeadm via the expired
+//     context. 6m gives a cold init genuine headroom.
+//   - Total=8m (NOT PerAttempt*MaxAttempts): Total is the hard wall-clock ceiling
+//     and is deliberately the lever that bounds FAILURE-detection latency. The
+//     worst case is a node pointed at an unreachable control-plane endpoint
+//     (the [wait-for-control-plane, run-join] plan): wait-for-control-plane polls
+//     until its context expires, so its surfacing time is driven by the budget.
+//     With Total=8m the first 6m attempt runs in full, the second attempt starts
+//     but is capped by the remaining 2m, and the failure surfaces at ~8m -- the
+//     same ceiling the old 3x2m budget produced, NOT the ~12m a 2x6m budget would.
+//     This preserves fail-fast responsiveness (design principle 4 / #4099-1) while
+//     still tolerating a transient blip via the partial second attempt.
+//
+// MaxAttempts=2 (down from 3): with a 6m PerAttempt window each attempt already
+// absorbs short transient failures, so the second attempt is a safety net rather
+// than a retry loop. Note Total < PerAttempt*MaxAttempts by design: Total wins, so
+// the second attempt may be truncated. This is intentional -- the ceiling on how
+// long a failing node may block later Kairos boot stages is the priority.
+//
+// KNOWN LIMITATION (follow-up: per-action budgets): a single global PerAttempt
+// conflates a slow legitimate waiter (cold init) with a pure reachability probe
+// (wait-for-control-plane against a bad endpoint), which ideally fails in ~60-90s.
+// Splitting these requires the Reconciler to select a budget per Action; that
+// wiring is a separate slice. Until then Total=8m caps the bad-endpoint case.
 func DefaultBudget() Budget {
 	return Budget{
-		PerAttempt:  2 * time.Minute,
-		MaxAttempts: 3,
+		PerAttempt:  6 * time.Minute,
+		MaxAttempts: 2,
 		Total:       8 * time.Minute,
 	}
 }

main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/kairos-io/provider-kubernetes/internal/kubeadm"
 	"github.com/kairos-io/provider-kubernetes/internal/kubeadm/credential"
 	"github.com/kairos-io/provider-kubernetes/internal/provider"
+	"github.com/kairos-io/provider-kubernetes/internal/reset"
 	"github.com/kairos-io/provider-kubernetes/version"
 )
 
@@ -39,6 +40,8 @@ func main() {
 		switch args[0] {
 		case "reconcile":
 			os.Exit(runReconcile(args[1:]))
+		case "reset":
+			os.Exit(runReset(args[1:]))
 		case "mint-join":
 			os.Exit(runMintJoin(args[1:]))
 		case "version", "--version", "-v":
@@ -162,12 +165,62 @@ func runMintJoin(args []string) int {
 	return 0
 }
 
+// runReset performs a bounded cluster reset from a serialized Cluster YAML, exactly
+// as HandleClusterReset does for the pluggable event. This subcommand is the e2e
+// harness's entry point for the reset scenario (ADR-13 Tier-1 scenario 3); it is
+// also a useful operator escape hatch. It reads the cluster_root_path from
+// ProviderOptions and the optional CRI socket from the user config, then calls
+// reset.Run -- no shell, no interpolation, bounded (design principle 1 / #4099-1).
+func runReset(args []string) int {
+	fs := flag.NewFlagSet("reset", flag.ContinueOnError)
+	clusterFile := fs.String("cluster-file", provider.ClusterStatePath, "path to the serialized Cluster YAML")
+	if err := fs.Parse(args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return 2
+	}
+
+	logrus.Infof("provider-kubernetes reset %s: reading %s", version.Version, *clusterFile)
+
+	data, err := os.ReadFile(*clusterFile)
+	if err != nil {
+		logrus.Errorf("read cluster file: %v", err)
+		return 1
+	}
+	var cluster clusterplugin.Cluster
+	if err := yaml.Unmarshal(data, &cluster); err != nil {
+		logrus.Errorf("parse cluster file: %v", err)
+		return 1
+	}
+
+	rootPath := "/"
+	if v := cluster.ProviderOptions["cluster_root_path"]; v != "" {
+		rootPath = v
+	}
+
+	var criSocket string
+	if uc, ucErr := provider.ParseUserConfig(cluster.Options); ucErr == nil {
+		criSocket = uc.InitConfiguration.NodeRegistration.CRISocket
+	}
+
+	if err := reset.Run(context.Background(), reset.Options{
+		Runner:    kubeadm.ExecRunner{},
+		RootPath:  rootPath,
+		CRISocket: criSocket,
+	}); err != nil {
+		logrus.Errorf("cluster reset: %v", err)
+		return 1
+	}
+	logrus.Info("provider-kubernetes reset: done")
+	return 0
+}
+
 func printUsage(w *os.File) {
 	_, _ = fmt.Fprintf(w, `agent-provider-kubernetes %s
 
 Usage:
   agent-provider-kubernetes                 run the Kairos clusterplugin event handler (default)
   agent-provider-kubernetes reconcile [...] run one bounded reconcile pass for a serialized Cluster
+  agent-provider-kubernetes reset [...]     run a bounded cluster reset from a serialized Cluster
   agent-provider-kubernetes mint-join [...] mint join material on a CP and print a join cloud-config
   agent-provider-kubernetes version         print the build version