fix: apply k3s version bumps declaratively through site.yml

Re-running site.yml with a raised k3s_version replaced the on-disk
binary but left agents and joined servers running the old runtime,
because the install script skips the service start and the roles only
restarted on a config change. A version-only bump was therefore not
applied until the services were restarted by hand.

Always restart the k3s service in the server and agent roles on a
site.yml run, so the cluster reliably picks up a new config or runtime
without any logic to detect whether the binary changed. Run the server
play one host at a time and wait for every restarted server, including
the first, to report a healthy API before continuing, so a rolling
change never drops the etcd quorum. The joined-server readiness check
now waits for the servers handled so far instead of the full set, which
is correct under both serial and parallel runs.

Keep the dedicated upgrade.yml integration test and add a second
upgrade through site.yml, asserting the running version the kubelet
reports rather than the on-disk binary, so the tests fail if the
restart regresses. Document the declarative upgrade path in the README.

Co-authored-by: Dominik Wombacher <dominik@wombacher.cc>

Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

Author: lexfrei

.github/workflows/integration.yml

@@ -19,6 +19,7 @@ jobs:
     env:
       STARTING_K3S_VERSION: v1.34.3+k3s1
       UPGRADE_K3S_VERSION: v1.35.1+k3s1
+      SITE_UPGRADE_K3S_VERSION: v1.35.6+k3s1
     # K3s requires privileged containers to run inside Docker and access to cgrougs.
     steps:
       - name: Set container OS based on service manager
@@ -138,6 +139,37 @@ jobs:
       - name: Verify K3s upgraded on Agent
         run: docker exec agent-node k3s --version | grep ${UPGRADE_K3S_VERSION}
 
+      # Second upgrade, this time re-running site.yml with a bumped version: it
+      # must upgrade the running runtime on both the server and the agent, not
+      # just replace the on-disk binary.
+      - name: Modify the k3s_version in inventory for a site.yml upgrade
+        run: |
+          sed -i "s/k3s_version: .*/k3s_version: ${SITE_UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml
+
+      - name: Run site Playbook to upgrade via a version bump
+        run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml
+
+      # The install script replaces the on-disk binary regardless of a restart,
+      # so assert the running version the kubelet reports to the API: it only
+      # advances once the service was actually restarted onto the new build.
+      # This is what fails if the version-bump restart regresses.
+      - name: Verify the running K3s version upgraded on Server and Agent
+        run: |
+          for node in server-node agent-node; do
+            for attempt in 1 2 3 4 5 6; do
+              running=$(docker exec server-node k3s kubectl get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
+              echo "$node running version: $running"
+              if echo "$running" | grep -q "${SITE_UPGRADE_K3S_VERSION}"; then
+                break
+              fi
+              if [ "$attempt" -eq 6 ]; then
+                echo "$node did not reach ${SITE_UPGRADE_K3S_VERSION}"
+                exit 1
+              fi
+              sleep 10
+            done
+          done
+
       - name: Wait for all deployments to be ready
         run: |
           for attempt in 1 2 3 4 5 6; do
@@ -324,6 +356,50 @@ jobs:
           done
           exit 1
 
+      # #485: re-running site.yml with a bumped version must upgrade every etcd
+      # server, restarting them one at a time so the quorum is never lost.
+      - name: Modify the k3s_version in inventory for a site.yml upgrade
+        run: |
+          sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml
+
+      - name: Run site Playbook to upgrade the servers via a version bump
+        run: ansible-playbook playbooks/site.yml -i tests/ha.yml
+
+      # Assert the running (kubelet-reported) version, not the on-disk binary:
+      # without a rolling restart the joined servers stay on the old runtime.
+      - name: Verify the running K3s version upgraded on all servers via site.yml
+        run: |
+          docker exec server-node1 k3s kubectl get nodes -o wide
+          for SERVER in $SERVERS; do
+            for attempt in 1 2 3 4 5 6; do
+              running=$(docker exec server-node1 k3s kubectl get node "$SERVER" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
+              echo "$SERVER running version: $running"
+              if echo "$running" | grep -q v1.34.3; then
+                break
+              fi
+              if [ "$attempt" -eq 6 ]; then
+                echo "$SERVER did not reach v1.34.3"
+                exit 1
+              fi
+              sleep 10
+            done
+          done
+
+      - name: Wait for all deployments to be ready after the site.yml upgrade
+        run: |
+          for attempt in 1 2 3 4 5 6; do
+            echo "Attempt $attempt: checking deployments"
+            output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
+            echo "$output"
+            if ! echo "$output" | grep -q "<no value>" && echo "$output" | awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
+              exit 0
+            fi
+            if [ "$attempt" -lt 6 ]; then
+              sleep 15
+            fi
+          done
+          exit 1
+
       - name: Debug nodes/pods on failure
         if: failure()
         run: |

README.md

@@ -161,6 +161,8 @@ ansible-playbook k3s.orchestration.upgrade -i inventory.yml
 ansible-playbook playbooks/upgrade.yml -i inventory.yml
 ```
 
+Re-running the `site.yml` playbook after bumping `k3s_version` performs the same upgrade declaratively: it restarts the k3s services so the cluster picks up the new runtime, with servers restarted one at a time to keep the etcd quorum intact. The dedicated `upgrade.yml` playbook remains available and unchanged.
+
 ## Airgap Install
 
 Airgap installation is supported via the `airgap_dir` variable. This variable should be set to the path of a directory containing the K3s binary and images. The release artifacts can be downloaded from the [K3s Releases](https://github.com/k3s-io/k3s/releases). You must download the appropriate images for you architecture (any of the compression formats will work). Additionally, you must run the `airgap` role to set up the airgapped environment.

playbooks/site.yml

@@ -8,10 +8,14 @@
     - role: airgap
     - role: raspberrypi
 
+# Servers are handled one at a time so a rolling change (config edit or version
+# bump re-applied through this playbook) never restarts more than one server at
+# once and cannot break the etcd quorum. Agents have no such limitation.
 - name: Setup K3S server
   hosts: server
   gather_facts: false
   become: true
+  serial: 1
   roles:
     - role: k3s_server
 

roles/k3s_agent/tasks/main.yml

@@ -95,8 +95,12 @@
   loop:
     - "K3S_TOKEN={{ token }}"
 
+# Always restart so a re-run of site.yml — whether it changed the agent config
+# or bumped k3s_version — reliably brings the agent onto the new state. The
+# install script leaves the service stopped (INSTALL_K3S_SKIP_START), so a plain
+# restart also covers the first install.
 - name: Enable and start K3s agent
   ansible.builtin.service:
     name: k3s-agent
-    state: "{{ 'restarted' if _agent_config_result.changed else 'started' }}"
+    state: restarted
     enabled: true

roles/k3s_server/tasks/main.yml

@@ -175,6 +175,19 @@
         state: restarted
         enabled: true
 
+    # Hold the play here until the first server is actually healthy. Under
+    # serial: 1 this keeps the rolling guarantee symmetric: the play does not
+    # advance to the next server while this one is only listening but not ready.
+    - name: Wait for the first server API to report healthy
+      when: not ansible_check_mode
+      ansible.builtin.command:
+        cmd: k3s kubectl get --raw=/healthz
+      register: k3s_server_first_healthz
+      until: k3s_server_first_healthz.rc == 0 and k3s_server_first_healthz.stdout == 'ok'
+      retries: 30
+      delay: 5
+      changed_when: false
+
     - name: Pause to allow first server startup
       when: (groups[server_group] | length) > 1
       ansible.builtin.pause:
@@ -294,14 +307,30 @@
       ansible.builtin.systemd:
         daemon_reload: true
 
-    - name: Restart K3s service
+    # Always restart a joined server so a re-run of site.yml — whether it changed
+    # config or bumped the version — reliably brings it onto the new state, the
+    # same way the first server above always restarts. The play runs with
+    # serial: 1, so this happens one server at a time; the health check below
+    # blocks the next server until etcd quorum is back.
+    - name: Restart K3s and wait for it to be healthy
       when:
         - ansible_facts.services['k3s.service'] is defined
         - ansible_facts.services['k3s.service'].state == 'running'
-        - k3s_server_config_result.changed
-      ansible.builtin.service:
-        name: k3s
-        state: restarted
+      block:
+        - name: Restart K3s service
+          ansible.builtin.service:
+            name: k3s
+            state: restarted
+
+        - name: Wait for the restarted server API to report healthy
+          when: not ansible_check_mode
+          ansible.builtin.command:
+            cmd: k3s kubectl get --raw=/healthz
+          register: k3s_server_healthz
+          until: k3s_server_healthz.rc == 0 and k3s_server_healthz.stdout == 'ok'
+          retries: 30
+          delay: 5
+          changed_when: false
 
     - name: Enable and start K3s service
       when: ansible_facts.services['k3s.service'] is not defined or ansible_facts.services['k3s.service'].state != 'running'
@@ -310,13 +339,18 @@
         state: started
         enabled: true
 
-    - name: Verify that all server nodes joined
+    # Under serial: 1 the servers come up one at a time, so wait for the count
+    # joined so far (this host's position in the play) rather than the full set.
+    # This condition is also satisfied when the play runs without serial.
+    - name: Verify that this and previously handled server nodes joined
       when: not ansible_check_mode and (groups[server_group] | length) > 1
+      vars:
+        _k3s_expected_servers: "{{ ansible_play_hosts_all.index(inventory_hostname) + 1 }}"
       ansible.builtin.command:
         cmd: >
           k3s kubectl get nodes -l "node-role.kubernetes.io/control-plane=true" -o=jsonpath="{.items[*].metadata.name}"
       register: k3s_server_nodes
-      until: k3s_server_nodes.rc == 0 and (k3s_server_nodes.stdout.split() | length) == (groups[server_group] | length)
+      until: k3s_server_nodes.rc == 0 and (k3s_server_nodes.stdout.split() | length) >= (_k3s_expected_servers | int)
       retries: 20
       delay: 10
       changed_when: false