fix: apply k3s version bumps declaratively through site.yml #117

Workflow file for this run

.github/workflows/integration.yml at f601d9a

	---
	name: Integration Test

	on:
	pull_request:
	workflow_dispatch:

	jobs:
	test:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	service_mgr: [systemd, openrc]
	inventory: [basic, notoken]
	exclude:
	- service_mgr: openrc
	inventory: notoken
	env:
	STARTING_K3S_VERSION: v1.34.3+k3s1
	UPGRADE_K3S_VERSION: v1.35.1+k3s1
	# K3s requires privileged containers to run inside Docker and access to cgrougs.
	steps:
	- name: Set container OS based on service manager
	id: set-container
	run: \|
	if [ "${{ matrix.service_mgr }}" == "systemd" ]; then
	echo "container_os=rancher/systemd-node:v0.0.8" >> $GITHUB_ENV
	echo "container_cmd=/usr/lib/systemd/systemd --unit=noop.target --show-status=true" >> $GITHUB_ENV
	echo "service_mgr=systemd" >> $GITHUB_ENV
	else
	echo "container_os=jrei/openrc-alpine" >> $GITHUB_ENV
	echo "service_mgr=openrc" >> $GITHUB_ENV
	fi

	- name: Checkout codebase
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

	- name: Set up Python 3.13.
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: '3.13.x'

	- name: Install Ansible and dependencies
	run: \|
	pip install ansible
	ansible-galaxy collection install -r collections/requirements.yml

	- name: Verify Inventory
	run: ansible-inventory -i tests/${{ matrix.inventory }}.yml --list

	- name: Create Docker Network
	run: docker network create k3s-ansible

	- name: Start Docker containers
	run: \|
	# Start the Server node
	docker run -d --name server-node \
	--hostname server-node \
	--privileged \
	-v /sys/fs/bpf:/sys/fs/bpf \
	--memory 2048m \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-ansible \
	${{ env.container_os }} ${{ env.container_cmd }}

	# Start the Agent node
	docker run -d --name agent-node \
	--hostname agent-node \
	--privileged \
	-v /sys/fs/bpf:/sys/fs/bpf \
	--memory 2048m \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-ansible \
	${{ env.container_os }} ${{ env.container_cmd }}

	- name: Install node dependencies (systemd)
	if: matrix.service_mgr == 'systemd'
	run: \|
	docker exec server-node zypper install -y python3-rpm
	docker exec agent-node zypper install -y python3-rpm

	- name: Install node dependencies (openrc)
	if: matrix.service_mgr == 'openrc'
	run: \|
	docker exec server-node apk add curl python3
	docker exec agent-node apk add curl python3

	- name: Replace k3s_version in inventory
	run: \|
	sed -i "s/k3s_version: .*/k3s_version: ${STARTING_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml

	- name: Run Playbook
	env:
	ANSIBLE_FORCE_COLOR: '1'
	run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

	- name: Verify K3s is running on Server
	run: docker exec server-node k3s kubectl get nodes \| grep Ready

	- name: Verify K3s is running on Agent (systemd)
	if: matrix.service_mgr == 'systemd'
	run: docker exec agent-node systemctl status k3s-agent \| grep running

	- name: Verify K3s is running on Agent (openrc)
	if: matrix.service_mgr == 'openrc'
	run: docker exec agent-node rc-service k3s-agent status \| grep started

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Modify the k3s_version in inventory for upgrade
	run: \|
	sed -i "s/k3s_version: .*/k3s_version: ${UPGRADE_K3S_VERSION}/" tests/${{ matrix.inventory }}.yml

	# Re-running site.yml with a bumped version must upgrade the running
	# runtime on both the server and the agent, not just replace the binary.
	- name: Run site Playbook to upgrade via a version bump
	run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

	# The install script replaces the on-disk binary regardless of a restart,
	# so assert the running version the kubelet reports to the API: it only
	# advances once the service was actually restarted onto the new build.
	# This is what fails if the version-bump restart regresses.
	- name: Verify the running K3s version upgraded on Server and Agent
	run: \|
	for node in server-node agent-node; do
	for attempt in 1 2 3 4 5 6; do
	running=$(docker exec server-node k3s kubectl get node "$node" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
	echo "$node running version: $running"
	if echo "$running" \| grep -q "${UPGRADE_K3S_VERSION}"; then
	break
	fi
	if [ "$attempt" -eq 6 ]; then
	echo "$node did not reach ${UPGRADE_K3S_VERSION}"
	exit 1
	fi
	sleep 10
	done
	done

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Add node-label to server config
	run: \|
	printf " server_config_yaml: \|\n node-label: foo=bar\n" >> tests/${{ matrix.inventory }}.yml

	- name: Run site Playbook again to apply new server args
	run: ansible-playbook playbooks/site.yml -i tests/${{ matrix.inventory }}.yml

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Verify new server args applied
	run: \|
	for attempt in 1 2 3; do
	echo "Attempt $attempt: checking node label"
	output=$(docker exec server-node k3s kubectl get nodes -o jsonpath='{.items[*].metadata.labels.foo}' 2>&1)
	echo "$output"
	if echo "$output" \| grep -q bar; then
	exit 0
	fi
	if [ "$attempt" -lt 3 ]; then
	sleep 10
	fi
	done
	exit 1

	- name: Debug nodes/pods on failure
	if: failure()
	run: \|
	echo "NODE INFO"
	docker exec server-node k3s kubectl get nodes -o wide
	echo "POD INFO"
	docker exec server-node k3s kubectl get pods -A -o wide
	echo "CONFIG FILE"
	docker exec server-node cat /etc/rancher/k3s/config.yaml
	echo "Server ENV"
	if [ $service_mgr == "systemd" ]; then
	docker exec server-node cat /etc/systemd/system/k3s.service.env
	elif [ $service_mgr == "openrc" ]; then
	docker exec server-node cat /etc/rancher/k3s/k3s.env
	fi
	echo "Agent ENV"
	if [ $service_mgr == "systemd" ]; then
	docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
	elif [ $service_mgr == "openrc" ]; then
	docker exec agent-node cat /etc/rancher/k3s/k3s-agent.env
	fi
	echo "SERVER LOGS"
	if [ $service_mgr == "systemd" ]; then
	docker exec server-node journalctl -u k3s -n 10
	elif [ $service_mgr == "openrc" ]; then
	tail -n 10 /var/log/k3s.log
	fi


	- name: Remove K3s from Server and Agent
	run: ansible-playbook playbooks/reset.yml -i tests/${{ matrix.inventory }}.yml

	- name: Stop and remove Docker containers
	run: \|
	docker stop server-node && docker rm -f server-node
	docker stop agent-node && docker rm -f agent-node
	docker network rm k3s-ansible

	test-ha:
	runs-on: ubuntu-latest
	# K3s requires privileged containers to run inside Docker and access to cgrougs.
	env:
	SERVERS: server-node1 server-node2 server-node3
	steps:
	- name: Checkout codebase
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

	- name: Set up Python 3.13.
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: '3.13.x'

	- name: Install Ansible and dependencies
	run: \|
	pip install ansible
	ansible-galaxy collection install -r collections/requirements.yml

	- name: Verify Inventory
	run: ansible-inventory -i tests/ha.yml --list

	- name: Create Docker Network
	run: docker network create k3s-ha-ansible

	- name: Start Docker containers
	run: \|
	for SERVER in $SERVERS; do
	docker run -d --name $SERVER \
	--hostname $SERVER \
	--privileged \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-ha-ansible \
	rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true
	done

	- name: Install server dependencies
	run: \|
	for SERVER in $SERVERS; do
	docker exec $SERVER zypper install -y python3-rpm
	done

	- name: Run Playbook
	env:
	ANSIBLE_FORCE_COLOR: '1'
	run: ansible-playbook playbooks/site.yml -i tests/ha.yml

	- name: Verify K3s is running on servers
	run: docker exec server-node1 k3s kubectl get nodes \| grep Ready \| wc -l \| grep 3

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Modify the k3s_version in inventory for upgrade
	run: \|
	sed -i 's/k3s_version: v1.33.4+k3s1/k3s_version: v1.34.1+k3s1/' tests/ha.yml

	- name: Run Upgrade Playbook
	run: ansible-playbook playbooks/upgrade.yml -i tests/ha.yml

	- name: Verify K3s upgraded on all servers
	run: \|
	sleep 5 && docker exec server-node1 k3s kubectl get nodes
	for SERVER in $SERVERS; do
	docker exec $SERVER k3s --version \| grep v1.34.
	done

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	# #485: re-running site.yml with a bumped version must upgrade every etcd
	# server, restarting them one at a time so the quorum is never lost.
	- name: Modify the k3s_version in inventory for a site.yml upgrade
	run: \|
	sed -i 's/k3s_version: v1.34.1+k3s1/k3s_version: v1.34.3+k3s1/' tests/ha.yml

	- name: Run site Playbook to upgrade the servers via a version bump
	run: ansible-playbook playbooks/site.yml -i tests/ha.yml

	# Assert the running (kubelet-reported) version, not the on-disk binary:
	# without a rolling restart the joined servers stay on the old runtime.
	- name: Verify the running K3s version upgraded on all servers via site.yml
	run: \|
	docker exec server-node1 k3s kubectl get nodes -o wide
	for SERVER in $SERVERS; do
	for attempt in 1 2 3 4 5 6; do
	running=$(docker exec server-node1 k3s kubectl get node "$SERVER" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>&1)
	echo "$SERVER running version: $running"
	if echo "$running" \| grep -q v1.34.3; then
	break
	fi
	if [ "$attempt" -eq 6 ]; then
	echo "$SERVER did not reach v1.34.3"
	exit 1
	fi
	sleep 10
	done
	done

	- name: Wait for all deployments to be ready after the site.yml upgrade
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node1 k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Debug nodes/pods on failure
	if: failure()
	run: \|
	echo "NODE INFO"
	docker exec server-node1 k3s kubectl get nodes -o wide
	echo "POD INFO"
	docker exec server-node1 k3s kubectl get pods -A -o wide
	echo "CONFIG FILES"
	for SERVER in $SERVERS; do
	docker exec $SERVER cat /etc/rancher/k3s/config.yaml
	done
	echo "Server ENVS"
	for SERVER in $SERVERS; do
	docker exec $SERVER cat /etc/systemd/system/k3s.service.env
	done
	echo "SERVER LOGSS"
	docker exec server-node1 journalctl -u k3s -n 10

	- name: Remove K3s from server nodes
	run: ansible-playbook playbooks/reset.yml -i tests/ha.yml

	- name: Stop and remove Docker containers
	run: \|
	for SERVER in $SERVERS; do
	docker stop $SERVER && docker rm -f $SERVER
	done
	docker network rm k3s-ha-ansible

	test-external:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout codebase
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

	- name: Set up Python 3.13.
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: '3.13.x'

	- name: Install Ansible and dependencies
	run: \|
	pip install ansible
	ansible-galaxy collection install -r collections/requirements.yml

	- name: Verify Inventory
	run: ansible-inventory -i tests/external.yml --list

	- name: Create Docker Network
	run: docker network create k3s-ext-ansible

	- name: Start Database container
	run: \|
	docker run -d --name mysqlDB \
	--network=k3s-ext-ansible \
	-p 3306:3306 -e MYSQL_ROOT_PASSWORD=e2e mysql:8.4

	- name: Start Server container
	run: \|
	docker run -d --name server-node \
	--privileged \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-ext-ansible \
	rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

	- name: Install server dependencies
	run: docker exec server-node zypper install -y python3-rpm

	- name: Run Playbook
	env:
	ANSIBLE_FORCE_COLOR: '1'
	run: ansible-playbook playbooks/site.yml -i tests/external.yml

	- name: Verify K3s is running on servers
	run: docker exec server-node k3s kubectl get nodes \| grep Ready \| wc -l \| grep 1

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Stop and remove Docker containers
	run: \|
	docker stop server-node && docker rm -f server-node
	docker stop mysqlDB && docker rm -f mysqlDB
	docker network rm k3s-ext-ansible

	test-airgap:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout codebase
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

	- name: Set up Python 3.13.
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: '3.13.x'

	- name: Install Ansible and dependencies
	run: \|
	pip install ansible
	ansible-galaxy collection install -r collections/requirements.yml

	- name: Verify Inventory
	run: ansible-inventory -i tests/airgap.yml --list

	- name: Create Docker Network
	run: docker network create k3s-airgp-ansible

	- name: Start containers
	run: \|
	docker run -d --name server-node \
	--privileged \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-airgp-ansible \
	rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

	docker run -d --name agent-node \
	--privileged \
	--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw \
	--volume=/lib/modules:/lib/modules:ro \
	--cgroupns=host \
	--network=k3s-airgp-ansible \
	rancher/systemd-node:v0.0.8 /usr/lib/systemd/systemd --unit=noop.target --show-status=true

	- name: Install node dependencies
	run: \|
	docker exec server-node zypper install -y python3-rpm
	docker exec agent-node zypper install -y python3-rpm

	- name: Download K3s airgap artifacts
	run: \|
	mkdir -p ./playbooks/test-airgap
	wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s-airgap-images-amd64.tar.gz
	wget -P ./playbooks/test-airgap https://github.com/k3s-io/k3s/releases/download/v1.33.8%2Bk3s1/k3s

	- name: Run Playbook
	env:
	ANSIBLE_FORCE_COLOR: '1'
	run: ansible-playbook playbooks/site.yml -i tests/airgap.yml

	- name: Verify K3s is running on servers
	run: docker exec server-node k3s kubectl get nodes \| grep Ready \| wc -l \| grep 2

	- name: Validate its the correct K3s version
	run: \|
	docker exec server-node k3s --version \| grep v1.33.8
	docker exec agent-node k3s --version \| grep v1.33.8

	- name: Wait for all deployments to be ready
	run: \|
	for attempt in 1 2 3 4 5 6; do
	echo "Attempt $attempt: checking deployments"
	output=$(docker exec server-node k3s kubectl get deployments -n kube-system -o jsonpath='{range .items[*]}{.metadata.name}={.status.readyReplicas}/{.spec.replicas}{"\n"}{end}' 2>&1)
	echo "$output"
	if ! echo "$output" \| grep -q "<no value>" && echo "$output" \| awk -F '[=/]' '{if ($2 != $3) exit 1}' ; then
	exit 0
	fi
	if [ "$attempt" -lt 6 ]; then
	sleep 15
	fi
	done
	exit 1

	- name: Debug nodes/pods on failure
	if: failure()
	run: \|
	echo "NODE INFO"
	docker exec server-node k3s kubectl get nodes -o wide
	echo "POD INFO"
	docker exec server-node k3s kubectl get pods -A -o wide
	echo "CONFIG FILE"
	docker exec server-node cat /etc/rancher/k3s/config.yaml
	echo "Server ENV"
	docker exec server-node cat /etc/systemd/system/k3s.service.env
	echo "Agent ENV"
	docker exec agent-node cat /etc/systemd/system/k3s-agent.service.env
	echo "SERVER LOGS"
	docker exec server-node journalctl -u k3s -n 10

	- name: Stop and remove Docker containers
	run: \|
	docker stop server-node && docker rm -f server-node
	docker stop agent-node && docker rm -f agent-node
	docker network rm k3s-airgp-ansible

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

fix: apply k3s version bumps declaratively through site.yml #117

Workflow file

fix: apply k3s version bumps declaratively through site.yml #117

Uh oh!

Workflow file for this run