Skip to content

Commit f80e82b

Browse files
authored
ci: retry cert-manager apply after k3s apiserver bounce (#593)
Nightly #592 died on a single-shot kubectl apply while k3s was still restarting kubelet after image import (OpenAPI download failed with ServiceUnavailable). Share one helper between nightly and PR CI: poll /readyz, retry apply three times, then wait for Deployments. Closes #592 Signed-off-by: Sebastien Tardif <sebtardif@ncf.ca>
1 parent 62c8d50 commit f80e82b

5 files changed

Lines changed: 242 additions & 21 deletions

File tree

.github/actions/setup-e2e-cluster/action.yaml

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -210,26 +210,13 @@ runs:
210210
211211
- name: Install cert-manager
212212
shell: bash -Eeuo pipefail -x {0}
213-
run: |
214-
for attempt in 1 2 3; do
215-
if kubectl apply -f /tmp/cert-manager.yaml; then
216-
break
217-
fi
218-
echo "::warning::cert-manager install attempt $attempt failed, retrying in 15s..."
219-
sleep 15
220-
if (( attempt == 3 )); then
221-
echo "::error::cert-manager install failed after 3 attempts"
222-
exit 1
223-
fi
224-
done
225-
kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=120s
226-
kubectl wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s
227-
kubectl wait --for=condition=Available deployment/cert-manager-cainjector -n cert-manager --timeout=120s
213+
run: bash hack/e2e-install-cert-manager.sh /tmp/cert-manager.yaml
228214

229215
- name: Load Prometheus and test images into cluster
230216
shell: bash -Eeuo pipefail -x {0}
231217
run: |
232218
k3d image import /tmp/prometheus.tar /tmp/stress-ng.tar -c "${{ inputs.cluster-name }}"
219+
bash hack/e2e-install-cert-manager.sh --wait-only
233220
234221
- name: Install Prometheus
235222
shell: bash -Eeuo pipefail -x {0}
@@ -293,6 +280,7 @@ runs:
293280
--bare --tags=e2e --platform=linux/$(go env GOARCH) \
294281
--tarball=/tmp/attune-e2e.tar --push=false
295282
k3d image import /tmp/attune-e2e.tar -c "${{ inputs.cluster-name }}"
283+
bash hack/e2e-install-cert-manager.sh --wait-only
296284
297285
- name: Install operator via Helm
298286
shell: bash -Eeuo pipefail -x {0}

.github/workflows/e2e-nightly.yaml

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -408,15 +408,13 @@ jobs:
408408
409409
- name: Install cert-manager
410410
shell: bash -Eeuo pipefail -x {0}
411-
run: |
412-
kubectl apply -f /tmp/cert-manager.yaml
413-
kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=120s
414-
kubectl wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s
415-
kubectl wait --for=condition=Available deployment/cert-manager-cainjector -n cert-manager --timeout=120s
411+
run: bash hack/e2e-install-cert-manager.sh /tmp/cert-manager.yaml
416412

417413
- name: Load Prometheus and test images into cluster
418414
shell: bash -Eeuo pipefail -x {0}
419-
run: k3d image import /tmp/prometheus.tar /tmp/stress-ng.tar /tmp/busybox.tar -c "$CLUSTER_NAME"
415+
run: |
416+
k3d image import /tmp/prometheus.tar /tmp/stress-ng.tar /tmp/busybox.tar -c "$CLUSTER_NAME"
417+
bash hack/e2e-install-cert-manager.sh --wait-only
420418
421419
- name: Install Prometheus
422420
shell: bash -Eeuo pipefail -x {0}
@@ -505,6 +503,7 @@ jobs:
505503
exit 1
506504
fi
507505
done
506+
bash hack/e2e-install-cert-manager.sh --wait-only
508507
509508
- name: Install operator via Helm
510509
shell: bash -Eeuo pipefail -x {0}

Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ python-test: ## Run helper script tests (fossa-filter, run-fuzz classifier, go-v
240240
bash scripts/test_verify_go_version_sync.sh
241241
bash scripts/test_verify_helm_image_tag.sh
242242
bash scripts/test_release_image_tags.sh
243+
bash scripts/test_e2e_install_cert_manager.sh
243244

244245
.PHONY: test-bench
245246
test-bench: ## Run benchmark tests

hack/e2e-install-cert-manager.sh

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2026 attune Authors
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Apply cert-manager after k3s may still be bouncing (kubelet restart
6+
# after k3d image import). Poll /readyz, retry kubectl apply, then wait
7+
# for the three cert-manager Deployments.
8+
#
9+
# Used by e2e-nightly.yaml and setup-e2e-cluster so the two paths cannot
10+
# drift. Override KUBECTL / READYZ_* / APPLY_* in unit tests.
11+
12+
set -euo pipefail
13+
14+
KUBECTL="${KUBECTL:-kubectl}"
15+
READYZ_ATTEMPTS="${READYZ_ATTEMPTS:-60}"
16+
READYZ_SLEEP="${READYZ_SLEEP:-2}"
17+
APPLY_ATTEMPTS="${APPLY_ATTEMPTS:-3}"
18+
APPLY_RETRY_SLEEP="${APPLY_RETRY_SLEEP:-15}"
19+
MODE="apply"
20+
MANIFEST=""
21+
22+
if [[ "${1:-}" == "--wait-only" ]]; then
23+
MODE="wait"
24+
elif [[ -n "${1:-}" ]]; then
25+
MANIFEST="$1"
26+
else
27+
echo "usage: e2e-install-cert-manager.sh MANIFEST | --wait-only" >&2
28+
exit 2
29+
fi
30+
31+
wait_readyz() {
32+
local attempt
33+
for attempt in $(seq 1 "${READYZ_ATTEMPTS}"); do
34+
if "${KUBECTL}" get --raw='/readyz' >/dev/null 2>&1; then
35+
echo "API server ready after ${attempt} attempt(s)"
36+
return 0
37+
fi
38+
if (( attempt == READYZ_ATTEMPTS )); then
39+
echo "::error::API server never became ready (readyz)"
40+
return 1
41+
fi
42+
sleep "${READYZ_SLEEP}"
43+
done
44+
}
45+
46+
wait_readyz
47+
if [[ "${MODE}" == "wait" ]]; then
48+
exit 0
49+
fi
50+
51+
attempt=1
52+
while (( attempt <= APPLY_ATTEMPTS )); do
53+
if "${KUBECTL}" apply -f "${MANIFEST}"; then
54+
break
55+
fi
56+
if (( attempt == APPLY_ATTEMPTS )); then
57+
echo "::error::cert-manager install failed after ${APPLY_ATTEMPTS} attempts"
58+
exit 1
59+
fi
60+
echo "::warning::cert-manager install attempt ${attempt} failed, retrying in ${APPLY_RETRY_SLEEP}s..."
61+
sleep "${APPLY_RETRY_SLEEP}"
62+
wait_readyz
63+
attempt=$((attempt + 1))
64+
done
65+
66+
"${KUBECTL}" wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=120s
67+
"${KUBECTL}" wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s
68+
"${KUBECTL}" wait --for=condition=Available deployment/cert-manager-cainjector -n cert-manager --timeout=120s
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2026 attune Authors
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Classifier for hack/e2e-install-cert-manager.sh: first-apply OpenAPI
6+
# failures must retry; a permanently down apiserver must fail closed.
7+
# Also assert nightly and the PR-CI composite both call the helper.
8+
9+
set -euo pipefail
10+
11+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
12+
SCRIPT="${ROOT}/hack/e2e-install-cert-manager.sh"
13+
NIGHTLY="${ROOT}/.github/workflows/e2e-nightly.yaml"
14+
COMPOSITE="${ROOT}/.github/actions/setup-e2e-cluster/action.yaml"
15+
16+
echo "PLAN: classify e2e-install-cert-manager.sh"
17+
18+
fail() {
19+
echo "FAIL: $*"
20+
echo "DONE: ok=false"
21+
exit 1
22+
}
23+
24+
[[ -f "${SCRIPT}" ]] || fail "missing ${SCRIPT}"
25+
26+
echo "DO: nightly and setup-e2e-cluster must call the helper"
27+
grep -F -q 'hack/e2e-install-cert-manager.sh' "${NIGHTLY}" \
28+
|| fail "e2e-nightly.yaml does not invoke hack/e2e-install-cert-manager.sh"
29+
grep -F -q 'hack/e2e-install-cert-manager.sh' "${COMPOSITE}" \
30+
|| fail "setup-e2e-cluster/action.yaml does not invoke hack/e2e-install-cert-manager.sh"
31+
if grep -n 'kubectl apply -f /tmp/cert-manager.yaml' "${NIGHTLY}" "${COMPOSITE}"; then
32+
fail "bare kubectl apply of cert-manager.yaml still present; use the helper"
33+
fi
34+
echo "OK: both install paths call the helper"
35+
36+
TMP="$(mktemp -d)"
37+
trap 'rm -rf "${TMP}"' EXIT
38+
MANIFEST="${TMP}/cert-manager.yaml"
39+
printf '%s\n' 'apiVersion: v1' 'kind: ConfigMap' >"${MANIFEST}"
40+
41+
write_kubectl() {
42+
local apply_fails="$1" readyz_fails="$2"
43+
cat >"${TMP}/kubectl" <<EOF
44+
#!/usr/bin/env bash
45+
set -euo pipefail
46+
STATE="${TMP}/state"
47+
mkdir -p "\${STATE}"
48+
cmd="\${1:-}"
49+
shift || true
50+
case "\${cmd}" in
51+
get)
52+
if [[ "\${*}" == *readyz* ]]; then
53+
n=\$(cat "\${STATE}/readyz" 2>/dev/null || echo 0)
54+
n=\$((n + 1))
55+
echo "\${n}" >"\${STATE}/readyz"
56+
if (( n <= ${readyz_fails} )); then
57+
echo "Error from server (ServiceUnavailable): apiserver not ready" >&2
58+
exit 1
59+
fi
60+
echo ok
61+
exit 0
62+
fi
63+
echo "unexpected kubectl get \$*" >&2
64+
exit 2
65+
;;
66+
apply)
67+
n=\$(cat "\${STATE}/apply" 2>/dev/null || echo 0)
68+
n=\$((n + 1))
69+
echo "\${n}" >"\${STATE}/apply"
70+
if (( n <= ${apply_fails} )); then
71+
echo 'error: error validating "${MANIFEST}": error validating data: failed to download openapi: the server is currently unable to handle the request' >&2
72+
exit 1
73+
fi
74+
echo "configmap/ok created"
75+
exit 0
76+
;;
77+
wait)
78+
echo "deployment.apps/cert-manager condition met"
79+
exit 0
80+
;;
81+
*)
82+
echo "unexpected kubectl \${cmd} \$*" >&2
83+
exit 2
84+
;;
85+
esac
86+
EOF
87+
chmod +x "${TMP}/kubectl"
88+
}
89+
90+
run_helper() {
91+
KUBECTL="${TMP}/kubectl" \
92+
READYZ_ATTEMPTS=5 \
93+
READYZ_SLEEP=0 \
94+
APPLY_ATTEMPTS=3 \
95+
APPLY_RETRY_SLEEP=0 \
96+
bash "${SCRIPT}" "${MANIFEST}"
97+
}
98+
99+
echo "DO: first apply OpenAPI failure then success must pass"
100+
rm -rf "${TMP}/state"
101+
write_kubectl 1 0
102+
if ! run_helper; then
103+
fail "retry after one apply failure should succeed"
104+
fi
105+
apply_n="$(cat "${TMP}/state/apply")"
106+
[[ "${apply_n}" == "2" ]] || fail "expected 2 apply calls, got ${apply_n}"
107+
echo "OK: retried apply after OpenAPI failure"
108+
109+
echo "DO: first readyz miss then apply success must pass"
110+
rm -rf "${TMP}/state"
111+
write_kubectl 0 1
112+
if ! run_helper; then
113+
fail "readyz miss then success should succeed"
114+
fi
115+
readyz_n="$(cat "${TMP}/state/readyz")"
116+
[[ "${readyz_n}" == "2" ]] || fail "expected 2 readyz calls, got ${readyz_n}"
117+
echo "OK: waited for readyz before apply"
118+
119+
echo "DO: three apply failures must fail closed"
120+
rm -rf "${TMP}/state"
121+
write_kubectl 99 0
122+
if run_helper; then
123+
fail "permanent apply failure should exit non-zero"
124+
fi
125+
apply_n="$(cat "${TMP}/state/apply")"
126+
[[ "${apply_n}" == "3" ]] || fail "expected 3 apply calls, got ${apply_n}"
127+
echo "OK: exhausted apply retries"
128+
129+
echo "DO: never-ready apiserver must fail closed"
130+
rm -rf "${TMP}/state"
131+
write_kubectl 0 99
132+
if run_helper; then
133+
fail "never-ready readyz should exit non-zero"
134+
fi
135+
if [[ -f "${TMP}/state/apply" ]]; then
136+
fail "apply must not run when readyz never succeeds"
137+
fi
138+
echo "OK: fail-closed when apiserver never becomes ready"
139+
140+
echo "DO: --wait-only must poll readyz and skip apply"
141+
rm -rf "${TMP}/state"
142+
write_kubectl 0 1
143+
if ! KUBECTL="${TMP}/kubectl" READYZ_ATTEMPTS=5 READYZ_SLEEP=0 \
144+
bash "${SCRIPT}" --wait-only; then
145+
fail "--wait-only should succeed after one readyz miss"
146+
fi
147+
if [[ -f "${TMP}/state/apply" ]]; then
148+
fail "--wait-only must not call kubectl apply"
149+
fi
150+
readyz_n="$(cat "${TMP}/state/readyz")"
151+
[[ "${readyz_n}" == "2" ]] || fail "--wait-only expected 2 readyz calls, got ${readyz_n}"
152+
echo "OK: --wait-only"
153+
154+
echo "DO: both paths must re-wait after Prometheus image import"
155+
grep -F -A6 'Load Prometheus and test images into cluster' "${NIGHTLY}" \
156+
| grep -F -q 'hack/e2e-install-cert-manager.sh --wait-only' \
157+
|| fail "e2e-nightly.yaml missing --wait-only after Prometheus image import"
158+
grep -F -A6 'Load Prometheus and test images into cluster' "${COMPOSITE}" \
159+
| grep -F -q 'hack/e2e-install-cert-manager.sh --wait-only' \
160+
|| fail "setup-e2e-cluster missing --wait-only after Prometheus image import"
161+
echo "OK: Prometheus import waits for readyz"
162+
163+
echo "DONE: ok=true"
164+
echo "NEXT: none"
165+
exit 0

0 commit comments

Comments
 (0)