Skip to content

Commit da95626

Browse files
efiacorefiacor
authored
Improve CI deploy reliability, move CR cache tests to nightly (kptdev#1002)
* Improve CI deploy reliability and move CR cache tests to nightly - Add 5-minute timeout to kpt live apply to prevent silent hangs - Add explicit --timeout to kubectl rollout status (fail fast, not silent) - Move CR cache e2e suites to a nightly schedule workflow - Keep DB cache + v1alpha2 CRD tests on every PR (primary path) - Expand cluster dump on failure: node resources, pod resource usage, services, PVCs, CRDs, APIServices, full porch-system pod details Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> * Address review: gate postgres wait on DB cache, guard grep in dump - Only wait for porch-postgresql statefulset when PORCH_CACHE_TYPE=DB - Add || true to grep fallback in cluster dump to prevent step failure when grep finds no matches under bash -e -o pipefail Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> * Address review: portable timeout, best-effort dump, robust log export - Replace GNU timeout with portable scripts/run-with-timeout.sh wrapper that falls back to POSIX background process on macOS - Add || true / 2>/dev/null to all namespace-scoped kubectl commands in cluster dump to prevent step abort if namespace doesn't exist - Use kubectl logs -l <selector> for log export instead of resolving a single pod name, handles zero/multi-pod cases gracefully Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> * Address review: gate rollout waits, harden timeout script Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> * Trigger CI Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> * Trigger CI Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech> --------- Signed-off-by: Fiachra Corcoran <fiachra.corcoran@est.tech>
1 parent f465a5c commit da95626

4 files changed

Lines changed: 306 additions & 28 deletions

File tree

‎.github/workflows/porch-e2e-ci-jobs.yaml‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -120,21 +120,11 @@ jobs:
120120
fail-fast: false
121121
matrix:
122122
include:
123-
- name: "Porch E2E Tests"
124-
make_target: "run-in-kind"
125-
test_path: "${GITHUB_WORKSPACE}/test/e2e/api"
126-
test_env: "E2E=1"
127-
log_prefix: "porch-e2e"
128123
- name: "Porch E2E Tests (DB Cache)"
129124
make_target: "run-in-kind-db-cache-push-drafts"
130125
test_path: "${GITHUB_WORKSPACE}/test/e2e/api"
131126
test_env: "E2E=1 DB_CACHE=1"
132127
log_prefix: "porch-e2e-dbcache"
133-
- name: "Porch CLI E2E Tests"
134-
make_target: "run-in-kind"
135-
test_path: "${GITHUB_WORKSPACE}/test/e2e/cli"
136-
test_env: "E2E=1"
137-
log_prefix: "porch-cli-e2e"
138128
- name: "Porch CLI E2E Tests (DB Cache)"
139129
make_target: "run-in-kind-db-cache-push-drafts"
140130
test_path: "${GITHUB_WORKSPACE}/test/e2e/cli"
@@ -181,34 +171,64 @@ jobs:
181171
- name: Dump cluster state on deploy failure
182172
if: failure()
183173
run: |
174+
echo "=== Node Resources ==="
175+
kubectl top nodes 2>/dev/null || kubectl describe nodes | grep -A 5 "Allocated resources" || true
176+
echo ""
184177
echo "=== Pods (all namespaces) ==="
185-
kubectl get pods -A
186-
echo "=== Events (porch-system) ==="
187-
kubectl get events -n porch-system --sort-by='.lastTimestamp' | tail -40
188-
echo "=== Non-running pods details ==="
189-
kubectl get pods -A -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | "\n--- \(.metadata.namespace)/\(.metadata.name) (\(.status.phase)) ---"'
190-
for pod in $(kubectl get pods -A -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | "\(.metadata.namespace)/\(.metadata.name)"'); do
178+
kubectl get pods -A -o wide || true
179+
echo ""
180+
echo "=== Pod Resource Usage ==="
181+
kubectl top pods -A 2>/dev/null || true
182+
echo ""
183+
echo "=== Services (porch-system) ==="
184+
kubectl get svc -n porch-system 2>/dev/null || true
185+
echo ""
186+
echo "=== PVCs (porch-system) ==="
187+
kubectl get pvc -n porch-system 2>/dev/null || true
188+
echo ""
189+
echo "=== ConfigMaps (porch-system) ==="
190+
kubectl get configmaps -n porch-system 2>/dev/null || true
191+
echo ""
192+
echo "=== CRDs ==="
193+
kubectl get crds | grep -E "porch|kpt" || true
194+
echo ""
195+
echo "=== APIServices ==="
196+
kubectl get apiservices | grep -E "porch|kpt" || true
197+
echo ""
198+
echo "=== Events (porch-system, last 60) ==="
199+
kubectl get events -n porch-system --sort-by='.lastTimestamp' 2>/dev/null | tail -60 || true
200+
echo ""
201+
echo "=== Events (kube-system, last 20) ==="
202+
kubectl get events -n kube-system --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || true
203+
echo ""
204+
echo "=== All pods details (porch-system) ==="
205+
for pod in $(kubectl get pods -n porch-system -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
206+
echo ""
207+
echo "--- Pod: porch-system/$pod ---"
208+
kubectl describe pod -n porch-system "$pod" 2>/dev/null | grep -A 30 "Conditions:" || true
209+
echo "--- Logs: porch-system/$pod (last 50 lines, all containers) ---"
210+
kubectl logs -n porch-system "$pod" --all-containers --tail=50 2>/dev/null || true
211+
done
212+
echo ""
213+
echo "=== Non-running pods (other namespaces) ==="
214+
for pod in $(kubectl get pods -A -o json 2>/dev/null | jq -r '.items[] | select(.metadata.namespace != "porch-system") | select(.status.phase != "Running" and .status.phase != "Succeeded") | "\(.metadata.namespace)/\(.metadata.name)"'); do
191215
ns=$(echo $pod | cut -d/ -f1)
192216
name=$(echo $pod | cut -d/ -f2)
193217
echo "--- Describe $ns/$name ---"
194-
kubectl describe pod -n $ns $name | tail -20
218+
kubectl describe pod -n "$ns" "$name" 2>/dev/null | tail -20 || true
195219
echo "--- Logs $ns/$name ---"
196-
kubectl logs -n $ns $name --all-containers --tail=30 2>/dev/null || true
220+
kubectl logs -n "$ns" "$name" --all-containers --tail=30 2>/dev/null || true
197221
done
198222
- name: Run E2E tests
199223
run: ${{ matrix.test_env }} go test -v -timeout 20m ${{ matrix.test_path }}
200224
- name: Export porch server logs
201225
if: always()
202226
run: |
203-
name=$(kubectl -n porch-system get pod -l app=porch-server -o custom-columns=NAME:.metadata.name --no-headers=true)
204-
kubectl -n porch-system logs $name > ${{ matrix.log_prefix }}-server.log
227+
kubectl -n porch-system logs -l app=porch-server --all-containers --tail=-1 > ${{ matrix.log_prefix }}-server.log 2>/dev/null || true
205228
- name: Export porch controllers logs
206229
if: always()
207230
run: |
208-
name=$(kubectl -n porch-system get pod -l k8s-app=porch-controllers -o custom-columns=NAME:.metadata.name --no-headers=true 2>/dev/null || true)
209-
if [ -n "$name" ]; then
210-
kubectl -n porch-system logs $name > ${{ matrix.log_prefix }}-controllers.log 2>/dev/null || true
211-
fi
231+
kubectl -n porch-system logs -l k8s-app=porch-controllers --all-containers --tail=-1 > ${{ matrix.log_prefix }}-controllers.log 2>/dev/null || true
212232
- name: Archive logs
213233
if: always()
214234
uses: actions/upload-artifact@v4

‎.github/workflows/porch-e2e-nightly.yaml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Copyright 2026 The kpt Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: "E2E Tests (CR Cache - Nightly)"
16+
on:
17+
schedule:
18+
- cron: "0 3 * * *" # 03:00 UTC daily
19+
workflow_dispatch:
20+
21+
concurrency:
22+
group: ${{ github.workflow }}-${{ github.ref }}
23+
cancel-in-progress: true
24+
25+
env:
26+
IMAGE_REPO: porch-kind
27+
KIND_CONTEXT_NAME: porch-test
28+
PORCH_FUNCTION_RUNNER_IMAGE: porch-function-runner
29+
PORCH_WRAPPER_SERVER_IMAGE: porch-wrapper-server
30+
PORCH_SERVER_IMAGE: porch-server
31+
PORCH_CONTROLLERS_IMAGE: porch-controllers
32+
33+
jobs:
34+
build:
35+
name: Build Dev Images
36+
runs-on: ubuntu-latest
37+
timeout-minutes: 15
38+
env:
39+
IMAGE_TAG: ${{ github.sha }}
40+
GOTOOLCHAIN: auto
41+
outputs:
42+
image-tag: ${{ steps.vars.outputs.image-tag }}
43+
steps:
44+
- name: Checkout Porch
45+
uses: actions/checkout@v4
46+
- name: Set up Go
47+
uses: ./.github/actions/setup-go-kpt
48+
with:
49+
install-kpt: "false"
50+
go-cache: "true"
51+
- name: Set variables
52+
id: vars
53+
run: |
54+
echo "image-tag=${IMAGE_TAG:0:8}" >> $GITHUB_OUTPUT
55+
echo "alpine-version=$(make --eval='print: ; @echo $(ALPINE_VERSION)' print)" >> $GITHUB_OUTPUT
56+
echo "golang-bookworm=$(make --eval='print: ; @echo $(GOLANG_BOOKWORM_VERSION)' print)" >> $GITHUB_OUTPUT
57+
echo "golang-alpine=$(make --eval='print: ; @echo $(GOLANG_ALPINE_VERSION)' print)" >> $GITHUB_OUTPUT
58+
- name: Build images in parallel
59+
run: |
60+
set -e
61+
export IMAGE_TAG=${IMAGE_TAG:0:8}
62+
export ALPINE_VERSION=${{ steps.vars.outputs.alpine-version }}
63+
export GOLANG_BOOKWORM_VERSION=${{ steps.vars.outputs.golang-bookworm }}
64+
export GOLANG_ALPINE_VERSION=${{ steps.vars.outputs.golang-alpine }}
65+
IMAGE_NAME=${PORCH_FUNCTION_RUNNER_IMAGE} WRAPPER_SERVER_IMAGE_NAME=${PORCH_WRAPPER_SERVER_IMAGE} make -C func/ build-image &
66+
pid1=$!
67+
IMAGE_NAME=${PORCH_SERVER_IMAGE} make -C build/ build-image &
68+
pid2=$!
69+
IMAGE_NAME=${PORCH_CONTROLLERS_IMAGE} make -C controllers/ build-image &
70+
pid3=$!
71+
wait $pid1 && wait $pid2 && wait $pid3
72+
- name: Save all images
73+
run: |
74+
export IMAGE_TAG=${IMAGE_TAG:0:8}
75+
docker save \
76+
${IMAGE_REPO}/${PORCH_SERVER_IMAGE}:${IMAGE_TAG} \
77+
${IMAGE_REPO}/${PORCH_CONTROLLERS_IMAGE}:${IMAGE_TAG} \
78+
${IMAGE_REPO}/${PORCH_FUNCTION_RUNNER_IMAGE}:${IMAGE_TAG} \
79+
${IMAGE_REPO}/${PORCH_WRAPPER_SERVER_IMAGE}:${IMAGE_TAG} \
80+
| gzip > porch-images.tar.gz
81+
- name: Upload images
82+
uses: actions/upload-artifact@v4
83+
with:
84+
name: porch-images
85+
path: porch-images.tar.gz
86+
compression-level: 0
87+
retention-days: 1
88+
89+
tests:
90+
name: ${{ matrix.name }}
91+
runs-on: ubuntu-latest
92+
timeout-minutes: 30
93+
needs: build
94+
env:
95+
IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
96+
strategy:
97+
fail-fast: false
98+
matrix:
99+
include:
100+
- name: "Porch E2E Tests (CR Cache)"
101+
make_target: "run-in-kind"
102+
test_path: "${GITHUB_WORKSPACE}/test/e2e/api"
103+
test_env: "E2E=1"
104+
log_prefix: "porch-e2e-crcache"
105+
- name: "Porch CLI E2E Tests (CR Cache)"
106+
make_target: "run-in-kind"
107+
test_path: "${GITHUB_WORKSPACE}/test/e2e/cli"
108+
test_env: "E2E=1"
109+
log_prefix: "porch-cli-e2e-crcache"
110+
111+
steps:
112+
- name: Checkout Porch
113+
uses: actions/checkout@v4
114+
- name: Set up Go and kpt
115+
uses: ./.github/actions/setup-go-kpt
116+
with:
117+
go-cache: "true"
118+
- name: Install Kind
119+
uses: helm/kind-action@v1
120+
with:
121+
version: v0.30.0
122+
install_only: true
123+
- name: Set up Git config
124+
run: |
125+
git config --global user.name "Porch E2E"
126+
git config --global user.email "porch-e2e@porch.dev"
127+
- name: Setup dev env
128+
run: ${GITHUB_WORKSPACE}/scripts/setup-dev-env.sh
129+
- name: Download image artifacts
130+
uses: actions/download-artifact@v4
131+
with:
132+
name: porch-images
133+
- name: Load images to kind
134+
run: |
135+
gunzip -c porch-images.tar.gz | docker load
136+
kind load docker-image ${IMAGE_REPO}/${PORCH_SERVER_IMAGE}:${{ needs.build.outputs.image-tag }} -n ${KIND_CONTEXT_NAME}
137+
kind load docker-image ${IMAGE_REPO}/${PORCH_CONTROLLERS_IMAGE}:${{ needs.build.outputs.image-tag }} -n ${KIND_CONTEXT_NAME}
138+
kind load docker-image ${IMAGE_REPO}/${PORCH_FUNCTION_RUNNER_IMAGE}:${{ needs.build.outputs.image-tag }} -n ${KIND_CONTEXT_NAME}
139+
kind load docker-image ${IMAGE_REPO}/${PORCH_WRAPPER_SERVER_IMAGE}:${{ needs.build.outputs.image-tag }} -n ${KIND_CONTEXT_NAME}
140+
- name: Deploy porch kpt pkg
141+
timeout-minutes: 10
142+
run: IMAGE_TAG=${{ needs.build.outputs.image-tag }} SKIP_IMG_BUILD=true make ${{ matrix.make_target }}
143+
- name: Dump cluster state on deploy failure
144+
if: failure()
145+
run: |
146+
echo "=== Node Resources ==="
147+
kubectl top nodes 2>/dev/null || kubectl describe nodes | grep -A 5 "Allocated resources" || true
148+
echo ""
149+
echo "=== Pods (all namespaces) ==="
150+
kubectl get pods -A -o wide || true
151+
echo ""
152+
echo "=== Pod Resource Usage ==="
153+
kubectl top pods -A 2>/dev/null || true
154+
echo ""
155+
echo "=== Services (porch-system) ==="
156+
kubectl get svc -n porch-system 2>/dev/null || true
157+
echo ""
158+
echo "=== PVCs (porch-system) ==="
159+
kubectl get pvc -n porch-system 2>/dev/null || true
160+
echo ""
161+
echo "=== CRDs ==="
162+
kubectl get crds | grep -E "porch|kpt" || true
163+
echo ""
164+
echo "=== APIServices ==="
165+
kubectl get apiservices | grep -E "porch|kpt" || true
166+
echo ""
167+
echo "=== Events (porch-system, last 60) ==="
168+
kubectl get events -n porch-system --sort-by='.lastTimestamp' 2>/dev/null | tail -60 || true
169+
echo ""
170+
echo "=== All pods details (porch-system) ==="
171+
for pod in $(kubectl get pods -n porch-system -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
172+
echo ""
173+
echo "--- Pod: porch-system/$pod ---"
174+
kubectl describe pod -n porch-system "$pod" 2>/dev/null | grep -A 30 "Conditions:" || true
175+
echo "--- Logs: porch-system/$pod (last 50 lines, all containers) ---"
176+
kubectl logs -n porch-system "$pod" --all-containers --tail=50 2>/dev/null || true
177+
done
178+
- name: Run E2E tests
179+
run: ${{ matrix.test_env }} go test -v -timeout 20m ${{ matrix.test_path }}
180+
- name: Export porch server logs
181+
if: always()
182+
run: |
183+
kubectl -n porch-system logs -l app=porch-server --all-containers --tail=-1 > ${{ matrix.log_prefix }}-server.log 2>/dev/null || true
184+
- name: Export porch controllers logs
185+
if: always()
186+
run: |
187+
kubectl -n porch-system logs -l k8s-app=porch-controllers --all-containers --tail=-1 > ${{ matrix.log_prefix }}-controllers.log 2>/dev/null || true
188+
- name: Archive logs
189+
if: always()
190+
uses: actions/upload-artifact@v4
191+
with:
192+
name: ${{ matrix.log_prefix }}-logs
193+
path: "*.log"
194+
compression-level: 0
195+
retention-days: 5

‎make/deploy.mk‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,10 +126,17 @@ load-images-to-kind:## Build porch images and load them into a kind cluster
126126
.PHONY: deploy-current-config
127127
deploy-current-config:## Deploy the configuration that is currently in $(DEPLOYPORCHCONFIGDIR)
128128
kpt live init $(DEPLOYPORCHCONFIGDIR) --name porch --namespace porch-system --inventory-id porch || true
129-
kpt live apply --inventory-policy=adopt --server-side --force-conflicts $(DEPLOYPORCHCONFIGDIR)
130-
@kubectl rollout status deployment function-runner --namespace porch-system 2>/dev/null || true
131-
@kubectl rollout status deployment porch-controllers --namespace porch-system 2>/dev/null || true
132-
@kubectl rollout status deployment porch-server --namespace porch-system 2>/dev/null || true
129+
./scripts/run-with-timeout.sh 300 kpt live apply --inventory-policy=adopt --server-side --force-conflicts $(DEPLOYPORCHCONFIGDIR)
130+
kubectl rollout status deployment function-runner --namespace porch-system --timeout=180s
131+
ifeq ($(PORCH_CACHE_TYPE),DB)
132+
kubectl rollout status statefulset porch-postgresql --namespace porch-system --timeout=180s
133+
endif
134+
ifneq ($(SKIP_PORCHSERVER_BUILD),true)
135+
kubectl rollout status deployment porch-server --namespace porch-system --timeout=180s
136+
endif
137+
ifneq ($(SKIP_CONTROLLER_BUILD),true)
138+
kubectl rollout status deployment porch-controllers --namespace porch-system --timeout=180s
139+
endif
133140
@echo "Done."
134141

135142
.PHONY: reload-function-runner

‎scripts/run-with-timeout.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2026 The kpt Authors
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Portable timeout wrapper that works on both Linux (GNU coreutils) and macOS.
17+
# Usage: run-with-timeout.sh <seconds> <command> [args...]
18+
19+
set -euo pipefail
20+
21+
if [ $# -lt 2 ]; then
22+
echo "Usage: $0 <timeout_seconds> <command> [args...]" >&2
23+
exit 1
24+
fi
25+
26+
TIMEOUT_SECS="$1"
27+
shift
28+
29+
if command -v timeout &>/dev/null; then
30+
# GNU coreutils timeout (Linux, or macOS with coreutils installed)
31+
timeout --kill-after=10 "${TIMEOUT_SECS}" "$@"
32+
else
33+
# POSIX fallback using process group for reliable subprocess cleanup.
34+
# Run the command in its own process group so we can kill the entire tree on timeout.
35+
set -m
36+
"$@" &
37+
pid=$!
38+
39+
(sleep "${TIMEOUT_SECS}" && kill -- -"$pid" 2>/dev/null && echo "ERROR: command timed out after ${TIMEOUT_SECS}s" >&2) &
40+
watchdog=$!
41+
42+
# Forward SIGINT/SIGTERM to the process group and clean up the watchdog.
43+
trap 'kill -- -"$pid" 2>/dev/null; kill "$watchdog" 2>/dev/null; exit 143' INT TERM
44+
45+
if wait "$pid"; then
46+
kill "$watchdog" 2>/dev/null || true
47+
wait "$watchdog" 2>/dev/null || true
48+
exit 0
49+
else
50+
status=$?
51+
kill -- -"$pid" 2>/dev/null || true
52+
kill "$watchdog" 2>/dev/null || true
53+
wait "$watchdog" 2>/dev/null || true
54+
exit $status
55+
fi
56+
fi

0 commit comments

Comments
 (0)