Skip to content

Commit bfbe8d4

Browse files
pdmackpdmack
authored
Merge branch 'main' into fix/fern-mdx-sanitization
2 parents 258b670 + 90a7f41 commit bfbe8d4

1 file changed

Lines changed: 96 additions & 24 deletions

File tree

‎.github/actions/e2e/action.yml‎

Lines changed: 96 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -43,38 +43,98 @@ runs:
4343
- name: Install E2E testing tools
4444
uses: ./.github/actions/install-e2e-tools
4545

46-
- name: Create Kind cluster
47-
shell: bash
48-
run: make cluster-create
49-
50-
- name: Deploy aicrd to Kind
46+
- name: Create Kind cluster and deploy aicrd
5147
shell: bash
5248
env:
5349
GOFLAGS: -mod=vendor
5450
run: |
55-
# Build and push aicrd image (replaces Tilt custom_build)
51+
# If any `wait` below fails, `set -e` (GitHub's default shell flags)
52+
# exits this script immediately - any background job not yet reached
53+
# by its own `wait` would otherwise keep running. Most consequential
54+
# for pid_cluster specifically: the job's always()-run Cleanup step
55+
# (`make cluster-delete`) could then race a still-in-flight
56+
# `make cluster-create`. Kill whatever is still alive on any exit.
57+
cleanup_background_jobs() {
58+
local rc=$?
59+
for pid in "$pid_cluster" "$pid_dep" "$pid_perf" "$pid_conf" "$pid_cli" "$pid_aicr" "$pid_validators"; do
60+
[ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && kill "$pid" 2>/dev/null
61+
done
62+
exit "$rc"
63+
}
64+
trap cleanup_background_jobs EXIT
65+
66+
# This step folds what used to be two separate steps (Create Kind
67+
# cluster / Deploy aicrd) into one, so GitHub's per-step red-X no
68+
# longer says at a glance which phase failed. Group markers fold the
69+
# log into the same phases instead.
70+
echo "::group::cluster-and-builds"
71+
# cluster-create (~35s locally; Kind bring-up + registry) and the
72+
# host compiles below touch none of each other's inputs/outputs, so
73+
# launch everything in the background up front. Each later command
74+
# waits only on the specific PID(s) its own inputs actually depend
75+
# on: aicrd's ko build needs just the cluster/registry (not the
76+
# validator/CLI binaries), so it must not be gated behind builds it
77+
# doesn't read from - an earlier version of this step wait'ed on all
78+
# four build PIDs before it, which measured ~19s slower in CI than
79+
# the original sequential script for no reason: nothing here reads
80+
# dist/validator/* or dist/e2e/aicr until the docker build step below.
81+
pid_cluster=""; pid_dep=""; pid_perf=""; pid_conf=""; pid_cli=""; pid_aicr=""; pid_validators=""
82+
make cluster-create > /tmp/cluster-create.log 2>&1 &
83+
pid_cluster=$!
84+
85+
mkdir -p dist/validator dist/e2e
86+
for phase in deployment performance conformance; do
87+
mkdir -p "validators/${phase}/testdata"
88+
done
89+
90+
# Compile validator binaries + the host aicr CLI on the runner (Go
91+
# build cache hit) while the cluster comes up. This is also much
92+
# faster than building inside Docker (no cache sharing there). Go's
93+
# build cache is safe for concurrent use by simultaneous go
94+
# commands, including ko's own internal compile of aicrd below.
95+
CGO_ENABLED=0 go build -trimpath -o dist/validator/deployment ./validators/deployment &
96+
pid_dep=$!
97+
CGO_ENABLED=0 go build -trimpath -o dist/validator/performance ./validators/performance &
98+
pid_perf=$!
99+
CGO_ENABLED=0 go build -trimpath -o dist/validator/conformance ./validators/conformance &
100+
pid_conf=$!
101+
go build -o dist/e2e/aicr ./cmd/aicr &
102+
pid_cli=$!
103+
104+
# aicrd only needs the cluster/registry - build and push it (replaces
105+
# Tilt custom_build) as soon as that's ready, not gated behind the
106+
# unrelated validator/CLI builds above.
107+
#
108+
# cluster-create's output was redirected to a file above so it
109+
# wouldn't interleave with the concurrent builds; print it on
110+
# failure too (not just success), since a failed `wait` here is the
111+
# most common, most opaque failure on this step and would otherwise
112+
# exit via the EXIT trap with zero diagnostic output.
113+
wait $pid_cluster || { cat /tmp/cluster-create.log; exit 1; }
114+
cat /tmp/cluster-create.log
115+
echo "::endgroup::"
116+
117+
echo "::group::deploy-aicrd"
56118
KO_DOCKER_REPO=localhost:5001/aicrd ko build --bare --tags=tilt ./cmd/aicrd
57119
58120
# Apply namespace first (must exist before deployment references it)
59121
kubectl apply -f tilt/k8s/namespace.yaml
60122
kubectl apply -f tilt/k8s/
61-
62-
# Build images + CLI binary in parallel while deployment starts
123+
echo "::endgroup::"
124+
125+
echo "::group::validator-images"
126+
# The validator images COPY the binaries built above, so wait for
127+
# those specifically (not pid_cli, which nothing here needs) before
128+
# building them.
129+
wait $pid_dep
130+
wait $pid_perf
131+
wait $pid_conf
132+
133+
# Build + push the remaining images. Validator binaries are already
134+
# compiled above; this is just the COPY-only image build + push.
63135
KO_DOCKER_REPO=localhost:5001/aicr ko build --bare --tags=local ./cmd/aicr &
64136
pid_aicr=$!
65-
# Compile validator binaries on host (Go build cache hit) then COPY-only images.
66-
# This is much faster than building inside Docker (no cache sharing).
67137
(
68-
mkdir -p dist/validator
69-
CGO_ENABLED=0 go build -trimpath -o dist/validator/deployment ./validators/deployment &
70-
CGO_ENABLED=0 go build -trimpath -o dist/validator/performance ./validators/performance &
71-
CGO_ENABLED=0 go build -trimpath -o dist/validator/conformance ./validators/conformance &
72-
wait
73-
# Build per-phase COPY-only images and push to local registry.
74-
# Ensure testdata dirs exist (conformance has none).
75-
for phase in deployment performance conformance; do
76-
mkdir -p "validators/${phase}/testdata"
77-
done
78138
for phase in deployment performance conformance; do
79139
docker build -t "localhost:5001/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
80140
FROM nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
@@ -99,10 +159,14 @@ runs:
99159
docker push "localhost:5001/aicr-validators/aiperf-bench:latest"
100160
) &
101161
pid_validators=$!
102-
go build -o dist/e2e/aicr ./cmd/aicr &
103-
pid_cli=$!
104-
wait $pid_aicr $pid_validators $pid_cli
105-
162+
# Separate waits, not `wait $pid_aicr $pid_validators`: bash returns
163+
# only the LAST pid's exit status from a combined wait, so a failed
164+
# pid_aicr would be silently masked whenever pid_validators succeeds.
165+
wait $pid_aicr
166+
wait $pid_validators
167+
echo "::endgroup::"
168+
169+
echo "::group::rollout-and-verify"
106170
# Wait for deployment rollout (may already be ready)
107171
kubectl rollout status deployment/aicrd -n aicr --timeout=120s
108172
@@ -112,6 +176,13 @@ runs:
112176
curl -sf http://localhost:5001/v2/aicr-validators/performance/tags/list
113177
curl -sf http://localhost:5001/v2/aicr-validators/conformance/tags/list
114178
curl -sf http://localhost:5001/v2/aicr-validators/aiperf-bench/tags/list
179+
echo "::endgroup::"
180+
181+
# Not needed by anything above, but its output (dist/e2e/aicr) is
182+
# used by a later step (AICR_BIN) - wait now so a build failure here
183+
# fails this step instead of surfacing as a confusing missing-binary
184+
# error two steps later, and so the step's own exit code reflects it.
185+
wait $pid_cli
115186
116187
- name: Set up fake GPU environment
117188
shell: bash
@@ -151,6 +222,7 @@ runs:
151222
shell: bash
152223
run: |
153224
mkdir -p /tmp/debug-artifacts
225+
cp /tmp/cluster-create.log /tmp/debug-artifacts/cluster-create.log 2>/dev/null || true
154226
kubectl get all --all-namespaces > /tmp/debug-artifacts/all-resources.txt || true
155227
kubectl get events --all-namespaces --sort-by='.lastTimestamp' > /tmp/debug-artifacts/events.txt || true
156228
kubectl logs -n aicr -l app.kubernetes.io/name=aicrd --tail=500 > /tmp/debug-artifacts/aicrd-logs.txt || true

0 commit comments

Comments
 (0)