@@ -43,38 +43,98 @@ runs:
4343 - name : Install E2E testing tools
4444 uses : ./.github/actions/install-e2e-tools
4545
46- - name : Create Kind cluster
47- shell : bash
48- run : make cluster-create
49-
50- - name : Deploy aicrd to Kind
46+ - name : Create Kind cluster and deploy aicrd
5147 shell : bash
5248 env :
5349 GOFLAGS : -mod=vendor
5450 run : |
55- # Build and push aicrd image (replaces Tilt custom_build)
51+ # If any `wait` below fails, `set -e` (GitHub's default shell flags)
52+ # exits this script immediately - any background job not yet reached
53+ # by its own `wait` would otherwise keep running. Most consequential
54+ # for pid_cluster specifically: the job's always()-run Cleanup step
55+ # (`make cluster-delete`) could then race a still-in-flight
56+ # `make cluster-create`. Kill whatever is still alive on any exit.
57+ cleanup_background_jobs() {
58+ local rc=$?
59+ for pid in "$pid_cluster" "$pid_dep" "$pid_perf" "$pid_conf" "$pid_cli" "$pid_aicr" "$pid_validators"; do
60+ [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && kill "$pid" 2>/dev/null
61+ done
62+ exit "$rc"
63+ }
64+ trap cleanup_background_jobs EXIT
65+
66+ # This step folds what used to be two separate steps (Create Kind
67+ # cluster / Deploy aicrd) into one, so GitHub's per-step red-X no
68+ # longer says at a glance which phase failed. Group markers fold the
69+ # log into the same phases instead.
70+ echo "::group::cluster-and-builds"
71+ # cluster-create (~35s locally; Kind bring-up + registry) and the
72+ # host compiles below touch none of each other's inputs/outputs, so
73+ # launch everything in the background up front. Each later command
74+ # waits only on the specific PID(s) its own inputs actually depend
75+ # on: aicrd's ko build needs just the cluster/registry (not the
76+ # validator/CLI binaries), so it must not be gated behind builds it
77+ # doesn't read from - an earlier version of this step wait'ed on all
78+ # four build PIDs before it, which measured ~19s slower in CI than
79+ # the original sequential script for no reason: nothing here reads
80+ # dist/validator/* or dist/e2e/aicr until the docker build step below.
81+ pid_cluster=""; pid_dep=""; pid_perf=""; pid_conf=""; pid_cli=""; pid_aicr=""; pid_validators=""
82+ make cluster-create > /tmp/cluster-create.log 2>&1 &
83+ pid_cluster=$!
84+
85+ mkdir -p dist/validator dist/e2e
86+ for phase in deployment performance conformance; do
87+ mkdir -p "validators/${phase}/testdata"
88+ done
89+
90+ # Compile validator binaries + the host aicr CLI on the runner (Go
91+ # build cache hit) while the cluster comes up. This is also much
92+ # faster than building inside Docker (no cache sharing there). Go's
93+ # build cache is safe for concurrent use by simultaneous go
94+ # commands, including ko's own internal compile of aicrd below.
95+ CGO_ENABLED=0 go build -trimpath -o dist/validator/deployment ./validators/deployment &
96+ pid_dep=$!
97+ CGO_ENABLED=0 go build -trimpath -o dist/validator/performance ./validators/performance &
98+ pid_perf=$!
99+ CGO_ENABLED=0 go build -trimpath -o dist/validator/conformance ./validators/conformance &
100+ pid_conf=$!
101+ go build -o dist/e2e/aicr ./cmd/aicr &
102+ pid_cli=$!
103+
104+ # aicrd only needs the cluster/registry - build and push it (replaces
105+ # Tilt custom_build) as soon as that's ready, not gated behind the
106+ # unrelated validator/CLI builds above.
107+ #
108+ # cluster-create's output was redirected to a file above so it
109+ # wouldn't interleave with the concurrent builds; print it on
110+ # failure too (not just success), since a failed `wait` here is the
111+ # most common, most opaque failure on this step and would otherwise
112+ # exit via the EXIT trap with zero diagnostic output.
113+ wait $pid_cluster || { cat /tmp/cluster-create.log; exit 1; }
114+ cat /tmp/cluster-create.log
115+ echo "::endgroup::"
116+
117+ echo "::group::deploy-aicrd"
56118 KO_DOCKER_REPO=localhost:5001/aicrd ko build --bare --tags=tilt ./cmd/aicrd
57119
58120 # Apply namespace first (must exist before deployment references it)
59121 kubectl apply -f tilt/k8s/namespace.yaml
60122 kubectl apply -f tilt/k8s/
61-
62- # Build images + CLI binary in parallel while deployment starts
123+ echo "::endgroup::"
124+
125+ echo "::group::validator-images"
126+ # The validator images COPY the binaries built above, so wait for
127+ # those specifically (not pid_cli, which nothing here needs) before
128+ # building them.
129+ wait $pid_dep
130+ wait $pid_perf
131+ wait $pid_conf
132+
133+ # Build + push the remaining images. Validator binaries are already
134+ # compiled above; this is just the COPY-only image build + push.
63135 KO_DOCKER_REPO=localhost:5001/aicr ko build --bare --tags=local ./cmd/aicr &
64136 pid_aicr=$!
65- # Compile validator binaries on host (Go build cache hit) then COPY-only images.
66- # This is much faster than building inside Docker (no cache sharing).
67137 (
68- mkdir -p dist/validator
69- CGO_ENABLED=0 go build -trimpath -o dist/validator/deployment ./validators/deployment &
70- CGO_ENABLED=0 go build -trimpath -o dist/validator/performance ./validators/performance &
71- CGO_ENABLED=0 go build -trimpath -o dist/validator/conformance ./validators/conformance &
72- wait
73- # Build per-phase COPY-only images and push to local registry.
74- # Ensure testdata dirs exist (conformance has none).
75- for phase in deployment performance conformance; do
76- mkdir -p "validators/${phase}/testdata"
77- done
78138 for phase in deployment performance conformance; do
79139 docker build -t "localhost:5001/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
80140 FROM nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
@@ -99,10 +159,14 @@ runs:
99159 docker push "localhost:5001/aicr-validators/aiperf-bench:latest"
100160 ) &
101161 pid_validators=$!
102- go build -o dist/e2e/aicr ./cmd/aicr &
103- pid_cli=$!
104- wait $pid_aicr $pid_validators $pid_cli
105-
162+ # Separate waits, not `wait $pid_aicr $pid_validators`: bash returns
163+ # only the LAST pid's exit status from a combined wait, so a failed
164+ # pid_aicr would be silently masked whenever pid_validators succeeds.
165+ wait $pid_aicr
166+ wait $pid_validators
167+ echo "::endgroup::"
168+
169+ echo "::group::rollout-and-verify"
106170 # Wait for deployment rollout (may already be ready)
107171 kubectl rollout status deployment/aicrd -n aicr --timeout=120s
108172
@@ -112,6 +176,13 @@ runs:
112176 curl -sf http://localhost:5001/v2/aicr-validators/performance/tags/list
113177 curl -sf http://localhost:5001/v2/aicr-validators/conformance/tags/list
114178 curl -sf http://localhost:5001/v2/aicr-validators/aiperf-bench/tags/list
179+ echo "::endgroup::"
180+
181+ # Not needed by anything above, but its output (dist/e2e/aicr) is
182+ # used by a later step (AICR_BIN) - wait now so a build failure here
183+ # fails this step instead of surfacing as a confusing missing-binary
184+ # error two steps later, and so the step's own exit code reflects it.
185+ wait $pid_cli
115186
116187name : Set up fake GPU environment
117188 shell : bash
@@ -151,6 +222,7 @@ runs:
151222 shell : bash
152223 run : |
153224 mkdir -p /tmp/debug-artifacts
225+ cp /tmp/cluster-create.log /tmp/debug-artifacts/cluster-create.log 2>/dev/null || true
154226 kubectl get all --all-namespaces > /tmp/debug-artifacts/all-resources.txt || true
155227 kubectl get events --all-namespaces --sort-by='.lastTimestamp' > /tmp/debug-artifacts/events.txt || true
156228 kubectl logs -n aicr -l app.kubernetes.io/name=aicrd --tail=500 > /tmp/debug-artifacts/aicrd-logs.txt || true
0 commit comments