ci: staged per-arch pipelines — deps→cache→build→fan-out on one hot runner (#78)

* perf(ci): lambda-tests follows the prod arch

The lambda binary ships and runs on aarch64 only; running its server-off
test config on the x86 lane too bought a duplicate workspace compile+test
(~3-4 min now that the checks actually run tests) without covering anything
the aarch64-linux and darwin lanes don't. Chaining it into nextest instead
was measured and rejected: feature unification with dev-deps makes the
test-profile lib a distinct compilation unit (699/699 tests pass either
way, but zero compile reuse — 229s chained vs 233s unchained locally).

* ci: staged per-arch pipelines — deps first, cache, build, fan out on the hot runner

Each arch runs one pipeline: build the crane dependency closures (crate
deps; wasm toolchain on arm) and publish the binary cache IMMEDIATELY, so
a failure in any later stage never costs the next run its warm deps and
every branch benefits from the first build; then the source-keyed
build/test/lint stage; then the cheap fan-out — image smoke and the node
addon — as steps on the same already-hot runner instead of fresh runners
each paying installer+eval+restore.

Arch-independent checks (clippy/fmt/no-node-spawn/wasm) move to the arm
lane (~1.8x faster per stage), rebalancing the two critical paths. The
release workflow stops saving the cache: the deps stage is the single
publisher, so entries hold exactly the reusable closure and no
source-keyed artifacts. buildId untouched (checks and workflows are
outside its hash set).

* ci: bound check parallelism (OOM) and strip ANSI in the ran-tests guard

The rebalanced arm lane's unbounded derivation parallelism OOM-killed the
16 GB runner (exit 143); --max-jobs 2 bounds memory with cores saturated.
The zero-tests guard could never match a CI log: CARGO_TERM_COLOR=always
wraps the nextest summary in escapes — strip before grepping.

* ci: ran-tests guard must not use grep -q under pipefail

-q exits on first match, SIGPIPEs sed, and pipefail converts the
successful match into a failure — it only fired when matches occur early
in a long log (lambda-tests), which is why nextest passed the same guard.

Author: eordano

.github/workflows/ci.yml

@@ -124,22 +124,57 @@ jobs:
           path: .win-green-marker
           key: windows-green-${{ needs.windows-gate.outputs.hash }}
 
-  node-addon:
-    name: node addon
-    if: github.event_name != 'push'
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
+  # One staged pipeline per arch, everything on a single runner: deps are
+  # built and published to the binary cache FIRST (a later stage failing
+  # never costs the next run its warm deps), then the source-keyed
+  # build/test/lint work, then the cheap fan-out (image, node addon) reuses
+  # the already-hot store instead of paying setup+restore on fresh runners.
+  pipeline:
+    name: ${{ matrix.system }} pipeline
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            system: x86_64-linux
+            deps: .#checks.x86_64-linux.deps
+            image: dockerImage
+          - runner: ubuntu-24.04-arm
+            system: aarch64-linux
+            deps: .#checks.aarch64-linux.deps .#checks.aarch64-linux.wasm-deps
+            image: lambdaImage
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+      # No standalone nix installer: the composite runs it with the
+      # extra-conf this job needs; a prior bare install made that conf
+      # silently a no-op.
+      - uses: ./.github/actions/nix-store-cache
+      - name: deps (dependency closures)
+        run: nix build --no-link ${{ matrix.deps }}
+      # Publish immediately and from any branch: the composite no-ops
+      # unless the exact key is new, so this fires once per input rotation.
+      - uses: ./.github/actions/nix-store-cache/save
+        continue-on-error: true
+      - name: build + checks
+        run: bash ci/nix-checks.sh ${{ matrix.system }}
+      # Fan-out: cheap on the hot store. Skipped on main pushes, where the
+      # release workflow builds the real thing from the same cache.
+      - name: image smoke (${{ matrix.image }})
+        if: github.event_name != 'push'
+        run: nix build --no-link .#${{ matrix.image }}
       - uses: ./.github/actions/rust-setup
+        if: github.event_name != 'push'
         with:
-          # Pure consumer of the cache the release napi linux leg saves from
-          # main pushes (same shared-key); this job is PR-only, so its own
-          # saves would be PR-scoped quota waste.
+          # Pure consumer of the cache the release napi linux legs save from
+          # main pushes (same shared-key); PR-scoped saves would be quota
+          # waste.
           workspaces: crate/abgen-node
           shared-key: abgen-node
           save-if: "false"
-      - name: abgen-node (build + smoke)
+      - name: node addon (build + smoke)
+        if: github.event_name != 'push'
         working-directory: crate/abgen-node
         run: |
           npm ci --no-audit --no-fund
@@ -153,29 +188,3 @@ jobs:
           bash ../../ci/check-glibc-floor.sh 2.34 \
             target/release/libabgen_node.so
 
-  nix-ci:
-    name: nix checks
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [ubuntu-24.04, ubuntu-24.04-arm]
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 45
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
-      # No standalone nix installer: the composite runs it with the
-      # extra-conf this job needs; a prior bare install made that conf
-      # silently a no-op.
-      - uses: ./.github/actions/nix-store-cache
-      - name: build flake checks
-        run: |
-          case "$(uname -m)" in
-            x86_64)  system=x86_64-linux ;;
-            aarch64) system=aarch64-linux ;;
-            *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;;
-          esac
-          bash ci/nix-checks.sh "$system"
-      - if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
-        uses: ./.github/actions/nix-store-cache/save
-        continue-on-error: true
-

.github/workflows/release.yml

@@ -392,10 +392,6 @@ jobs:
             cargo build --release --locked --target ${{ matrix.target }} -p abgen-native
           fi
 
-      - if: matrix.builder == 'nix' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
-        uses: ./.github/actions/nix-store-cache/save
-        continue-on-error: true
-
       - name: windows self-containment
         if: matrix.target == 'x86_64-pc-windows-gnu'
         run: |
@@ -829,9 +825,6 @@ jobs:
             echo "REF=${ref}"
             echo "IMG_SIZE=$(ls -lLh result | awk '{print $5}')"
           } >> "$GITHUB_ENV"
-      - if: needs.build-id.outputs.image_docker == '' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
-        uses: ./.github/actions/nix-store-cache/save
-        continue-on-error: true
       - name: upload image artifact (input-addressed)
         if: needs.build-id.outputs.image_docker == ''
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
@@ -906,9 +899,6 @@ jobs:
             echo "REF=${ref}"
             echo "IMG_SIZE=$(ls -lLh result | awk '{print $5}')"
           } >> "$GITHUB_ENV"
-      - if: needs.build-id.outputs.image_lambda == '' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
-        uses: ./.github/actions/nix-store-cache/save
-        continue-on-error: true
       - name: upload image artifact (input-addressed)
         if: needs.build-id.outputs.image_lambda == ''
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a

ci/nix-checks.sh

@@ -17,15 +17,21 @@ done <<<"$names"
 printf 'building %s check(s):\n' "${#attrs[@]}"
 printf '  %s\n' "${attrs[@]}"
 
-nix build --keep-going --no-link --log-format raw "${attrs[@]}" \
-  || nix build --keep-going --no-link --print-build-logs "${attrs[@]}"
+# --max-jobs 2: the full check set holds several workspace-scale compiles;
+# unbounded derivation parallelism OOM-kills the 16 GB runners. Two at a
+# time bounds memory while keeping the cores saturated.
+nix build --keep-going --no-link --log-format raw --max-jobs 2 "${attrs[@]}" \
+  || nix build --keep-going --no-link --print-build-logs --max-jobs 2 "${attrs[@]}"
 
 # A test check that executed zero tests is a silent no-op (doCheck=false
 # reached the derivation once and nobody noticed for weeks) — make that
 # state red forever: the built check's log must show a nonzero test count.
 assert_ran_tests() {
   local attr="$1" pattern="$2"
-  grep -qE "$pattern" <(nix log "$attr") \
+  # CARGO_TERM_COLOR=always wraps the summary in ANSI escapes; strip them
+  # or the pattern can never match a CI log. No grep -q: under pipefail its
+  # early exit SIGPIPEs sed and fails the pipeline on a successful match.
+  nix log "$attr" | sed $'s/\x1b\\[[0-9;]*[A-Za-z]//g' | grep -E "$pattern" > /dev/null \
     || { echo "$attr built green but ran no tests ($pattern not in its log)" >&2; exit 1; }
 }
 while IFS= read -r name; do

nix/checks.nix

@@ -7,6 +7,10 @@ let
   abgenRoot = ''export ABGEN_ROOT="$PWD"'';
 
   archIndependent = {
+    # The wasm toolchain + registry closure, exposed so the pipeline's deps
+    # stage can build and cache it before any source-keyed work starts.
+    wasm-deps = wasmCheck.cargoArtifacts;
+
     fmt = craneLib.cargoFmt {
       inherit (commonArgs) pname version src;
     };
@@ -25,6 +29,11 @@ let
   };
 
   archDependent = {
+    # The crane dependency closure as a first-class attr: the pipeline's
+    # deps stage builds exactly this, then publishes the binary cache, so a
+    # failure in any later stage never costs the next run its warm deps.
+    deps = cargoArtifacts;
+
     nextest = craneLib.cargoNextest (withArtifacts // {
       doCheck = true;
       __darwinAllowLocalNetworking = true;
@@ -33,15 +42,6 @@ let
       preCheck = abgenRoot;
     });
 
-    lambda-tests = craneLib.cargoTest (withArtifacts // {
-      doCheck = true;
-      __darwinAllowLocalNetworking = true;
-      pname = "abgen-lambda-tests";
-      cargoExtraArgs = "--locked";
-      cargoTestExtraArgs = "-p abgen-lambda -p abgen-native --tests";
-      preCheck = abgenRoot;
-    });
-
     native-smoke =
       let
         libName = "libabgen${pkgs.stdenv.hostPlatform.extensions.sharedLibrary}";
@@ -58,5 +58,26 @@ let
         touch $out
       '';
   };
+
+  # server-off config. The lambda binary ships and runs on aarch64 only, so
+  # its test lane follows the prod arch; the config stays covered fleet-wide
+  # (aarch64-linux in CI, darwin locally) without a second server-off
+  # workspace compile+test on the slower x86 lane. Folding it into nextest
+  # is rejected: feature unification differs, a merged compile would not
+  # test the no-server config the mac/windows legs ship.
+  lambdaTests = {
+    lambda-tests = craneLib.cargoTest (withArtifacts // {
+      doCheck = true;
+      __darwinAllowLocalNetworking = true;
+      pname = "abgen-lambda-tests";
+      cargoExtraArgs = "--locked";
+      cargoTestExtraArgs = "-p abgen-lambda -p abgen-native --tests";
+      preCheck = abgenRoot;
+    });
+  };
 in
-archDependent // lib.optionalAttrs (system == "x86_64-linux") archIndependent
+# Arch-independent checks ride the aarch64 lane: those runners are ~1.8x
+# faster per stage, which rebalances the two pipelines' critical paths.
+archDependent
+// lib.optionalAttrs (system != "x86_64-linux") lambdaTests
+// lib.optionalAttrs (system == "aarch64-linux") archIndependent