ci: checkfast test profile (4.9x compile cut) + green-marker memoization #339
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| schedule: | |
| - cron: "23 4 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| RUSTFLAGS: -D warnings | |
| jobs: | |
| sanitizers: | |
| name: address sanitizer | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| env: | |
| RUSTUP_TOOLCHAIN: nightly | |
| RUSTFLAGS: -Zsanitizer=address | |
| CFLAGS: -fsanitize=address -fno-omit-frame-pointer | |
| CXXFLAGS: -fsanitize=address -fno-omit-frame-pointer | |
| ASAN_OPTIONS: detect_leaks=0:abort_on_error=1 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| - uses: ./.github/actions/rust-setup | |
| with: | |
| toolchain: nightly | |
| components: rust-src | |
| - name: cargo test under ASan | |
| run: | | |
| ABGEN_ROOT="$GITHUB_WORKSPACE" cargo test --locked --workspace --lib \ | |
| --target x86_64-unknown-linux-gnu -- --test-threads=1 | |
| windows-gate: | |
| name: windows gate | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| mode: ${{ steps.decide.outputs.mode }} | |
| hash: ${{ steps.hash.outputs.h }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| with: | |
| fetch-depth: 0 | |
| - id: hash | |
| run: echo "h=${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml', '.github/workflows/ci.yml', '.github/actions/rust-setup/**', 'ci/abgen-smoke.ps1', '.config/nextest.toml', 'crate/Cargo.toml', 'crate/build.rs', 'crate/third_party/**', 'crate/abgen-native/**', 'crate/dcl-contents/**', 'crate/src/abcdn/**', 'crate/src/builder/**', 'crate/src/export/**', 'crate/src/lodgen/**', 'crate/src/gpu/**', 'crate/src/live.rs', 'crate/src/jitcache.rs', 'crate/src/local_store.rs', 'crate/src/texencode_cache.rs', 'crate/src/tmppath.rs', 'crate/src/clihelp.rs', 'crate/src/ffi.rs', 'crate/src/worlds.rs', 'crate/src/lods.rs', 'crate/src/lz4.rs', 'crate/src/manifest.rs', 'crate/src/space.rs', 'crate/src/shader.rs', 'crate/src/lib.rs', 'crate/src/gltf/scene_build.rs', 'crate/src/unity/bundle_file.rs', 'crate/abgen-wasm/test/fixtures/normal-quad.glb') }}" >> "$GITHUB_OUTPUT" | |
| - id: memo | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| with: | |
| path: .win-green-marker | |
| key: windows-green-${{ steps.hash.outputs.h }} | |
| lookup-only: true | |
| # The save runs on windows-2025 while this lookup runs on linux; | |
| # without the cross-OS flag on BOTH sides the cache versions never | |
| # match and the marker is permanently invisible. | |
| enableCrossOsArchive: true | |
| - id: decide | |
| run: | | |
| if [ "${{ steps.memo.outputs.cache-hit }}" = "true" ]; then mode=skip | |
| elif [ "${{ github.event_name }}" != "pull_request" ]; then mode=full | |
| elif ! git diff --quiet ${{ github.event.pull_request.base.sha }}...HEAD -- 'Cargo.lock' 'Cargo.toml' 'rust-toolchain.toml' '.github/workflows/ci.yml' '.github/actions/rust-setup' 'ci/abgen-smoke.ps1' '.config/nextest.toml' 'crate/Cargo.toml' 'crate/build.rs' 'crate/third_party' 'crate/abgen-native' 'crate/dcl-contents' 'crate/src/abcdn' 'crate/src/builder' 'crate/src/export' 'crate/src/lodgen' 'crate/src/gpu' 'crate/src/live.rs' 'crate/src/jitcache.rs' 'crate/src/local_store.rs' 'crate/src/texencode_cache.rs' 'crate/src/tmppath.rs' 'crate/src/clihelp.rs' 'crate/src/ffi.rs' 'crate/src/worlds.rs' 'crate/src/lods.rs' 'crate/src/lz4.rs' 'crate/src/manifest.rs' 'crate/src/space.rs' 'crate/src/shader.rs' 'crate/src/lib.rs' 'crate/src/gltf/scene_build.rs' 'crate/src/unity/bundle_file.rs' 'crate/abgen-wasm/test/fixtures/normal-quad.glb'; then mode=full | |
| elif ! git diff --quiet ${{ github.event.pull_request.base.sha }}...HEAD -- '**/*.rs' 'Cargo.*' '**/Cargo.toml'; then mode=quick | |
| else mode=quick; fi | |
| echo "mode=$mode" >> "$GITHUB_OUTPUT" | |
| windows: | |
| name: windows tests + smoke | |
| needs: windows-gate | |
| runs-on: windows-2025 | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| - uses: ./.github/actions/rust-setup | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| - name: install cargo-nextest | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| shell: pwsh | |
| run: curl.exe -LsSf https://get.nexte.st/0.9.143/windows-tar | tar -xzf - -C $env:USERPROFILE\.cargo\bin | |
| - name: skip notice | |
| if: needs.windows-gate.outputs.mode == 'skip' | |
| run: echo "windows suite already green for input hash ${{ needs.windows-gate.outputs.hash }} (memoized marker); skipping." | |
| - name: quick tests + smoke | |
| if: needs.windows-gate.outputs.mode == 'quick' | |
| shell: pwsh | |
| run: | | |
| $env:ABGEN_ROOT = $env:GITHUB_WORKSPACE | |
| cargo nextest run --locked -P windows-quick -p abgen -p dcl-contents -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| cargo build -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| pwsh -NoProfile -ExecutionPolicy Bypass -File ci/abgen-smoke.ps1 -Tree $env:GITHUB_WORKSPACE -Profile debug | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| - name: full tests + smoke | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| shell: pwsh | |
| run: | | |
| $env:ABGEN_ROOT = $env:GITHUB_WORKSPACE | |
| cargo nextest run --locked --workspace | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| cargo build --release -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| pwsh -NoProfile -ExecutionPolicy Bypass -File ci/abgen-smoke.ps1 -Tree $env:GITHUB_WORKSPACE | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| - name: write green marker | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| shell: pwsh | |
| run: Set-Content -Path .win-green-marker -Value green | |
| # Cache saves from PR branches are scoped to that branch; the | |
| # globally visible marker is minted by full runs on main pushes. | |
| - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| with: | |
| path: .win-green-marker | |
| key: windows-green-${{ needs.windows-gate.outputs.hash }} | |
| enableCrossOsArchive: true | |
| # Everything races in parallel off the shared binary cache: each lane | |
| # restores, builds its deps closures, publishes the cache immediately | |
| # (no-op unless the key is new), then runs only its slice of the checks. | |
| # Image artifacts are produced by the release workflow's build legs, | |
| # which start on the same push in parallel with all of this; publishing | |
| # is what gates on green, not building. | |
| # Windows-gate pattern for the nix lanes: a PR pays only for the lanes | |
| # its diff can affect. Pushes to main and schedules always run everything. | |
| # crate -> workspace tests/build (crate/, template/, roots, nix build wiring) | |
| # lambda -> the lambda test config (also affected by crate/) | |
| # lint -> anything lintable at all (rust/nix/toml sources) | |
| nix-gate: | |
| name: nix gate | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| run_crate: ${{ steps.decide.outputs.crate }} | |
| run_lambda: ${{ steps.decide.outputs.lambda }} | |
| run_lint: ${{ steps.decide.outputs.lint }} | |
| # Hash of every input the nix lanes actually consume. Pushes that | |
| # leave it unchanged (hash-manifest records, workflow edits, docs) | |
| # skip lanes that are already green for it, windows-marker style. | |
| inputs_hash: ${{ steps.hash.outputs.h }} | |
| skip_x86-nextest: ${{ steps.memo.outputs.m_x86-nextest }} | |
| skip_x86-smoke: ${{ steps.memo.outputs.m_x86-smoke }} | |
| skip_arm-nextest: ${{ steps.memo.outputs.m_arm-nextest }} | |
| skip_arm-lambda: ${{ steps.memo.outputs.m_arm-lambda }} | |
| skip_arm-lints: ${{ steps.memo.outputs.m_arm-lints }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| with: | |
| fetch-depth: 0 | |
| - id: hash | |
| run: echo "h=${{ hashFiles('crate/**', 'lambda/**', 'template/**', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'flake.nix', 'flake.lock', 'nix/**', 'ci/nix-checks.sh', '.config/nextest.toml') }}" >> "$GITHUB_OUTPUT" | |
| - id: memo | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| H: ${{ steps.hash.outputs.h }} | |
| run: | | |
| keys="$(gh api "/repos/$GITHUB_REPOSITORY/actions/caches?key=nix-green-&per_page=100" \ | |
| --jq '.actions_caches[].key' 2>/dev/null || true)" | |
| for lane in x86-nextest x86-smoke arm-nextest arm-lambda arm-lints; do | |
| if printf '%s\n' "$keys" | grep -qx "nix-green-$lane-$H"; then | |
| echo "m_$lane=true" >> "$GITHUB_OUTPUT" | |
| echo "$lane: green marker for $H — skip" | |
| else | |
| echo "m_$lane=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| done | |
| - id: decide | |
| run: | | |
| if [ "${{ github.event_name }}" != "pull_request" ]; then | |
| printf 'crate=true\nlambda=true\nlint=true\n' >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| base="${{ github.event.pull_request.base.sha }}" | |
| changed() { ! git diff --quiet "$base"...HEAD -- "$@"; } | |
| crate=false; lambda=false; lint=false | |
| changed 'crate' 'template' 'Cargo.lock' 'Cargo.toml' \ | |
| 'rust-toolchain.toml' 'nix/build.nix' 'flake.nix' 'flake.lock' \ | |
| '.config/nextest.toml' && crate=true | |
| changed 'lambda' && lambda=true | |
| # crate changes reach the lambda tests too (shared workspace). | |
| [ "$crate" = "true" ] && lambda=true | |
| changed '**/*.rs' '**/*.toml' 'nix' 'flake.nix' 'flake.lock' \ | |
| 'ci/nix-checks.sh' && lint=true | |
| printf 'crate=%s\nlambda=%s\nlint=%s\n' "$crate" "$lambda" "$lint" >> "$GITHUB_OUTPUT" | |
| echo "crate=$crate lambda=$lambda lint=$lint" | |
| nix-ci: | |
| name: ${{ matrix.lane }} | |
| needs: nix-gate | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # One workspace-scale compile per lane: nextest's test-target | |
| # build and abgen-all (behind native-smoke) each cost ~10 min on | |
| # 4 cores, so pairing them serialized the verdict. Wall time is | |
| # the slowest single lane. `gate` names the nix-gate output that | |
| # decides whether the lane runs for this diff. | |
| - lane: x86 nextest | |
| runner: ubuntu-24.04 | |
| system: x86_64-linux | |
| deps: .#checks.x86_64-linux.deps-checkfast | |
| attrs: nextest | |
| gate: run_crate | |
| marker: x86-nextest | |
| flavor: checkfast | |
| - lane: x86 build + smoke | |
| runner: ubuntu-24.04 | |
| system: x86_64-linux | |
| deps: .#checks.x86_64-linux.deps | |
| attrs: native-smoke | |
| gate: run_crate | |
| marker: x86-smoke | |
| - lane: arm nextest | |
| runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| deps: .#checks.aarch64-linux.deps-checkfast | |
| attrs: nextest | |
| gate: run_crate | |
| marker: arm-nextest | |
| flavor: checkfast | |
| - lane: arm lambda + smoke | |
| runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| deps: .#checks.aarch64-linux.deps-checkfast | |
| attrs: lambda-tests native-smoke | |
| gate: run_lambda | |
| marker: arm-lambda | |
| flavor: checkfast | |
| - lane: arm lints | |
| runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| deps: .#checks.aarch64-linux.deps .#checks.aarch64-linux.wasm-deps | |
| attrs: clippy fmt no-node-spawn wasm-check | |
| gate: run_lint | |
| marker: arm-lints | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 45 | |
| steps: | |
| - name: skip notice | |
| if: needs.nix-gate.outputs[matrix.gate] != 'true' || needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] == 'true' | |
| run: echo "skipping (gate=${{ needs.nix-gate.outputs[matrix.gate] }}, green-marker=${{ needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] }})" | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| # No standalone nix installer: the composite runs it with the | |
| # extra-conf this job needs; a prior bare install made that conf | |
| # silently a no-op. | |
| - uses: ./.github/actions/nix-store-cache | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| with: | |
| flavor: ${{ matrix.flavor || 'build' }} | |
| - name: deps (dependency closures) | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| run: nix build --no-link ${{ matrix.deps }} | |
| # Publish immediately and from any branch: the composite no-ops | |
| # unless the exact key is new, so this fires once per input rotation | |
| # (lanes sharing a key race; first save wins, the rest skip). | |
| - uses: ./.github/actions/nix-store-cache/save | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| continue-on-error: true | |
| - name: checks (${{ matrix.attrs }}) | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| run: bash ci/nix-checks.sh ${{ matrix.system }} ${{ matrix.attrs }} | |
| - name: write green marker | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| run: echo green > .nix-green-marker | |
| # Identical inputs => identical verdict (the checks are hermetic), | |
| # so a marker minted anywhere skips the lane everywhere. Lookup | |
| # goes through the caches API, which sees every scope. | |
| - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| if: needs.nix-gate.outputs[matrix.gate] == 'true' && needs.nix-gate.outputs[format('skip_{0}', matrix.marker)] != 'true' | |
| with: | |
| path: .nix-green-marker | |
| key: nix-green-${{ matrix.marker }}-${{ needs.nix-gate.outputs.inputs_hash }} | |
| node-addon: | |
| name: node addon | |
| if: github.event_name != 'push' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| - id: hash | |
| run: echo "h=${{ hashFiles('crate/**', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'ci/check-glibc-floor.sh', '.github/workflows/ci.yml', '.github/actions/rust-setup/**') }}" >> "$GITHUB_OUTPUT" | |
| - id: memo | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| with: | |
| path: .node-green-marker | |
| key: node-green-${{ steps.hash.outputs.h }} | |
| lookup-only: true | |
| - name: skip notice | |
| if: steps.memo.outputs.cache-hit == 'true' | |
| run: echo "node addon already green for input hash ${{ steps.hash.outputs.h }} (memoized marker); skipping." | |
| - uses: ./.github/actions/rust-setup | |
| if: steps.memo.outputs.cache-hit != 'true' | |
| with: | |
| # Pure consumer of the cache the release napi linux legs save from | |
| # main pushes (same shared-key); PR-scoped saves would be quota | |
| # waste. | |
| workspaces: crate/abgen-node | |
| shared-key: abgen-node | |
| save-if: "false" | |
| - name: abgen-node (build + smoke) | |
| if: steps.memo.outputs.cache-hit != 'true' | |
| working-directory: crate/abgen-node | |
| run: | | |
| npm ci --no-audit --no-fund | |
| npx napi build --platform --release -- --locked | |
| git diff --exit-code -- index.js index.d.ts \ | |
| || { echo "index.js/index.d.ts are stale; run 'napi build --platform --release' and commit them" >&2; exit 1; } | |
| node test/smoke.mjs | |
| # The addon is dlopen'd into node and so is bound by the host's | |
| # glibc exactly like libabgen.so is by Unity's. Paths are relative to | |
| # crate/abgen-node now that the step runs there. | |
| bash ../../ci/check-glibc-floor.sh 2.34 \ | |
| target/release/libabgen_node.so | |
| - name: write green marker | |
| if: steps.memo.outputs.cache-hit != 'true' | |
| run: echo green > .node-green-marker | |
| - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| if: steps.memo.outputs.cache-hit != 'true' | |
| with: | |
| path: .node-green-marker | |
| key: node-green-${{ steps.hash.outputs.h }} |