feat(lambda): shared Redis hit-cache in front of S3 existence probes #317
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| schedule: | |
| - cron: "23 4 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_PROFILE_TEST_DEBUG: "0" | |
| RUSTFLAGS: -D warnings | |
| jobs: | |
| sanitizers: | |
| name: address sanitizer | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| env: | |
| RUSTUP_TOOLCHAIN: nightly | |
| RUSTFLAGS: -Zsanitizer=address | |
| CFLAGS: -fsanitize=address -fno-omit-frame-pointer | |
| CXXFLAGS: -fsanitize=address -fno-omit-frame-pointer | |
| ASAN_OPTIONS: detect_leaks=0:abort_on_error=1 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| - uses: ./.github/actions/rust-setup | |
| with: | |
| toolchain: nightly | |
| components: rust-src | |
| - name: cargo test under ASan | |
| run: | | |
| ABGEN_ROOT="$GITHUB_WORKSPACE" cargo test --locked --workspace --lib \ | |
| --target x86_64-unknown-linux-gnu -- --test-threads=1 | |
| windows-gate: | |
| name: windows gate | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| mode: ${{ steps.decide.outputs.mode }} | |
| hash: ${{ steps.hash.outputs.h }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| with: | |
| fetch-depth: 0 | |
| - id: hash | |
| run: echo "h=${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml', '.github/workflows/ci.yml', '.github/actions/rust-setup/**', 'ci/abgen-smoke.ps1', '.config/nextest.toml', 'crate/Cargo.toml', 'crate/build.rs', 'crate/third_party/**', 'crate/abgen-native/**', 'crate/dcl-contents/**', 'crate/src/abcdn/**', 'crate/src/builder/**', 'crate/src/export/**', 'crate/src/lodgen/**', 'crate/src/gpu/**', 'crate/src/live.rs', 'crate/src/jitcache.rs', 'crate/src/local_store.rs', 'crate/src/texencode_cache.rs', 'crate/src/tmppath.rs', 'crate/src/clihelp.rs', 'crate/src/ffi.rs', 'crate/src/worlds.rs', 'crate/src/lods.rs', 'crate/src/lz4.rs', 'crate/src/manifest.rs', 'crate/src/space.rs', 'crate/src/shader.rs', 'crate/src/lib.rs', 'crate/src/gltf/scene_build.rs', 'crate/src/unity/bundle_file.rs', 'crate/abgen-wasm/test/fixtures/normal-quad.glb') }}" >> "$GITHUB_OUTPUT" | |
| - id: memo | |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| with: | |
| path: .win-green-marker | |
| key: windows-green-${{ steps.hash.outputs.h }} | |
| lookup-only: true | |
| - id: decide | |
| run: | | |
| if [ "${{ steps.memo.outputs.cache-hit }}" = "true" ]; then mode=skip | |
| elif [ "${{ github.event_name }}" != "pull_request" ]; then mode=full | |
| elif ! git diff --quiet ${{ github.event.pull_request.base.sha }}...HEAD -- 'Cargo.lock' 'Cargo.toml' 'rust-toolchain.toml' '.github/workflows/ci.yml' '.github/actions/rust-setup' 'ci/abgen-smoke.ps1' '.config/nextest.toml' 'crate/Cargo.toml' 'crate/build.rs' 'crate/third_party' 'crate/abgen-native' 'crate/dcl-contents' 'crate/src/abcdn' 'crate/src/builder' 'crate/src/export' 'crate/src/lodgen' 'crate/src/gpu' 'crate/src/live.rs' 'crate/src/jitcache.rs' 'crate/src/local_store.rs' 'crate/src/texencode_cache.rs' 'crate/src/tmppath.rs' 'crate/src/clihelp.rs' 'crate/src/ffi.rs' 'crate/src/worlds.rs' 'crate/src/lods.rs' 'crate/src/lz4.rs' 'crate/src/manifest.rs' 'crate/src/space.rs' 'crate/src/shader.rs' 'crate/src/lib.rs' 'crate/src/gltf/scene_build.rs' 'crate/src/unity/bundle_file.rs' 'crate/abgen-wasm/test/fixtures/normal-quad.glb'; then mode=full | |
| elif ! git diff --quiet ${{ github.event.pull_request.base.sha }}...HEAD -- '**/*.rs' 'Cargo.*' '**/Cargo.toml'; then mode=quick | |
| else mode=quick; fi | |
| echo "mode=$mode" >> "$GITHUB_OUTPUT" | |
| windows: | |
| name: windows tests + smoke | |
| needs: windows-gate | |
| runs-on: windows-2025 | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| - uses: ./.github/actions/rust-setup | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| - name: install cargo-nextest | |
| if: needs.windows-gate.outputs.mode != 'skip' | |
| shell: pwsh | |
| run: curl.exe -LsSf https://get.nexte.st/0.9.143/windows-tar | tar -xzf - -C $env:USERPROFILE\.cargo\bin | |
| - name: skip notice | |
| if: needs.windows-gate.outputs.mode == 'skip' | |
| run: echo "windows suite already green for input hash ${{ needs.windows-gate.outputs.hash }} (memoized marker); skipping." | |
| - name: quick tests + smoke | |
| if: needs.windows-gate.outputs.mode == 'quick' | |
| shell: pwsh | |
| run: | | |
| $env:ABGEN_ROOT = $env:GITHUB_WORKSPACE | |
| cargo nextest run --locked -P windows-quick -p abgen -p dcl-contents -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| cargo build -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| pwsh -NoProfile -ExecutionPolicy Bypass -File ci/abgen-smoke.ps1 -Tree $env:GITHUB_WORKSPACE -Profile debug | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| - name: full tests + smoke | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| shell: pwsh | |
| run: | | |
| $env:ABGEN_ROOT = $env:GITHUB_WORKSPACE | |
| cargo nextest run --locked --workspace | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| cargo build --release -p abgen-native | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| pwsh -NoProfile -ExecutionPolicy Bypass -File ci/abgen-smoke.ps1 -Tree $env:GITHUB_WORKSPACE | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| - name: write green marker | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| shell: pwsh | |
| run: Set-Content -Path .win-green-marker -Value green | |
| # Cache saves from PR branches are scoped to that branch; the | |
| # globally visible marker is minted by full runs on main pushes. | |
| - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 | |
| if: needs.windows-gate.outputs.mode == 'full' | |
| with: | |
| path: .win-green-marker | |
| key: windows-green-${{ needs.windows-gate.outputs.hash }} | |
| # Everything races in parallel off the shared binary cache: each lane | |
| # restores, builds its deps closures, publishes the cache immediately | |
| # (no-op unless the key is new), then runs only its slice of the checks. | |
| # Image artifacts are produced by the release workflow's build legs, | |
| # which start on the same push in parallel with all of this; publishing | |
| # is what gates on green, not building. | |
| nix-ci: | |
| name: ${{ matrix.lane }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - lane: x86 tests | |
| runner: ubuntu-24.04 | |
| system: x86_64-linux | |
| deps: .#checks.x86_64-linux.deps | |
| attrs: nextest native-smoke | |
| - lane: arm tests | |
| runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| deps: .#checks.aarch64-linux.deps | |
| attrs: nextest lambda-tests native-smoke | |
| - lane: arm lints | |
| runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| deps: .#checks.aarch64-linux.deps .#checks.aarch64-linux.wasm-deps | |
| attrs: clippy fmt no-node-spawn wasm-check | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| # No standalone nix installer: the composite runs it with the | |
| # extra-conf this job needs; a prior bare install made that conf | |
| # silently a no-op. | |
| - uses: ./.github/actions/nix-store-cache | |
| - name: deps (dependency closures) | |
| run: nix build --no-link ${{ matrix.deps }} | |
| # Publish immediately and from any branch: the composite no-ops | |
| # unless the exact key is new, so this fires once per input rotation | |
| # (lanes sharing a key race; first save wins, the rest skip). | |
| - uses: ./.github/actions/nix-store-cache/save | |
| continue-on-error: true | |
| - name: checks (${{ matrix.attrs }}) | |
| run: bash ci/nix-checks.sh ${{ matrix.system }} ${{ matrix.attrs }} | |
| node-addon: | |
| name: node addon | |
| if: github.event_name != 'push' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 | |
| - uses: ./.github/actions/rust-setup | |
| with: | |
| # Pure consumer of the cache the release napi linux legs save from | |
| # main pushes (same shared-key); PR-scoped saves would be quota | |
| # waste. | |
| workspaces: crate/abgen-node | |
| shared-key: abgen-node | |
| save-if: "false" | |
| - name: abgen-node (build + smoke) | |
| working-directory: crate/abgen-node | |
| run: | | |
| npm ci --no-audit --no-fund | |
| npx napi build --platform --release -- --locked | |
| git diff --exit-code -- index.js index.d.ts \ | |
| || { echo "index.js/index.d.ts are stale; run 'napi build --platform --release' and commit them" >&2; exit 1; } | |
| node test/smoke.mjs | |
| # The addon is dlopen'd into node and so is bound by the host's | |
| # glibc exactly like libabgen.so is by Unity's. Paths are relative to | |
| # crate/abgen-node now that the step runs there. | |
| bash ../../ci/check-glibc-floor.sh 2.34 \ | |
| target/release/libabgen_node.so |