Benchmark Run 339 #339
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| run-name: Benchmark Run ${{ github.run_number }} | |
| on: | |
| schedule: | |
| # Daily at 21:00 UTC | |
| - cron: '0 21 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| gateway: | |
| description: 'Gateway to benchmark' | |
| required: false | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - apollo-gateway | |
| - apollo-router | |
| - cosmo | |
| - grafbase | |
| - hive-gateway | |
| - hive-gateway-router-runtime | |
| - hive-router | |
| - fusion | |
| - fusion-nightly | |
| - fusion-nightly-net11 | |
| - fusion-nightly-fed | |
| - feddi | |
| benchmark: | |
| description: 'Benchmark mode' | |
| required: false | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - constant | |
| - latency | |
| - burst | |
| runs: | |
| description: 'Number of benchmark runs per gateway' | |
| required: false | |
| default: '9' | |
| duration: | |
| description: 'Duration per run in seconds' | |
| required: false | |
| default: '120' | |
| diagnostic_run: | |
| description: 'Diagnostic run (collect diagnostics artifacts, skip result documents)' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: gateway-benchmarks | |
| cancel-in-progress: false | |
| permissions: | |
| actions: read | |
| contents: write | |
| jobs: | |
| prepare-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| gateways: ${{ steps.build.outputs.gateways }} | |
| gateways_net: ${{ steps.build.outputs.gateways_net }} | |
| gateways_constant: ${{ steps.build.outputs.gateways_constant }} | |
| not_run_constant: ${{ steps.build.outputs.not_run_constant }} | |
| gateways_burst: ${{ steps.build.outputs.gateways_burst }} | |
| not_run_burst: ${{ steps.build.outputs.not_run_burst }} | |
| gateway_builds: ${{ steps.build.outputs.gateway_builds }} | |
| subgraph_builds: ${{ steps.build.outputs.subgraph_builds }} | |
| is_diagnostic: ${{ steps.flags.outputs.is_diagnostic }} | |
| should_publish: ${{ steps.flags.outputs.should_publish }} | |
| run_constant: ${{ steps.flags.outputs.run_constant }} | |
| run_burst: ${{ steps.flags.outputs.run_burst }} | |
| run_latency: ${{ steps.flags.outputs.run_latency }} | |
| steps: | |
| - name: Compute run flags | |
| id: flags | |
| run: | | |
| BENCHMARK="${{ inputs.benchmark || 'all' }}" | |
| GATEWAY="${{ inputs.gateway || 'all' }}" | |
| DIAGNOSTIC="${{ inputs.diagnostic_run }}" | |
| IS_DIAGNOSTIC="false" | |
| if [[ "$DIAGNOSTIC" == "true" ]]; then | |
| IS_DIAGNOSTIC="true" | |
| fi | |
| SHOULD_PUBLISH="false" | |
| if [[ "$GATEWAY" == "all" && "$BENCHMARK" == "all" && "$IS_DIAGNOSTIC" == "false" ]]; then | |
| SHOULD_PUBLISH="true" | |
| fi | |
| RUN_CONSTANT="false" | |
| if [[ "$BENCHMARK" == "all" || "$BENCHMARK" == "constant" ]]; then | |
| RUN_CONSTANT="true" | |
| fi | |
| RUN_BURST="false" | |
| if [[ "$BENCHMARK" == "all" || "$BENCHMARK" == "burst" ]]; then | |
| RUN_BURST="true" | |
| fi | |
| RUN_LATENCY="false" | |
| if [[ "$BENCHMARK" == "all" || "$BENCHMARK" == "latency" ]]; then | |
| RUN_LATENCY="true" | |
| fi | |
| { | |
| echo "is_diagnostic=$IS_DIAGNOSTIC" | |
| echo "should_publish=$SHOULD_PUBLISH" | |
| echo "run_constant=$RUN_CONSTANT" | |
| echo "run_burst=$RUN_BURST" | |
| echo "run_latency=$RUN_LATENCY" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build gateway matrix | |
| id: build | |
| env: | |
| SELECTED_GATEWAY: ${{ inputs.gateway || 'all' }} | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| import sys | |
| import urllib.request | |
| def _parse_core(core): | |
| nums = [] | |
| for part in core.split("."): | |
| if not part.isdigit(): | |
| return None | |
| nums.append(int(part)) | |
| return tuple(nums) | |
| def _prerelease_key(pre): | |
| # SemVer 2.0: numeric identifiers rank below alphanumeric and compare numerically. | |
| key = [] | |
| for ident in pre.split("."): | |
| if ident.isdigit(): | |
| key.append((0, int(ident), "")) | |
| else: | |
| key.append((1, 0, ident)) | |
| return tuple(key) | |
| def resolve_hotchocolate_channels(versions): | |
| # Return (preview_ahead, latest_stable, latest_preview). preview_ahead is | |
| # True only when the newest prerelease is on a higher core (X.Y.Z) line than | |
| # the newest stable release (an in-flight preview ahead of what has shipped). | |
| stable = [] | |
| preview = [] | |
| for raw in versions: | |
| core, sep, pre = raw.partition("-") | |
| core_tuple = _parse_core(core) | |
| if core_tuple is None: | |
| continue | |
| if sep: | |
| preview.append((core_tuple, _prerelease_key(pre), raw)) | |
| else: | |
| stable.append((core_tuple, raw)) | |
| latest_stable = max(stable, key=lambda x: x[0]) if stable else None | |
| latest_preview = max(preview, key=lambda x: (x[0], x[1])) if preview else None | |
| stable_str = latest_stable[1] if latest_stable else None | |
| preview_str = latest_preview[2] if latest_preview else None | |
| if latest_preview is None: | |
| ahead = False | |
| elif latest_stable is None: | |
| ahead = True | |
| else: | |
| ahead = latest_preview[0] > latest_stable[0] | |
| return ahead, stable_str, preview_str | |
| def hotchocolate_preview_is_ahead(): | |
| # Query NuGet for the HotChocolate version feed. On any failure return | |
| # (False, None, None) so fusion-nightly is skipped unless explicitly asked for. | |
| url = "https://api.nuget.org/v3-flatcontainer/hotchocolate.aspnetcore/index.json" | |
| try: | |
| with urllib.request.urlopen(url, timeout=30) as resp: | |
| versions = json.load(resp).get("versions", []) | |
| return resolve_hotchocolate_channels(versions) | |
| except Exception as exc: # noqa: BLE001 | |
| print( | |
| f"Warning: could not query NuGet for HotChocolate preview status ({exc}); " | |
| "fusion-nightly will be skipped unless explicitly selected." | |
| ) | |
| return False, None, None | |
| selected = os.environ.get("SELECTED_GATEWAY", "all") | |
| aliases = { | |
| "hc": "fusion", | |
| "hc-rust": "fusion-rust", | |
| "hc-net": "fusion-net", | |
| "hotchocolate": "fusion", | |
| } | |
| selected = aliases.get(selected, selected) | |
| selected_gateway = selected | |
| selected_subgraph_tech = None | |
| if selected.endswith("-rust"): | |
| selected_gateway = selected[:-5] | |
| selected_subgraph_tech = "rust" | |
| elif selected.endswith("-net"): | |
| selected_gateway = selected[:-4] | |
| selected_subgraph_tech = ".net" | |
| base_gateways = [ | |
| { | |
| "path": "apollo-federation/gateways/apollo-gateway", | |
| "name": "apollo-gateway", | |
| "gateway_artifact": "prebuilt-gateway-apollo-gateway", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/apollo-router", | |
| "name": "apollo-router", | |
| "gateway_artifact": "prebuilt-gateway-apollo-router", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/cosmo", | |
| "name": "cosmo", | |
| "gateway_artifact": "prebuilt-gateway-cosmo", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/grafbase", | |
| "name": "grafbase", | |
| "gateway_artifact": "prebuilt-gateway-grafbase", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/hive-gateway", | |
| "name": "hive-gateway", | |
| "gateway_artifact": "prebuilt-gateway-hive-gateway", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/hive-gateway-router-runtime", | |
| "name": "hive-gateway-router-runtime", | |
| "gateway_artifact": "prebuilt-gateway-hive-gateway-router-runtime", | |
| }, | |
| { | |
| "path": "apollo-federation/gateways/hive-router", | |
| "name": "hive-router", | |
| "gateway_artifact": "prebuilt-gateway-hive-router", | |
| }, | |
| { | |
| "path": "composite-schema/gateways/fusion", | |
| "name": "fusion", | |
| "gateway_artifact": "prebuilt-gateway-fusion", | |
| "channel": "stable", | |
| "dotnet": "10", | |
| }, | |
| { | |
| "path": "composite-schema/gateways/fusion", | |
| "name": "fusion-nightly", | |
| "gateway_artifact": "prebuilt-gateway-fusion-nightly", | |
| "channel": "preview", | |
| "dotnet": "10", | |
| }, | |
| { | |
| "path": "composite-schema/gateways/fusion", | |
| "name": "fusion-nightly-net11", | |
| "gateway_artifact": "prebuilt-gateway-fusion-nightly-net11", | |
| "channel": "preview", | |
| "dotnet": "11", | |
| }, | |
| { | |
| # Fusion (preview channel) composing the Apollo Federation subgraphs | |
| # via Apollo Federation connectors. | |
| "path": "apollo-federation/gateways/fusion-nightly-fed", | |
| "name": "fusion-nightly-fed", | |
| "gateway_artifact": "prebuilt-gateway-fusion-nightly-fed", | |
| "channel": "preview", | |
| "dotnet": "10", | |
| }, | |
| { | |
| "path": "composite-schema/gateways/feddi", | |
| "name": "feddi", | |
| "gateway_artifact": "prebuilt-gateway-feddi", | |
| }, | |
| ] | |
| subgraph_variants_by_category = { | |
| "apollo-federation": [ | |
| { | |
| "dir": "subgraphs-rust", | |
| "tech": "rust", | |
| "artifact": "prebuilt-apollo-federation-subgraphs-rust", | |
| "suffix": "rust", | |
| }, | |
| { | |
| "dir": "subgraphs-net", | |
| "tech": ".net", | |
| "artifact": "prebuilt-apollo-federation-subgraphs-net", | |
| "suffix": "net", | |
| }, | |
| ], | |
| "composite-schema": [ | |
| { | |
| "dir": "subgraphs-rust", | |
| "tech": "rust", | |
| "artifact": "prebuilt-composite-schema-subgraphs-rust", | |
| "suffix": "rust", | |
| }, | |
| { | |
| "dir": "subgraphs-net", | |
| "tech": ".net", | |
| "artifact": "prebuilt-composite-schema-subgraphs-net", | |
| "suffix": "net", | |
| }, | |
| ], | |
| } | |
| gateways = [] | |
| for gateway in base_gateways: | |
| category = gateway["path"].split("/", 1)[0] | |
| variants = subgraph_variants_by_category.get(category, []) | |
| for variant in variants: | |
| gateways.append({ | |
| "path": gateway["path"], | |
| "name": f"{gateway['name']}-{variant['suffix']}", | |
| "base_name": gateway["name"], | |
| "channel": gateway.get("channel", "stable"), | |
| "dotnet": gateway.get("dotnet", "10"), | |
| "subgraphs": variant["dir"], | |
| "subgraph_tech": variant["tech"], | |
| "display_name": f"{gateway['name']} ({variant['tech']} subgraphs)", | |
| "subgraphs_artifact": variant["artifact"], | |
| "gateway_artifact": gateway["gateway_artifact"], | |
| }) | |
| if selected == "all": | |
| chosen = gateways | |
| else: | |
| chosen = [entry for entry in gateways if entry["base_name"] == selected_gateway] | |
| if selected_subgraph_tech: | |
| chosen = [entry for entry in chosen if entry["subgraph_tech"] == selected_subgraph_tech] | |
| # Skip the fusion-nightly (preview) variants — both the .NET 10 and the | |
| # .NET 11 build — unless a HotChocolate prerelease newer than the latest | |
| # stable exists; otherwise "nightly" would benchmark a build older than (or | |
| # equal to) the stable run. An explicit fusion-nightly* selection is always | |
| # honored. | |
| nightly_base_names = ("fusion-nightly", "fusion-nightly-net11", "fusion-nightly-fed") | |
| nightly_in_chosen = any(e["base_name"] in nightly_base_names for e in chosen) | |
| nightly_explicit = selected_gateway in nightly_base_names | |
| if nightly_in_chosen and not nightly_explicit: | |
| preview_ahead, latest_stable_str, latest_preview_str = hotchocolate_preview_is_ahead() | |
| if preview_ahead: | |
| print( | |
| f"Including fusion-nightly variants: preview {latest_preview_str} is " | |
| f"ahead of stable {latest_stable_str}." | |
| ) | |
| else: | |
| chosen = [e for e in chosen if e["base_name"] not in nightly_base_names] | |
| print( | |
| f"Skipping fusion-nightly variants: newest prerelease " | |
| f"({latest_preview_str or 'none'}) is not ahead of latest stable " | |
| f"({latest_stable_str or 'none'})." | |
| ) | |
| if not chosen: | |
| print(f"Error: no gateway entries match selection {selected!r}", file=sys.stderr) | |
| sys.exit(1) | |
| matrix_json = json.dumps(chosen, separators=(",", ":")) | |
| chosen_net = [e for e in chosen if e["subgraph_tech"] == ".net"] | |
| matrix_net_json = json.dumps(chosen_net, separators=(",", ":")) | |
| # Per-mode exclusions: gateway/subgraph combinations that are intentionally | |
| # not benchmarked in a given mode. Excluded combinations are removed from the | |
| # run matrix but still surface in that mode's results table marked "not run" | |
| # (via NOT_RUN_GATEWAYS_JSON -> generate-results.py), so nothing is silently | |
| # dropped. | |
| # | |
| # feddi — crashes under the burst stress scenario (upstream | |
| # heavy-query planner defect); excluded from burst. | |
| def is_excluded(entry, mode): | |
| if entry["base_name"] == "feddi" and mode == "burst": | |
| return True | |
| return False | |
| def split_for_mode(entries, mode): | |
| run = [e for e in entries if not is_excluded(e, mode)] | |
| not_run = [e for e in entries if is_excluded(e, mode)] | |
| return run, not_run | |
| chosen_constant, not_run_constant = split_for_mode(chosen, "constant") | |
| matrix_constant_json = json.dumps(chosen_constant, separators=(",", ":")) | |
| not_run_constant_json = json.dumps(not_run_constant, separators=(",", ":")) | |
| chosen_burst, not_run_burst = split_for_mode(chosen, "burst") | |
| matrix_burst_json = json.dumps(chosen_burst, separators=(",", ":")) | |
| not_run_burst_json = json.dumps(not_run_burst, separators=(",", ":")) | |
| builds_by_artifact = {} | |
| for entry in chosen: | |
| artifact = entry["gateway_artifact"] | |
| if artifact in builds_by_artifact: | |
| continue | |
| builds_by_artifact[artifact] = { | |
| "path": entry["path"], | |
| "artifact": artifact, | |
| "channel": entry["channel"], | |
| "dotnet": entry["dotnet"], | |
| } | |
| build_matrix_json = json.dumps(list(builds_by_artifact.values()), separators=(",", ":")) | |
| subgraphs_by_artifact = {} | |
| for entry in chosen: | |
| artifact = entry["subgraphs_artifact"] | |
| if artifact in subgraphs_by_artifact: | |
| continue | |
| category = entry["path"].split("/", 1)[0] | |
| subgraphs_by_artifact[artifact] = { | |
| "path": f"{category}/{entry['subgraphs']}", | |
| "artifact": artifact, | |
| } | |
| subgraph_matrix_json = json.dumps(list(subgraphs_by_artifact.values()), separators=(",", ":")) | |
| with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh: | |
| fh.write(f"gateways={matrix_json}\n") | |
| fh.write(f"gateways_net={matrix_net_json}\n") | |
| fh.write(f"gateways_constant={matrix_constant_json}\n") | |
| fh.write(f"not_run_constant={not_run_constant_json}\n") | |
| fh.write(f"gateways_burst={matrix_burst_json}\n") | |
| fh.write(f"not_run_burst={not_run_burst_json}\n") | |
| fh.write(f"gateway_builds={build_matrix_json}\n") | |
| fh.write(f"subgraph_builds={subgraph_matrix_json}\n") | |
| print("Selected gateway entries:") | |
| for entry in chosen: | |
| print(f" - {entry['base_name']} [{entry['subgraph_tech']}]") | |
| print("Subgraph builds:") | |
| for sub in subgraphs_by_artifact.values(): | |
| print(f" - {sub['path']} ({sub['artifact']})") | |
| PY | |
| build-subgraphs: | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| subgraphs: ${{ fromJson(needs.prepare-matrix.outputs.subgraph_builds) }} | |
| name: build-${{ matrix.subgraphs.path }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup .NET SDK | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| global-json-file: global.json | |
| - name: Build subgraphs | |
| run: | | |
| chmod +x "${{ matrix.subgraphs.path }}/build.sh" | |
| (cd "${{ matrix.subgraphs.path }}" && bash build.sh) | |
| - name: Pack prebuilt subgraphs | |
| run: | | |
| tar -czf "prebuilt-subgraphs-${{ matrix.subgraphs.artifact }}.tar.gz" "${{ matrix.subgraphs.path }}" | |
| - name: Upload prebuilt subgraphs | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.subgraphs.artifact }} | |
| path: prebuilt-subgraphs-${{ matrix.subgraphs.artifact }}.tar.gz | |
| build-gateways: | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| gateway: ${{ fromJson(needs.prepare-matrix.outputs.gateway_builds) }} | |
| name: build-${{ matrix.gateway.artifact }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup .NET SDK | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| global-json-file: global.json | |
| - name: Build gateway | |
| env: | |
| BENCH_GATEWAY_CHANNEL: ${{ matrix.gateway.channel || 'stable' }} | |
| BENCH_GATEWAY_DOTNET: ${{ matrix.gateway.dotnet || '10' }} | |
| run: | | |
| chmod +x "${{ matrix.gateway.path }}/install.sh" | |
| (cd "${{ matrix.gateway.path }}" && bash install.sh) | |
| touch "${{ matrix.gateway.path }}/.prebuilt-gateway-ready" | |
| - name: Pack prebuilt gateway | |
| run: | | |
| tar -czf "prebuilt-gateway-${{ matrix.gateway.artifact }}.tar.gz" "${{ matrix.gateway.path }}" | |
| - name: Upload prebuilt gateway | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.gateway.artifact }} | |
| path: prebuilt-gateway-${{ matrix.gateway.artifact }}.tar.gz | |
| benchmark-constant: | |
| needs: | |
| - prepare-matrix | |
| - build-subgraphs | |
| - build-gateways | |
| - benchmark-constant-latency | |
| if: ${{ always() && !cancelled() && needs.prepare-matrix.outputs.run_constant == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # gateways_constant excludes combinations not run in constant — those are | |
| # marked "not run" in the constant results table instead (see prepare-matrix). | |
| gateway: ${{ fromJson(needs.prepare-matrix.outputs.gateways_constant) }} | |
| runs-on: | |
| group: benchmarking-1 | |
| name: ${{ matrix.gateway.name }} (constant) | |
| env: | |
| BENCH_RUNS: ${{ inputs.runs || '9' }} | |
| MEASURE_SECONDS: ${{ inputs.duration || '120' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download prebuilt gateway | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.gateway_artifact }} | |
| path: prebuilt-gateway/ | |
| - name: Restore prebuilt gateway | |
| run: | | |
| GATEWAY_ARCHIVE="$(find prebuilt-gateway -type f -name '*.tar.gz' | head -n1)" | |
| if [[ -z "$GATEWAY_ARCHIVE" ]]; then | |
| echo "Error: prebuilt gateway archive not found" | |
| find prebuilt-gateway -maxdepth 3 -print | |
| exit 1 | |
| fi | |
| tar -xzf "$GATEWAY_ARCHIVE" -C . | |
| if [[ -f apollo-federation/gateways/apollo-router/router ]]; then | |
| chmod +x apollo-federation/gateways/apollo-router/router | |
| fi | |
| if [[ -f apollo-federation/gateways/cosmo/cosmo ]]; then | |
| chmod +x apollo-federation/gateways/cosmo/cosmo | |
| fi | |
| if [[ -f apollo-federation/gateways/grafbase/grafbase-gateway ]]; then | |
| chmod +x apollo-federation/gateways/grafbase/grafbase-gateway | |
| fi | |
| if [[ -f apollo-federation/gateways/hive-router/hive_router ]]; then | |
| chmod +x apollo-federation/gateways/hive-router/hive_router | |
| fi | |
| - name: Download prebuilt subgraphs | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.subgraphs_artifact }} | |
| path: prebuilt-subgraphs/ | |
| - name: Restore prebuilt subgraphs | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GATEWAY_PATH: ${{ matrix.gateway.path }} | |
| SUBGRAPHS_DIR: ${{ matrix.gateway.subgraphs }} | |
| SUBGRAPHS_ARTIFACT: ${{ matrix.gateway.subgraphs_artifact }} | |
| run: | | |
| set -euo pipefail | |
| find_subgraphs_archive() { | |
| find prebuilt-subgraphs -type f -name '*.tar.gz' | head -n1 | |
| } | |
| redownload_subgraphs_artifact() { | |
| rm -rf prebuilt-subgraphs | |
| mkdir -p prebuilt-subgraphs | |
| gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --name "$SUBGRAPHS_ARTIFACT" --dir prebuilt-subgraphs | |
| } | |
| ATTEMPTS=3 | |
| for attempt in $(seq 1 "$ATTEMPTS"); do | |
| SUBGRAPHS_ARCHIVE="$(find_subgraphs_archive)" | |
| if [[ -z "$SUBGRAPHS_ARCHIVE" ]]; then | |
| echo "Subgraphs archive not found (attempt $attempt/$ATTEMPTS)" | |
| elif tar -tzf "$SUBGRAPHS_ARCHIVE" >/dev/null 2>&1; then | |
| tar -xzf "$SUBGRAPHS_ARCHIVE" -C . | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| exit 0 | |
| else | |
| echo "Subgraphs archive integrity check failed (attempt $attempt/$ATTEMPTS): $SUBGRAPHS_ARCHIVE" | |
| ls -lh "$SUBGRAPHS_ARCHIVE" || true | |
| fi | |
| if [[ "$attempt" -lt "$ATTEMPTS" ]]; then | |
| echo "Re-downloading artifact '$SUBGRAPHS_ARTIFACT' for retry..." | |
| if ! redownload_subgraphs_artifact; then | |
| echo "Artifact re-download failed (attempt $((attempt + 1))/$ATTEMPTS)." | |
| fi | |
| sleep $((attempt * 5)) | |
| fi | |
| done | |
| find prebuilt-subgraphs -maxdepth 3 -print || true | |
| SUBGRAPHS_ROOT="${GATEWAY_PATH%%/*}" | |
| SUBGRAPHS_PATH="${SUBGRAPHS_ROOT}/${SUBGRAPHS_DIR}" | |
| echo "Archive restore failed after $ATTEMPTS attempts; falling back to local rebuild: $SUBGRAPHS_PATH" | |
| if [[ ! -f "$SUBGRAPHS_PATH/build.sh" ]]; then | |
| echo "Error: fallback build script not found: $SUBGRAPHS_PATH/build.sh" | |
| exit 1 | |
| fi | |
| chmod +x "$SUBGRAPHS_PATH/build.sh" | |
| (cd "$SUBGRAPHS_PATH" && bash build.sh) | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| - name: Kill stale perfrunner processes | |
| run: | | |
| sudo pkill -u perfrunner || true | |
| sleep 1 | |
| sudo pkill -9 -u perfrunner || true | |
| - name: Prepare workspace for perfrunner | |
| run: | | |
| # Both users need write access during a run: | |
| # - perfrunner writes dependency artifacts (node_modules, target, etc.) | |
| # - github-runner writes benchmark outputs (results/, summaries) | |
| sudo chmod -R a+rX,g+rwX . | |
| sudo chown -R perfrunner:github-runner . | |
| # Preserve github-runner as group owner for newly created files/dirs. | |
| sudo find . -type d -exec chmod g+s {} + | |
| - name: Prepare diagnostics directory | |
| if: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| mkdir -p "${{ matrix.gateway.path }}/diagnostics" | |
| { | |
| echo "diagnostic_run=true" | |
| echo "mode=constant" | |
| echo "gateway=${{ matrix.gateway.name }}" | |
| echo "started_utc=${{ github.run_started_at || 'n/a' }}" | |
| echo "run_id=${{ github.run_id }}" | |
| } > "${{ matrix.gateway.path }}/diagnostics/run-info.txt" | |
| sudo chown -R perfrunner:github-runner "${{ matrix.gateway.path }}/diagnostics" | |
| sudo chmod -R u+rwX,g+rwX "${{ matrix.gateway.path }}/diagnostics" | |
| - name: Run benchmark | |
| run: ./k6/benchmark.sh ${{ matrix.gateway.path }} ${{ matrix.gateway.subgraphs }} constant | |
| env: | |
| BENCH_DISPLAY_NAME: ${{ matrix.gateway.display_name || '' }} | |
| BENCH_GATEWAY_NAME: ${{ matrix.gateway.base_name || '' }} | |
| BENCH_SUBGRAPH_TECH: ${{ matrix.gateway.subgraph_tech || '' }} | |
| BENCH_DIAGNOSTIC_RUN: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' && '1' || '0' }} | |
| BENCH_DIAGNOSTICS_DIR: ${{ github.workspace }}/${{ matrix.gateway.path }}/diagnostics | |
| USE_PREBUILT_SUBGRAPHS: "1" | |
| USE_PREBUILT_GATEWAY: "1" | |
| - name: Kill all perfrunner processes | |
| if: always() | |
| run: | | |
| echo "Killing all perfrunner processes..." | |
| sudo pkill -u perfrunner || true | |
| sleep 2 | |
| sudo pkill -9 -u perfrunner || true | |
| echo "Done." | |
| - name: Reclaim workspace ownership | |
| if: always() | |
| run: sudo chown -R $(id -u):$(id -g) . | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: results-${{ matrix.gateway.name }}-constant | |
| path: ${{ matrix.gateway.path }}/results/ | |
| - name: Inspect diagnostics directory | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| echo "Diagnostics directory: ${{ matrix.gateway.path }}/diagnostics" | |
| ls -la "${{ matrix.gateway.path }}/diagnostics" || true | |
| find "${{ matrix.gateway.path }}/diagnostics" -maxdepth 2 -type f -print | sort || true | |
| - name: Upload diagnostics | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diagnostics-${{ matrix.gateway.name }}-constant | |
| path: ${{ matrix.gateway.path }}/diagnostics/ | |
| if-no-files-found: ignore | |
| benchmark-burst: | |
| needs: | |
| - prepare-matrix | |
| - build-subgraphs | |
| - build-gateways | |
| - benchmark-constant | |
| if: ${{ always() && !cancelled() && needs.prepare-matrix.outputs.run_burst == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # gateways_burst excludes feddi — it is not run in the burst benchmark and is | |
| # instead marked "not run" in the burst results table (see prepare-matrix). | |
| gateway: ${{ fromJson(needs.prepare-matrix.outputs.gateways_burst) }} | |
| runs-on: | |
| group: benchmarking-1 | |
| name: ${{ matrix.gateway.name }} (burst) | |
| env: | |
| BENCH_RUNS: ${{ inputs.runs || '9' }} | |
| MEASURE_SECONDS: ${{ inputs.duration || '120' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download prebuilt gateway | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.gateway_artifact }} | |
| path: prebuilt-gateway/ | |
| - name: Restore prebuilt gateway | |
| run: | | |
| GATEWAY_ARCHIVE="$(find prebuilt-gateway -type f -name '*.tar.gz' | head -n1)" | |
| if [[ -z "$GATEWAY_ARCHIVE" ]]; then | |
| echo "Error: prebuilt gateway archive not found" | |
| find prebuilt-gateway -maxdepth 3 -print | |
| exit 1 | |
| fi | |
| tar -xzf "$GATEWAY_ARCHIVE" -C . | |
| if [[ -f apollo-federation/gateways/apollo-router/router ]]; then | |
| chmod +x apollo-federation/gateways/apollo-router/router | |
| fi | |
| if [[ -f apollo-federation/gateways/cosmo/cosmo ]]; then | |
| chmod +x apollo-federation/gateways/cosmo/cosmo | |
| fi | |
| if [[ -f apollo-federation/gateways/grafbase/grafbase-gateway ]]; then | |
| chmod +x apollo-federation/gateways/grafbase/grafbase-gateway | |
| fi | |
| if [[ -f apollo-federation/gateways/hive-router/hive_router ]]; then | |
| chmod +x apollo-federation/gateways/hive-router/hive_router | |
| fi | |
| - name: Download prebuilt subgraphs | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.subgraphs_artifact }} | |
| path: prebuilt-subgraphs/ | |
| - name: Restore prebuilt subgraphs | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GATEWAY_PATH: ${{ matrix.gateway.path }} | |
| SUBGRAPHS_DIR: ${{ matrix.gateway.subgraphs }} | |
| SUBGRAPHS_ARTIFACT: ${{ matrix.gateway.subgraphs_artifact }} | |
| run: | | |
| set -euo pipefail | |
| find_subgraphs_archive() { | |
| find prebuilt-subgraphs -type f -name '*.tar.gz' | head -n1 | |
| } | |
| redownload_subgraphs_artifact() { | |
| rm -rf prebuilt-subgraphs | |
| mkdir -p prebuilt-subgraphs | |
| gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --name "$SUBGRAPHS_ARTIFACT" --dir prebuilt-subgraphs | |
| } | |
| ATTEMPTS=3 | |
| for attempt in $(seq 1 "$ATTEMPTS"); do | |
| SUBGRAPHS_ARCHIVE="$(find_subgraphs_archive)" | |
| if [[ -z "$SUBGRAPHS_ARCHIVE" ]]; then | |
| echo "Subgraphs archive not found (attempt $attempt/$ATTEMPTS)" | |
| elif tar -tzf "$SUBGRAPHS_ARCHIVE" >/dev/null 2>&1; then | |
| tar -xzf "$SUBGRAPHS_ARCHIVE" -C . | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| exit 0 | |
| else | |
| echo "Subgraphs archive integrity check failed (attempt $attempt/$ATTEMPTS): $SUBGRAPHS_ARCHIVE" | |
| ls -lh "$SUBGRAPHS_ARCHIVE" || true | |
| fi | |
| if [[ "$attempt" -lt "$ATTEMPTS" ]]; then | |
| echo "Re-downloading artifact '$SUBGRAPHS_ARTIFACT' for retry..." | |
| if ! redownload_subgraphs_artifact; then | |
| echo "Artifact re-download failed (attempt $((attempt + 1))/$ATTEMPTS)." | |
| fi | |
| sleep $((attempt * 5)) | |
| fi | |
| done | |
| find prebuilt-subgraphs -maxdepth 3 -print || true | |
| SUBGRAPHS_ROOT="${GATEWAY_PATH%%/*}" | |
| SUBGRAPHS_PATH="${SUBGRAPHS_ROOT}/${SUBGRAPHS_DIR}" | |
| echo "Archive restore failed after $ATTEMPTS attempts; falling back to local rebuild: $SUBGRAPHS_PATH" | |
| if [[ ! -f "$SUBGRAPHS_PATH/build.sh" ]]; then | |
| echo "Error: fallback build script not found: $SUBGRAPHS_PATH/build.sh" | |
| exit 1 | |
| fi | |
| chmod +x "$SUBGRAPHS_PATH/build.sh" | |
| (cd "$SUBGRAPHS_PATH" && bash build.sh) | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| - name: Kill stale perfrunner processes | |
| run: | | |
| sudo pkill -u perfrunner || true | |
| sleep 1 | |
| sudo pkill -9 -u perfrunner || true | |
| - name: Prepare workspace for perfrunner | |
| run: | | |
| # Both users need write access during a run: | |
| # - perfrunner writes dependency artifacts (node_modules, target, etc.) | |
| # - github-runner writes benchmark outputs (results/, summaries) | |
| sudo chmod -R a+rX,g+rwX . | |
| sudo chown -R perfrunner:github-runner . | |
| # Preserve github-runner as group owner for newly created files/dirs. | |
| sudo find . -type d -exec chmod g+s {} + | |
| - name: Prepare diagnostics directory | |
| if: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| mkdir -p "${{ matrix.gateway.path }}/diagnostics" | |
| { | |
| echo "diagnostic_run=true" | |
| echo "mode=burst" | |
| echo "gateway=${{ matrix.gateway.name }}" | |
| echo "started_utc=${{ github.run_started_at || 'n/a' }}" | |
| echo "run_id=${{ github.run_id }}" | |
| } > "${{ matrix.gateway.path }}/diagnostics/run-info.txt" | |
| sudo chown -R perfrunner:github-runner "${{ matrix.gateway.path }}/diagnostics" | |
| sudo chmod -R u+rwX,g+rwX "${{ matrix.gateway.path }}/diagnostics" | |
| - name: Run benchmark | |
| # Feddi is excluded from the burst matrix entirely (gateways_burst), so no | |
| # per-gateway failure tolerance is needed here — it is reported as "not run" | |
| # in the burst results table instead. | |
| run: ./k6/benchmark.sh ${{ matrix.gateway.path }} ${{ matrix.gateway.subgraphs }} burst | |
| env: | |
| BENCH_DISPLAY_NAME: ${{ matrix.gateway.display_name || '' }} | |
| BENCH_GATEWAY_NAME: ${{ matrix.gateway.base_name || '' }} | |
| BENCH_SUBGRAPH_TECH: ${{ matrix.gateway.subgraph_tech || '' }} | |
| BENCH_DIAGNOSTIC_RUN: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' && '1' || '0' }} | |
| BENCH_DIAGNOSTICS_DIR: ${{ github.workspace }}/${{ matrix.gateway.path }}/diagnostics | |
| USE_PREBUILT_SUBGRAPHS: "1" | |
| USE_PREBUILT_GATEWAY: "1" | |
| - name: Kill all perfrunner processes | |
| if: always() | |
| run: | | |
| echo "Killing all perfrunner processes..." | |
| sudo pkill -u perfrunner || true | |
| sleep 2 | |
| sudo pkill -9 -u perfrunner || true | |
| echo "Done." | |
| - name: Reclaim workspace ownership | |
| if: always() | |
| run: sudo chown -R $(id -u):$(id -g) . | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: results-${{ matrix.gateway.name }}-burst | |
| path: ${{ matrix.gateway.path }}/results/ | |
| - name: Inspect diagnostics directory | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| echo "Diagnostics directory: ${{ matrix.gateway.path }}/diagnostics" | |
| ls -la "${{ matrix.gateway.path }}/diagnostics" || true | |
| find "${{ matrix.gateway.path }}/diagnostics" -maxdepth 2 -type f -print | sort || true | |
| - name: Upload diagnostics | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diagnostics-${{ matrix.gateway.name }}-burst | |
| path: ${{ matrix.gateway.path }}/diagnostics/ | |
| if-no-files-found: ignore | |
| benchmark-constant-latency: | |
| needs: | |
| - prepare-matrix | |
| - build-subgraphs | |
| - build-gateways | |
| if: ${{ always() && !cancelled() && needs.prepare-matrix.outputs.run_latency == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| gateway: ${{ fromJson(needs.prepare-matrix.outputs.gateways_net) }} | |
| runs-on: | |
| group: benchmarking-1 | |
| name: ${{ matrix.gateway.name }} (constant-latency) | |
| env: | |
| BENCH_RUNS: ${{ inputs.runs || '9' }} | |
| MEASURE_SECONDS: ${{ inputs.duration || '120' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download prebuilt gateway | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.gateway_artifact }} | |
| path: prebuilt-gateway/ | |
| - name: Restore prebuilt gateway | |
| run: | | |
| GATEWAY_ARCHIVE="$(find prebuilt-gateway -type f -name '*.tar.gz' | head -n1)" | |
| if [[ -z "$GATEWAY_ARCHIVE" ]]; then | |
| echo "Error: prebuilt gateway archive not found" | |
| find prebuilt-gateway -maxdepth 3 -print | |
| exit 1 | |
| fi | |
| tar -xzf "$GATEWAY_ARCHIVE" -C . | |
| if [[ -f apollo-federation/gateways/apollo-router/router ]]; then | |
| chmod +x apollo-federation/gateways/apollo-router/router | |
| fi | |
| if [[ -f apollo-federation/gateways/cosmo/cosmo ]]; then | |
| chmod +x apollo-federation/gateways/cosmo/cosmo | |
| fi | |
| if [[ -f apollo-federation/gateways/grafbase/grafbase-gateway ]]; then | |
| chmod +x apollo-federation/gateways/grafbase/grafbase-gateway | |
| fi | |
| if [[ -f apollo-federation/gateways/hive-router/hive_router ]]; then | |
| chmod +x apollo-federation/gateways/hive-router/hive_router | |
| fi | |
| - name: Download prebuilt subgraphs | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ${{ matrix.gateway.subgraphs_artifact }} | |
| path: prebuilt-subgraphs/ | |
| - name: Restore prebuilt subgraphs | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GATEWAY_PATH: ${{ matrix.gateway.path }} | |
| SUBGRAPHS_DIR: ${{ matrix.gateway.subgraphs }} | |
| SUBGRAPHS_ARTIFACT: ${{ matrix.gateway.subgraphs_artifact }} | |
| run: | | |
| set -euo pipefail | |
| find_subgraphs_archive() { | |
| find prebuilt-subgraphs -type f -name '*.tar.gz' | head -n1 | |
| } | |
| redownload_subgraphs_artifact() { | |
| rm -rf prebuilt-subgraphs | |
| mkdir -p prebuilt-subgraphs | |
| gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --name "$SUBGRAPHS_ARTIFACT" --dir prebuilt-subgraphs | |
| } | |
| ATTEMPTS=3 | |
| for attempt in $(seq 1 "$ATTEMPTS"); do | |
| SUBGRAPHS_ARCHIVE="$(find_subgraphs_archive)" | |
| if [[ -z "$SUBGRAPHS_ARCHIVE" ]]; then | |
| echo "Subgraphs archive not found (attempt $attempt/$ATTEMPTS)" | |
| elif tar -tzf "$SUBGRAPHS_ARCHIVE" >/dev/null 2>&1; then | |
| tar -xzf "$SUBGRAPHS_ARCHIVE" -C . | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| exit 0 | |
| else | |
| echo "Subgraphs archive integrity check failed (attempt $attempt/$ATTEMPTS): $SUBGRAPHS_ARCHIVE" | |
| ls -lh "$SUBGRAPHS_ARCHIVE" || true | |
| fi | |
| if [[ "$attempt" -lt "$ATTEMPTS" ]]; then | |
| echo "Re-downloading artifact '$SUBGRAPHS_ARTIFACT' for retry..." | |
| if ! redownload_subgraphs_artifact; then | |
| echo "Artifact re-download failed (attempt $((attempt + 1))/$ATTEMPTS)." | |
| fi | |
| sleep $((attempt * 5)) | |
| fi | |
| done | |
| find prebuilt-subgraphs -maxdepth 3 -print || true | |
| SUBGRAPHS_ROOT="${GATEWAY_PATH%%/*}" | |
| SUBGRAPHS_PATH="${SUBGRAPHS_ROOT}/${SUBGRAPHS_DIR}" | |
| echo "Archive restore failed after $ATTEMPTS attempts; falling back to local rebuild: $SUBGRAPHS_PATH" | |
| if [[ ! -f "$SUBGRAPHS_PATH/build.sh" ]]; then | |
| echo "Error: fallback build script not found: $SUBGRAPHS_PATH/build.sh" | |
| exit 1 | |
| fi | |
| chmod +x "$SUBGRAPHS_PATH/build.sh" | |
| (cd "$SUBGRAPHS_PATH" && bash build.sh) | |
| if [[ -f apollo-federation/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x apollo-federation/subgraphs-rust/target/release/subgraphs | |
| fi | |
| if [[ -f composite-schema/subgraphs-rust/target/release/subgraphs ]]; then | |
| chmod +x composite-schema/subgraphs-rust/target/release/subgraphs | |
| fi | |
| - name: Kill stale perfrunner processes | |
| run: | | |
| sudo pkill -u perfrunner || true | |
| sleep 1 | |
| sudo pkill -9 -u perfrunner || true | |
| - name: Prepare workspace for perfrunner | |
| run: | | |
| sudo chmod -R a+rX,g+rwX . | |
| sudo chown -R perfrunner:github-runner . | |
| sudo find . -type d -exec chmod g+s {} + | |
| - name: Prepare diagnostics directory | |
| if: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| mkdir -p "${{ matrix.gateway.path }}/diagnostics" | |
| { | |
| echo "diagnostic_run=true" | |
| echo "mode=constant-latency" | |
| echo "gateway=${{ matrix.gateway.name }}" | |
| echo "started_utc=${{ github.run_started_at || 'n/a' }}" | |
| echo "run_id=${{ github.run_id }}" | |
| } > "${{ matrix.gateway.path }}/diagnostics/run-info.txt" | |
| sudo chown -R perfrunner:github-runner "${{ matrix.gateway.path }}/diagnostics" | |
| sudo chmod -R u+rwX,g+rwX "${{ matrix.gateway.path }}/diagnostics" | |
| - name: Run benchmark | |
| run: ./k6/benchmark.sh ${{ matrix.gateway.path }} ${{ matrix.gateway.subgraphs }} constant-latency | |
| env: | |
| BENCH_DISPLAY_NAME: ${{ matrix.gateway.display_name || '' }} | |
| BENCH_GATEWAY_NAME: ${{ matrix.gateway.base_name || '' }} | |
| BENCH_SUBGRAPH_TECH: ${{ matrix.gateway.subgraph_tech || '' }} | |
| BENCH_DIAGNOSTIC_RUN: ${{ needs.prepare-matrix.outputs.is_diagnostic == 'true' && '1' || '0' }} | |
| BENCH_DIAGNOSTICS_DIR: ${{ github.workspace }}/${{ matrix.gateway.path }}/diagnostics | |
| USE_PREBUILT_SUBGRAPHS: "1" | |
| USE_PREBUILT_GATEWAY: "1" | |
| - name: Kill all perfrunner processes | |
| if: always() | |
| run: | | |
| echo "Killing all perfrunner processes..." | |
| sudo pkill -u perfrunner || true | |
| sleep 2 | |
| sudo pkill -9 -u perfrunner || true | |
| echo "Done." | |
| - name: Reclaim workspace ownership | |
| if: always() | |
| run: sudo chown -R $(id -u):$(id -g) . | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: results-${{ matrix.gateway.name }}-constant-latency | |
| path: ${{ matrix.gateway.path }}/results/ | |
| - name: Inspect diagnostics directory | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| run: | | |
| echo "Diagnostics directory: ${{ matrix.gateway.path }}/diagnostics" | |
| ls -la "${{ matrix.gateway.path }}/diagnostics" || true | |
| find "${{ matrix.gateway.path }}/diagnostics" -maxdepth 2 -type f -print | sort || true | |
| - name: Upload diagnostics | |
| if: ${{ always() && needs.prepare-matrix.outputs.is_diagnostic == 'true' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diagnostics-${{ matrix.gateway.name }}-constant-latency | |
| path: ${{ matrix.gateway.path }}/diagnostics/ | |
| if-no-files-found: ignore | |
| generate-results-constant: | |
| needs: | |
| - prepare-matrix | |
| - benchmark-constant | |
| - generate-results-constant-latency | |
| if: ${{ always() && !cancelled() && needs.prepare-matrix.outputs.should_publish == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: results-*-constant | |
| path: artifacts/ | |
| - name: Generate constant result document | |
| env: | |
| EXPECTED_GATEWAYS_JSON: ${{ needs.prepare-matrix.outputs.gateways }} | |
| EXPECTED_GATEWAYS_NET_JSON: ${{ needs.prepare-matrix.outputs.gateways_net }} | |
| # Gateways deliberately not run in constant — rendered as "not run". | |
| NOT_RUN_GATEWAYS_JSON: ${{ needs.prepare-matrix.outputs.not_run_constant }} | |
| RESULT_MODES: constant | |
| run: python3 k6/generate-results.py artifacts/ benchmark-runs/${{ github.run_number }} | |
| - name: Write constant summary | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT.md" | |
| echo "# Benchmark Results (Constant)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [[ -f "$CONSTANT_RESULTS_FILE" ]]; then | |
| cat "$CONSTANT_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No constant results were generated." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Commit constant results | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT.md" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| if [[ -f "$CONSTANT_RESULTS_FILE" ]]; then | |
| git add "$CONSTANT_RESULTS_FILE" | |
| fi | |
| git diff --cached --quiet || git commit -m "Add constant benchmark results for run #${{ github.run_number }}" | |
| git push | |
| - name: Upload combined constant results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: all-results-constant | |
| path: artifacts/ | |
| generate-results-burst: | |
| needs: | |
| - prepare-matrix | |
| - benchmark-burst | |
| - generate-results-constant | |
| if: ${{ !cancelled() && needs.prepare-matrix.outputs.should_publish == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: results-*-burst | |
| path: artifacts/ | |
| - name: Generate burst result document | |
| env: | |
| EXPECTED_GATEWAYS_JSON: ${{ needs.prepare-matrix.outputs.gateways }} | |
| EXPECTED_GATEWAYS_NET_JSON: ${{ needs.prepare-matrix.outputs.gateways_net }} | |
| # Gateways deliberately not run in burst (feddi) — rendered as "not run". | |
| NOT_RUN_GATEWAYS_JSON: ${{ needs.prepare-matrix.outputs.not_run_burst }} | |
| RESULT_MODES: burst | |
| run: python3 k6/generate-results.py artifacts/ benchmark-runs/${{ github.run_number }} | |
| - name: Write burst summary | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| BURST_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_BURST.md" | |
| echo "# Benchmark Results (Burst)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [[ -f "$BURST_RESULTS_FILE" ]]; then | |
| cat "$BURST_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No burst results were generated." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Commit burst results | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| BURST_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_BURST.md" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| if [[ -f "$BURST_RESULTS_FILE" ]]; then | |
| git add "$BURST_RESULTS_FILE" | |
| fi | |
| git diff --cached --quiet || git commit -m "Add burst benchmark results for run #${{ github.run_number }}" | |
| git push | |
| - name: Upload combined burst results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: all-results-burst | |
| path: artifacts/ | |
| generate-results-constant-latency: | |
| needs: | |
| - prepare-matrix | |
| - benchmark-constant-latency | |
| if: ${{ !cancelled() && needs.prepare-matrix.outputs.should_publish == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: results-*-constant-latency | |
| path: artifacts/ | |
| - name: Generate constant-latency result document | |
| env: | |
| EXPECTED_GATEWAYS_JSON: ${{ needs.prepare-matrix.outputs.gateways }} | |
| EXPECTED_GATEWAYS_NET_JSON: ${{ needs.prepare-matrix.outputs.gateways_net }} | |
| RESULT_MODES: constant-latency | |
| run: python3 k6/generate-results.py artifacts/ benchmark-runs/${{ github.run_number }} | |
| - name: Write constant-latency summary | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_LATENCY_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT_LATENCY.md" | |
| echo "# Benchmark Results (Constant Latency)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [[ -f "$CONSTANT_LATENCY_RESULTS_FILE" ]]; then | |
| cat "$CONSTANT_LATENCY_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No constant-latency results were generated." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Commit constant-latency results | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_LATENCY_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT_LATENCY.md" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| if [[ -f "$CONSTANT_LATENCY_RESULTS_FILE" ]]; then | |
| git add "$CONSTANT_LATENCY_RESULTS_FILE" | |
| fi | |
| git diff --cached --quiet || git commit -m "Add constant-latency benchmark results for run #${{ github.run_number }}" | |
| git push | |
| - name: Upload combined constant-latency results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: all-results-constant-latency | |
| path: artifacts/ | |
| benchmark-summary: | |
| needs: | |
| - benchmark-constant | |
| - benchmark-burst | |
| - benchmark-constant-latency | |
| - generate-results-constant | |
| - generate-results-burst | |
| - generate-results-constant-latency | |
| if: ${{ !cancelled() && needs.prepare-matrix.outputs.should_publish == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| - name: Write final benchmark summary | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT.md" | |
| BURST_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_BURST.md" | |
| CONSTANT_LATENCY_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT_LATENCY.md" | |
| echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Run: **Benchmark #${{ github.run_number }}**" >> $GITHUB_STEP_SUMMARY | |
| echo "- Started (UTC): **${{ github.run_started_at || 'start-time-unavailable' }}**" >> $GITHUB_STEP_SUMMARY | |
| echo "- Gateway selection: \`${{ inputs.gateway || 'all' }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "- Benchmark selection: \`${{ inputs.benchmark || 'all' }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "- Diagnostic run: \`${{ inputs.diagnostic_run || 'false' }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "- Runs per gateway: \`${{ inputs.runs || '9' }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "- Duration per run: \`${{ inputs.duration || '120' }}s\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [[ -f "$CONSTANT_RESULTS_FILE" ]]; then | |
| cat "$CONSTANT_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No constant results were generated." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [[ -f "$BURST_RESULTS_FILE" ]]; then | |
| cat "$BURST_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No burst results were generated." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [[ -f "$CONSTANT_LATENCY_RESULTS_FILE" ]]; then | |
| cat "$CONSTANT_LATENCY_RESULTS_FILE" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "No constant-latency results were generated." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Publish latest root results | |
| if: ${{ needs.benchmark-constant.result == 'success' && needs.benchmark-burst.result == 'success' && needs.benchmark-constant-latency.result == 'success' && needs.generate-results-constant.result == 'success' && needs.generate-results-burst.result == 'success' && needs.generate-results-constant-latency.result == 'success' }} | |
| run: | | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| CONSTANT_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT.md" | |
| BURST_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_BURST.md" | |
| CONSTANT_LATENCY_RESULTS_FILE="$RUN_RESULTS_DIR/RESULTS_CONSTANT_LATENCY.md" | |
| for file in "$CONSTANT_RESULTS_FILE" "$BURST_RESULTS_FILE" "$CONSTANT_LATENCY_RESULTS_FILE"; do | |
| if [[ ! -f "$file" ]]; then | |
| echo "Error: expected run result file is missing: $file" | |
| exit 1 | |
| fi | |
| done | |
| cp "$CONSTANT_RESULTS_FILE" RESULTS_CONSTANT.md | |
| cp "$BURST_RESULTS_FILE" RESULTS_BURST.md | |
| cp "$CONSTANT_LATENCY_RESULTS_FILE" RESULTS_CONSTANT_LATENCY.md | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add RESULTS_CONSTANT.md RESULTS_BURST.md RESULTS_CONSTANT_LATENCY.md | |
| git diff --cached --quiet || git commit -m "Update latest benchmark results from run #${{ github.run_number }}" | |
| git push | |
| - name: Skip latest root publish (errors detected) | |
| if: ${{ needs.benchmark-constant.result != 'success' || needs.benchmark-burst.result != 'success' || needs.benchmark-constant-latency.result != 'success' || needs.generate-results-constant.result != 'success' || needs.generate-results-burst.result != 'success' || needs.generate-results-constant-latency.result != 'success' }} | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Latest root files were not updated because one or more benchmark jobs failed." >> $GITHUB_STEP_SUMMARY | |
| - name: Delete failed run results | |
| if: ${{ needs.benchmark-constant.result != 'success' || needs.benchmark-burst.result != 'success' || needs.benchmark-constant-latency.result != 'success' || needs.generate-results-constant.result != 'success' || needs.generate-results-burst.result != 'success' || needs.generate-results-constant-latency.result != 'success' }} | |
| run: | | |
| # benchmark-runs/ must only contain successful runs. The generate-results | |
| # jobs commit per-mode files even when benchmarks failed, so a failed run | |
| # leaves a stale directory behind — remove it here. | |
| RUN_RESULTS_DIR="benchmark-runs/${{ github.run_number }}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| if [[ -d "$RUN_RESULTS_DIR" ]]; then | |
| git rm -r --ignore-unmatch "$RUN_RESULTS_DIR" | |
| git diff --cached --quiet || git commit -m "Remove failed benchmark run #${{ github.run_number }} results" | |
| git push | |
| echo "- Removed \`$RUN_RESULTS_DIR\` because the run failed." >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "- No \`$RUN_RESULTS_DIR\` directory to remove." >> $GITHUB_STEP_SUMMARY | |
| fi |