bench.yml

name: Geniex Bench

on:
  workflow_dispatch:
    inputs:
      device:
        description: "QDC chipsets, comma-separated (blank = all)"
        type: string
        default: ""
      model:
        description: "Model names from bench-models.json, comma-separated (blank = all)"
        type: string
        default: ""

permissions:
  contents: read
  packages: read

jobs:
  build-sdk:
    uses: ./.github/workflows/_build-sdk.yml
    secrets: inherit
    with:
      upload_artifact: true

  load-models:
    needs: [build-sdk]
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set.outputs.matrix }}
      devices: ${{ steps.set.outputs.devices }}
    steps:
      - uses: actions/checkout@v7
      - id: set
        env:
          PICK_DEVICE: ${{ github.event.inputs.device }}
          PICK_MODEL: ${{ github.event.inputs.model }}
        run: |
          set -euo pipefail
          MODELS_FILE=sdk/benchmark/qdc/bench-models.json

          all_devs='["QCS9075M","SC8380XP","SC8480XP","SM8750","SM8850"]'
          if [ -n "$PICK_DEVICE" ]; then
            devices=$(jq -cn --arg s "$PICK_DEVICE" \
              '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))')
          else
            devices="$all_devs"
          fi
          echo "devices=$devices" >> "$GITHUB_OUTPUT"

          if [ -n "$PICK_MODEL" ]; then
            matrix=$(jq -c --arg s "$PICK_MODEL" '
              ($s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))) as $pick
              | map(select(.name as $n | $pick | index($n)))
            ' "$MODELS_FILE")
            picked_n=$(jq 'length' <<<"$matrix")
            want_n=$(jq -n --arg s "$PICK_MODEL" \
              '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0)) | length')
            if [ "$picked_n" != "$want_n" ]; then
              echo "::error::model input has unknown name(s): $PICK_MODEL"
              echo "::error::available: $(jq -r '.[].name' "$MODELS_FILE" | paste -sd, -)"
              exit 1
            fi
          else
            matrix=$(jq -c . "$MODELS_FILE")
          fi
          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"

  bench:
    name: ${{ matrix.device }} · ${{ matrix.model.plugin }} · ${{ matrix.model.name }}
    needs: [build-sdk, load-models]
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        device: ${{ fromJson(needs.load-models.outputs.devices) }}
        model: ${{ fromJson(needs.load-models.outputs.matrix) }}
    env:
      QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.11"
      - id: sdk
        env:
          DEVICE: ${{ matrix.device }}
        run: |
          case "$DEVICE" in
            SM*)         echo "name=sdk-android-arm64" ;;
            SC*|CRD*|X*) echo "name=sdk-windows-arm64" ;;
            *)           echo "name=sdk-linux-arm64" ;;
          esac >> "$GITHUB_OUTPUT"
      - uses: actions/download-artifact@v8
        with:
          name: ${{ steps.sdk.outputs.name }}
          path: pkg-geniex
      - uses: ./.github/actions/install-qdc-sdk
      - env:
          DEVICE: ${{ matrix.device }}
          MODEL_NAME: ${{ matrix.model.name }}
        run: |
          python sdk/benchmark/qdc/run_qdc_jobs.py \
            --pkg-dir pkg-geniex \
            --device "$DEVICE" \
            --model-name "$MODEL_NAME" \
            --cells-out "$DEVICE-$MODEL_NAME.json"
      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: bench-cells-${{ matrix.device }}-${{ matrix.model.name }}
          path: ${{ matrix.device }}-${{ matrix.model.name }}.json
          if-no-files-found: ignore

  aggregate:
    name: aggregate · ${{ matrix.device }}
    needs: [load-models, bench]
    if: always()
    runs-on: ubuntu-latest
    strategy:
      matrix:
        device: ${{ fromJson(needs.load-models.outputs.devices) }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.11"
      - uses: actions/download-artifact@v8
        with:
          pattern: bench-cells-${{ matrix.device }}-*
          path: cells
      - env:
          DEVICE: ${{ matrix.device }}
        run: |
          python sdk/benchmark/qdc/run_qdc_jobs.py \
            --device "$DEVICE" \
            --render-dir cells