vllm-plugin-FL/.github/workflows/_benchmark_test.yml at main · flagos-ai/vllm-plugin-FL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
# Copyright 2026 FlagOS Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Reusable benchmark smoke workflow.
#
# Runs lightweight benchmark smoke tests for vllm bench throughput,
# latency, and serve. These tests validate benchmark entrypoints and
# basic plugin integration, not performance regression.

name: Benchmark Smoke Test

on:
  workflow_call:
    inputs:
      platform:
        description: "Platform name (e.g., cuda, ascend)"
        required: true
        type: string
      ci_image:
        description: "Docker image for the test container"
        required: true
        type: string
      runner_labels:
        description: "JSON array of runner labels"
        required: true
        type: string
      container_volumes:
        description: "JSON array of volume mounts"
        required: false
        type: string
        default: "[]"
      container_options:
        description: "Docker container options string"
        required: false
        type: string
        default: ""
      timeout:
        description: "Timeout for the job in minutes"
        required: false
        type: number
        default: 30

jobs:
  benchmark-test:
    name: "Benchmark Smoke (${{ inputs.platform }})"
    runs-on: ${{ fromJson(inputs.runner_labels) }}
    timeout-minutes: ${{ inputs.timeout }}
    container:
      image: ${{ inputs.ci_image }}
      volumes: ${{ fromJson(inputs.container_volumes) }}
      options: ${{ inputs.container_options }}

    steps:
      - name: Checkout (attempt 1)
        id: checkout1
        uses: actions/checkout@v4
        continue-on-error: true
      - name: Checkout (attempt 2)
        id: checkout2
        if: steps.checkout1.outcome == 'failure'
        uses: actions/checkout@v4
        continue-on-error: true
      - name: Checkout (attempt 3)
        if: steps.checkout2.outcome == 'failure'
        uses: actions/checkout@v4

      - name: Check device availability
        run: bash .github/scripts/${{ inputs.platform }}/check.sh

      - name: Install project
        run: bash .github/scripts/${{ inputs.platform }}/setup.sh

      - name: Run benchmark smoke tests
        run: |
          python tests/run.py \
            --platform ${{ inputs.platform }} \
            --scope benchmark

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        continue-on-error: true
        with:
          name: benchmark-${{ inputs.platform }}
          path: |
            test-results-${{ inputs.platform }}.xml
            test-results-${{ inputs.platform }}.json