[CICD]: Add H20 Qwen3.6 E2E cases (#321)

Additional NVIDIA H20 E2E testing:

- Qwen3.6-27B TP2
- Qwen3.6-35B-A3B TP2
- Covers text and image requests
- Models uniformly referenced at /data/models/Qwen/

Manually verified on H20 machine using CUDA CI container, both cases
passed.
CUDA CI is configured to use the `vllm-plugin-ci-h20` runner label.

Author: hooray03

.github/configs/cuda.yml

@@ -29,6 +29,11 @@ runner_labels:
   - gpu-8
   - cuda130
 
+# Device-specific runner labels override the platform defaults for E2E jobs.
+device_runner_labels:
+  h20:
+    - vllm-plugin-ci-h20
+
 # Runner labels for this hardware (Platform hosted)
 # runner_labels:
 #   - ci-vllm-plugin-fl

.github/scripts/load_config.py

@@ -45,6 +45,7 @@ def load_platform_config(platform: str) -> dict:
 
     # Ensure the platform key is present
     config.setdefault("platform", platform)
+    config.setdefault("device_runner_labels", {})
     return config
 
 

.github/workflows/_platform_test.yml

@@ -140,7 +140,7 @@ jobs:
       device: ${{ matrix.test.device }}
       cases: ${{ matrix.test.cases }}
       ci_image: ${{ matrix.test.image || fromJson(needs.setup.outputs.config).ci_image }}
-      runner_labels: ${{ toJson(fromJson(needs.setup.outputs.config).runner_labels) }}
+      runner_labels: ${{ toJson(fromJson(needs.setup.outputs.config).device_runner_labels[matrix.test.device] || fromJson(needs.setup.outputs.config).runner_labels) }}
       container_volumes: ${{ toJson(fromJson(needs.setup.outputs.config).container_volumes) }}
       container_options: ${{ fromJson(needs.setup.outputs.config).container_options }}
       timeout: ${{ matrix.test.timeout || 60 }}

pyproject.toml

@@ -65,6 +65,7 @@ test = [
     "numpy",
     "requests",
     "openai",
+    "pillow",
     "decorator",
     "vllm[audio]==0.20.2",
     "modelscope>=1.18.1",

tests/README.md

@@ -239,6 +239,7 @@ generate:
 | `serve.endpoints` | list | No | Endpoints to test: `completion`, `chat` |
 | `serve.completion_prompt` | str | No | Prompt for `/v1/completions` |
 | `serve.chat_messages` | list | No | Messages for `/v1/chat/completions` |
+| `serve.chat_cases` | list | No | Named chat requests run against one server; `expected` validates required response substrings and `generated_image: true` creates the local image fixture |
 | `serve.max_tokens` | int | No | Max tokens for serving requests (default: 50) |
 | `serve.api_key` | str | No | API key for authenticated endpoints |
 | `serve.extra_engine` | dict | No | Engine param overrides for serving only |

tests/e2e_tests/serving/test_serving_smoke.py

@@ -15,7 +15,9 @@
 controlled by the ``serve.stream`` flag in the model YAML.
 """
 
+import base64
 import os
+from io import BytesIO
 
 import pytest
 import requests
@@ -138,20 +140,75 @@ def _run_chat(base_url: str, headers: dict) -> None:
     Otherwise, uses a plain requests POST.
     """
     serve = _CFG.serve
-    messages = serve.chat_messages or [{"role": "user", "content": "Hello"}]
-
-    if serve.stream:
-        _run_chat_streaming(base_url, serve, messages)
-    else:
-        _run_chat_non_streaming(base_url, headers, serve, messages)
+    chat_cases = serve.chat_cases or [
+        {
+            "name": "default",
+            "messages": serve.chat_messages or [{"role": "user", "content": "Hello"}],
+        }
+    ]
+
+    for chat_case in chat_cases:
+        case_name = chat_case.get("name", "unnamed")
+        if chat_case.get("generated_image"):
+            messages = _generated_image_messages(
+                chat_case.get("prompt", "Describe this image.")
+            )
+        else:
+            messages = chat_case.get("messages", [])
+
+        assert messages, f"Chat case '{case_name}' has no messages"
+        print(f"\nRunning chat case: {case_name}")
+        if serve.stream:
+            response_text = _run_chat_streaming(base_url, serve, messages)
+        else:
+            response_text = _run_chat_non_streaming(base_url, headers, serve, messages)
+
+        expected = chat_case.get("expected")
+        if expected:
+            expected_values = expected if isinstance(expected, list) else [expected]
+            normalized_response = response_text.casefold()
+            missing = [
+                value
+                for value in expected_values
+                if value.casefold() not in normalized_response
+            ]
+            is_correct = not missing
+            print(f"Answer validation: {'CORRECT' if is_correct else 'INCORRECT'}")
+            assert is_correct, f"Chat case '{case_name}' is missing expected: {missing}"
+
+
+def _generated_image_messages(prompt: str) -> list[dict]:
+    """Create the local image payload used by the Qwen multimodal smoke case."""
+    from PIL import Image, ImageDraw
+
+    image = Image.new("RGB", (300, 200), color="white")
+    draw = ImageDraw.Draw(image)
+    draw.rectangle((50, 50, 250, 150), fill="blue")
+    draw.text((90, 80), "Hello VLM", fill="yellow")
+
+    buffer = BytesIO()
+    image.save(buffer, format="JPEG")
+    encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
+    return [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
+                },
+                {"type": "text", "text": prompt},
+            ],
+        }
+    ]
 
 
 def _run_chat_non_streaming(
     base_url: str,
     headers: dict,
     serve,
     messages: list[dict],
-) -> None:
+) -> str:
     """Non-streaming chat completions via raw requests."""
     payload: dict = {
         "model": _REQUEST_MODEL,
@@ -177,13 +234,14 @@ def _run_chat_non_streaming(
     content = data["choices"][0]["message"]["content"]
     assert len(content.strip()) > 0, "Assistant message is empty"
     print(f"\nResponse: {content}")
+    return content
 
 
 def _run_chat_streaming(
     base_url: str,
     serve,
     messages: list[dict],
-) -> None:
+) -> str:
     """Streaming chat completions via OpenAI SDK."""
     import httpx
     from openai import OpenAI
@@ -214,6 +272,7 @@ def _run_chat_streaming(
 
     assert len(text.strip()) > 0, "Streaming response is empty"
     print(f"\nStreaming response: {text}")
+    return text
 
 
 def _run_embedding(base_url: str, headers: dict) -> None:

tests/models/qwen3_6/27b_tp2_262k.yaml

@@ -0,0 +1,43 @@
+# Copyright 2026 FlagOS Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Qwen3.6-27B 262K serving configuration (TP=2).
+
+llm:
+  model: "/data/models/Qwen/Qwen3.6-27B"
+  tensor_parallel_size: 2
+  pipeline_parallel_size: 1
+  max_model_len: 262144
+  trust_remote_code: false
+
+serve:
+  served_model_name: "qwen"
+  startup_retries: 120
+  endpoints: ["chat"]
+  stream: true
+  max_tokens: 1024
+  sampling:
+    temperature: 0.0
+  chat_cases:
+    - name: text
+      messages:
+        - role: "user"
+          content: "The capital of France is"
+      expected: "Paris"
+    - name: image
+      generated_image: true
+      prompt: "Describe all visible text, colors, and shapes in English."
+      expected:
+        - "Hello VLM"
+        - "blue rectangle"

tests/models/qwen3_6/35b_a3b_tp2_262k.yaml

@@ -0,0 +1,43 @@
+# Copyright 2026 FlagOS Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Qwen3.6-35B-A3B 262K serving configuration (TP=2).
+
+llm:
+  model: "/data/models/Qwen/Qwen3.6-35B-A3B"
+  tensor_parallel_size: 2
+  pipeline_parallel_size: 1
+  max_model_len: 262144
+  trust_remote_code: false
+
+serve:
+  served_model_name: "qwen"
+  startup_retries: 120
+  endpoints: ["chat"]
+  stream: true
+  max_tokens: 1024
+  sampling:
+    temperature: 0.0
+  chat_cases:
+    - name: text
+      messages:
+        - role: "user"
+          content: "The capital of France is"
+      expected: "Paris"
+    - name: image
+      generated_image: true
+      prompt: "Describe all visible text, colors, and shapes in English."
+      expected:
+        - "Hello VLM"
+        - "blue rectangle"

tests/platforms/cuda.yaml

@@ -15,7 +15,7 @@
 # CUDA-Based Platform Configuration
 # Flexible test selection mechanism for NVIDIA GPU environments
 #
-# This platform configuration supports multiple device types (a100, a800, h100)
+# This platform configuration supports multiple device types (a100, a800, h100, h20)
 # Users can modify the following:
 # - functional: Add/remove test case names in the support lists
 #   Example: aquila: ["tp2_pp2", "tp4_pp2"] -> add or remove items
@@ -45,6 +45,10 @@ device_types:
     compute_capability: "9.0"
     memory_gb: 80
     tags: [hopper, fp8, bf16]
+  h20:
+    compute_capability: "9.0"
+    memory_gb: 140
+    tags: [hopper, fp8, bf16, china-variant]
 
 # Default numerical tolerance for result comparison (used when device doesn't override)
 tolerance:
@@ -97,3 +101,10 @@ a100:
     benchmark:
       enabled: true
       smoke: ["throughput", "latency", "serve"]
+
+h20:
+  name: "h20"
+  tests:
+    e2e:
+      serving:
+        qwen3_6: ["27b_tp2_262k", "35b_a3b_tp2_262k"]

tests/utils/model_config.py

@@ -108,6 +108,7 @@ class ServeConfig:
                    ``"embedding"``).
         completion_prompt: Prompt string for ``/v1/completions`` endpoint.
         chat_messages: Messages list for ``/v1/chat/completions`` endpoint.
+        chat_cases: Named chat cases to run against one server instance.
         max_tokens: Max tokens for serving requests.
         served_model_name: Alias passed via ``--served-model-name``.
             Empty string means use the model path directly.
@@ -127,6 +128,7 @@ class ServeConfig:
     endpoints: list[str] = field(default_factory=list)
     completion_prompt: str = "Hello"
     chat_messages: list[dict[str, Any]] = field(default_factory=list)
+    chat_cases: list[dict[str, Any]] = field(default_factory=list)
     max_tokens: int = 50
     served_model_name: str = ""
     startup_retries: int = 60
@@ -147,6 +149,7 @@ def from_dict(cls, raw: dict[str, Any]) -> ServeConfig:
             endpoints=raw.get("endpoints", []),
             completion_prompt=raw.get("completion_prompt", "Hello"),
             chat_messages=raw.get("chat_messages", []),
+            chat_cases=raw.get("chat_cases", []),
             max_tokens=raw.get("max_tokens", 50),
             served_model_name=raw.get("served_model_name", ""),
             startup_retries=int(raw.get("startup_retries", 60)),