Add qwen3_6 dense model support for v0.1.0. (#160)

### PR Category
Core | Vendor

### PR Type
New Features

### Description
Add qwen3_5 dense model support for v0.1.0.

### Related Issues
None

### Changes
<!-- List the key changes made in this PR. -->
-

### Testing
<!-- How has this change been tested? Include test commands, hardware
used, etc. -->
-

### Checklist
- [ ] I have run the existing tests and they pass
- [ ] I have added tests for my changes (if applicable)
- [ ] I have updated the documentation (if applicable)

---------

Signed-off-by: ftgreat <ldwang@baai.ac.cn>
Co-authored-by: XMing <xmhubj@gmail.com>

Author: ftgreat

vllm_fl/__init__.py

@@ -35,6 +35,20 @@ def register():
     """Register the FL platform."""
     _patch_transformers_compat()
 
+    try:
+        from vllm.transformers_utils.config import _CONFIG_REGISTRY
+        from vllm_fl.configs.qwen3_5 import Qwen3_5Config
+        _CONFIG_REGISTRY["qwen3_5"] = Qwen3_5Config
+    except Exception as e:
+        logger.error(f"Register Qwen3.5 config in platform plugin error: {str(e)}")
+
+    try:
+        from vllm.transformers_utils.config import _CONFIG_REGISTRY
+        from vllm_fl.configs.qwen3_5_moe import Qwen3_5MoeConfig
+        _CONFIG_REGISTRY["qwen3_5_moe"] = Qwen3_5MoeConfig
+    except Exception as e:
+        logger.error(f"Register Qwen3.5 MoE config in platform plugin error: {str(e)}")
+
     # Model-specific platform patches
     from vllm_fl.patches.glm_moe_dsa import apply_platform_patches as glm5_platform
     glm5_platform()
@@ -68,6 +82,14 @@ def register_model():
     except Exception as e:
         logger.error(f"Register Qwen3.5 MoE config error: {str(e)}")
 
+    # Register Qwen3.5 (non-MoE) config
+    try:
+        from vllm.transformers_utils.config import _CONFIG_REGISTRY
+        from vllm_fl.configs.qwen3_5 import Qwen3_5Config
+        _CONFIG_REGISTRY["qwen3_5"] = Qwen3_5Config
+    except Exception as e:
+        logger.error(f"Register Qwen3.5 config error: {str(e)}")
+
     # Register Qwen3Next model
     try:
         import vllm.model_executor.models.qwen3_next as qwen3_next_module
@@ -95,6 +117,15 @@ def register_model():
     except Exception as e:
         logger.error(f"Register Qwen3.5 MoE model error: {str(e)}")
 
+    # Register Qwen3.5 (non-MoE) model
+    try:
+        ModelRegistry.register_model(
+            "Qwen3_5ForConditionalGeneration",
+            "vllm_fl.models.qwen3_5:Qwen3_5ForConditionalGeneration"
+        )
+    except Exception as e:
+        logger.error(f"Register Qwen3.5 model error: {str(e)}")
+
     # Register MiniCPMO model
     try:
         ModelRegistry.register_model(

vllm_fl/configs/qwen3_5.py

@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
+# All rights reserved.
+"""Qwen3.5 (non-MoE) model configuration for vLLM plugin."""
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+def _layer_type_validation(layer_types, num_hidden_layers):
+    if layer_types is not None and num_hidden_layers is not None:
+        if len(layer_types) != num_hidden_layers:
+            raise ValueError(
+                f"Length of layer_types ({len(layer_types)}) must match "
+                f"num_hidden_layers ({num_hidden_layers})"
+            )
+
+
+class Qwen3_5TextConfig(PretrainedConfig):
+    model_type = "qwen3_5_text"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    base_config_key = "text_config"
+
+    def __init__(
+        self,
+        vocab_size=248320,
+        hidden_size=4096,
+        intermediate_size=12288,
+        num_hidden_layers=32,
+        num_attention_heads=16,
+        num_key_value_heads=4,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_parameters=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        head_dim=256,
+        linear_conv_kernel_dim=4,
+        linear_key_head_dim=128,
+        linear_value_head_dim=128,
+        linear_num_key_heads=16,
+        linear_num_value_heads=32,
+        layer_types=None,
+        full_attention_interval=4,
+        attn_output_gate=True,
+        pad_token_id=None,
+        bos_token_id=None,
+        eos_token_id=None,
+        **kwargs,
+    ):
+        kwargs.pop("ignore_keys_at_rope_validation", None)
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.head_dim = head_dim
+        self.rope_parameters = rope_parameters
+        self.attn_output_gate = attn_output_gate
+
+        self.layer_types = layer_types
+        if self.layer_types is None:
+            self.layer_types = [
+                "linear_attention"
+                if (i + 1) % full_attention_interval != 0
+                else "full_attention"
+                for i in range(num_hidden_layers)
+            ]
+        _layer_type_validation(self.layer_types, num_hidden_layers)
+
+        self.full_attention_interval = full_attention_interval
+        self.linear_conv_kernel_dim = linear_conv_kernel_dim
+        self.linear_key_head_dim = linear_key_head_dim
+        self.linear_value_head_dim = linear_value_head_dim
+        self.linear_num_key_heads = linear_num_key_heads
+        self.linear_num_value_heads = linear_num_value_heads
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+class Qwen3_5VisionConfig(PretrainedConfig):
+    model_type = "qwen3_5"
+
+    def __init__(
+        self,
+        depth=27,
+        hidden_size=1152,
+        hidden_act="gelu_pytorch_tanh",
+        intermediate_size=4304,
+        num_heads=16,
+        in_channels=3,
+        patch_size=16,
+        spatial_merge_size=2,
+        temporal_patch_size=2,
+        num_position_embeddings=2304,
+        out_hidden_size=4096,
+        initializer_range=0.02,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.depth = depth
+        self.hidden_size = hidden_size
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.num_heads = num_heads
+        self.in_channels = in_channels
+        self.patch_size = patch_size
+        self.spatial_merge_size = spatial_merge_size
+        self.temporal_patch_size = temporal_patch_size
+        self.num_position_embeddings = num_position_embeddings
+        self.out_hidden_size = out_hidden_size
+        self.initializer_range = initializer_range
+
+
+class Qwen3_5Config(PretrainedConfig):
+    model_type = "qwen3_5"
+    sub_configs = {
+        "vision_config": Qwen3_5VisionConfig,
+        "text_config": Qwen3_5TextConfig,
+    }
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        text_config=None,
+        vision_config=None,
+        image_token_id=248056,
+        video_token_id=248057,
+        vision_start_token_id=248053,
+        vision_end_token_id=248054,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        if isinstance(vision_config, dict):
+            self.vision_config = self.sub_configs["vision_config"](**vision_config)
+        elif vision_config is None:
+            self.vision_config = self.sub_configs["vision_config"]()
+
+        if isinstance(text_config, dict):
+            self.text_config = self.sub_configs["text_config"](**text_config)
+        elif text_config is None:
+            self.text_config = self.sub_configs["text_config"]()
+
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+        self.vision_start_token_id = vision_start_token_id
+        self.vision_end_token_id = vision_end_token_id
+        super().__init__(**kwargs)
+        self.tie_word_embeddings = tie_word_embeddings
+
+
+__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"]