[feat] support for iluvatar hardware backend (#58)

### PR Category
Vendor

### PR Types
New Features

### PR Description
Support Qwen3-next on iluvatar hardware backend

Author: darrenzhang1007

requirements_iluvatar.txt

@@ -0,0 +1,10 @@
+#vllm==0.13.0
+decorator
+pyyaml
+scipy
+setuptools <= 79.0.1
+setuptools-scm
+scikit-build-core==0.11
+pybind11
+ninja
+cmake

vllm_fl/dispatch/backends/vendor/iluvatar/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+ILUVATAR backend for vllm-plugin-FL dispatch.
+"""
+
+from .iluvatar import IluvatarBackend
+
+__all__ = ["IluvatarBackend"]

vllm_fl/dispatch/backends/vendor/iluvatar/iluvatar.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+ILUVATAR backend implementation.
+
+This backend provides operator implementations for Iluvatar GPUs.
+Iluvatar uses a CUDA-compatible architecture.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Union
+
+import torch
+
+from vllm_fl.dispatch.backends.base import Backend
+
+
+class IluvatarBackend(Backend):
+    """
+    Iluvatar backend for operator implementations.
+
+    This backend uses Iluvatar libraries to provide high-performance
+    operator implementations for Iluvatar GPUs.
+    """
+
+    _available: Optional[bool] = None
+
+    @property
+    def name(self) -> str:
+        return "iluvatar"
+
+    @property
+    def vendor(self) -> Optional[str]:
+        return "iluvatar"
+
+    def is_available(self) -> bool:
+        """
+        Check if Iluvatar hardware and libraries are available.
+
+        This method uses the platform's vendor information to determine
+        if the device is an Iluvatar GPU.
+        """
+        if IluvatarBackend._available is None:
+            try:
+                from vllm.platforms import current_platform
+                # Iluvatar GPUs should be detected via vendor_name
+                if hasattr(current_platform, 'vendor_name') and current_platform.vendor_name == "iluvatar":
+                    IluvatarBackend._available = True
+                else:
+                    # Fallback: check if CUDA is available with iluvatar device
+                    if torch.cuda.is_available():
+                        # Try to detect Iluvatar GPU
+                        # Iluvatar GPUs typically expose CUDA-compatible interface
+                        # We can check device name if available
+                        device_name = torch.cuda.get_device_name(0)
+                        if "iluvatar" in device_name.lower():
+                            IluvatarBackend._available = True
+                        else:
+                            IluvatarBackend._available = False
+    
+                    else:
+                        IluvatarBackend._available = False
+            except Exception:
+                IluvatarBackend._available = False
+        return IluvatarBackend._available
+
+    # ==================== Operator Implementations ====================
+
+    def silu_and_mul(self, obj, x: torch.Tensor) -> torch.Tensor:
+        """
+        SiLU activation followed by element-wise multiplication.
+
+        Args:
+            obj: The calling obj (for interface consistency)
+            x: Input tensor of shape [..., 2*d]
+
+        Returns:
+            Output tensor of shape [..., d]
+        """
+        from .impl.activation import silu_and_mul_iluvatar
+
+        return silu_and_mul_iluvatar(obj, x)
+
+    def rms_norm(
+        self,
+        obj,
+        x: torch.Tensor,
+        residual: Optional[torch.Tensor] = None,
+    ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+        """
+        RMS normalization.
+
+        Args:
+            obj: The calling obj (e.g., RMSNorm layer)
+            x: Input tensor
+            residual: Optional residual tensor
+
+        Returns:
+            Normalized tensor, or tuple of (normalized, residual) if residual is provided
+        """
+        from .impl.normalization import rms_norm_iluvatar
+
+        return rms_norm_iluvatar(obj, x, residual)
+
+    def rotary_embedding(
+        self,
+        obj,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+        position_ids: torch.Tensor,
+        rotary_interleaved: bool = False,
+        inplace: bool = True,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Apply rotary position embedding.
+
+        Args:
+            obj: The calling obj (for interface consistency)
+            query: Query tensor
+            key: Key tensor
+            cos: Cosine cache
+            sin: Sine cache
+            position_ids: Position indices
+            rotary_interleaved: Whether to use interleaved rotary
+            inplace: Whether to modify tensors in-place
+
+        Returns:
+            Tuple of (embedded_query, embedded_key)
+        """
+        from .impl.rotary import rotary_embedding_iluvatar
+
+        return rotary_embedding_iluvatar(
+            obj,
+            query,
+            key,
+            cos,
+            sin,
+            position_ids,
+            rotary_interleaved=rotary_interleaved,
+            inplace=inplace,
+        )
+
+    def attention_backend(self, use_mla: bool = False) -> str:
+        """
+        Get the attention backend class path for Iluvatar.
+
+        Args:
+            use_mla: Whether to use Multi-head Latent Attention (MLA)
+
+        Returns:
+            Fully qualified class path string
+        """
+        from vllm.attention.backends.registry import AttentionBackendEnum
+
+        if use_mla:
+            return AttentionBackendEnum.FLASHMLA.get_path()
+
+        return AttentionBackendEnum.FLASH_ATTN.get_path()

vllm_fl/dispatch/backends/vendor/iluvatar/impl/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+ILUVATAR operator implementations.
+"""

vllm_fl/dispatch/backends/vendor/iluvatar/impl/activation.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+Reference activation operator implementations using PyTorch.
+"""
+
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+
+
+def silu_and_mul_iluvatar(obj, x: torch.Tensor) -> torch.Tensor:
+    """
+    SiLU activation followed by element-wise multiplication using PyTorch.
+
+    Args:
+        obj: The calling obj (for interface consistency)
+        x: Input tensor of shape [..., 2*d]
+
+    Returns:
+        Output tensor of shape [..., d]
+    """
+    d = x.shape[-1] // 2
+    x1, x2 = x[..., :d], x[..., d:]
+    return F.silu(x1) * x2

vllm_fl/dispatch/backends/vendor/iluvatar/impl/normalization.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+Reference normalization operator implementations using PyTorch.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Union
+
+import torch
+
+
+def rms_norm_iluvatar(
+    obj,
+    x: torch.Tensor,
+    residual: Optional[torch.Tensor] = None,
+) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+    """
+    RMS normalization using PyTorch.
+
+    Args:
+        obj: The calling obj (e.g., RMSNorm layer)
+        x: Input tensor
+        residual: Optional residual tensor
+
+    Returns:
+        Normalized tensor, or tuple of (normalized, residual) if residual is provided
+    """
+    # Get weight and epsilon from obj
+    weight = obj.weight
+    epsilon = obj.variance_epsilon
+
+    if residual is not None:
+        x = x + residual
+        residual = x
+
+    variance = x.pow(2).mean(-1, keepdim=True)
+    x = x * torch.rsqrt(variance + epsilon)
+    output = weight * x
+
+    if residual is not None:
+        return output, residual
+    return output

vllm_fl/dispatch/backends/vendor/iluvatar/impl/rotary.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""
+ILUVATAR rotary embedding operator implementations.
+
+NOTE: This is a template/stub implementation using PyTorch reference code.
+Replace with actual Iluvatar-optimized implementations when available.
+"""
+
+from __future__ import annotations
+
+import torch
+
+
+def rotary_embedding_iluvatar(
+    obj,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+    position_ids: torch.Tensor,
+    rotary_interleaved: bool = False,
+    inplace: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Apply rotary position embedding using Iluvatar.
+
+    This is a placeholder implementation using PyTorch reference code.
+    TODO: Replace with actual Iluvatar GPU optimized implementation.
+
+    Args:
+        obj: The calling obj (for interface consistency)
+        query: Query tensor [batch, num_heads, seq_len, head_dim] or [seq_len, num_heads, head_dim]
+        key: Key tensor [batch, num_heads, seq_len, head_dim] or [seq_len, num_heads, head_dim]
+        cos: Cosine cache [max_seq_len, rotary_dim] where rotary_dim = head_dim or head_dim // 2
+        sin: Sine cache [max_seq_len, rotary_dim] where rotary_dim = head_dim or head_dim // 2
+        position_ids: Position indices [batch, seq_len] or [seq_len]
+        rotary_interleaved: Whether to use interleaved rotary
+        inplace: Whether to modify tensors in-place (ignored in reference impl)
+
+    Returns:
+        Tuple of (embedded_query, embedded_key)
+    """
+    # Get cos/sin for the positions
+    # position_ids can be [batch, seq_len] or [seq_len]
+    if position_ids.dim() == 1:
+        # [seq_len] -> [seq_len, rotary_dim]
+        cos_selected = cos[position_ids]
+        sin_selected = sin[position_ids]
+    else:
+        # [batch, seq_len] -> [batch, seq_len, rotary_dim]
+        cos_selected = cos[position_ids]
+        sin_selected = sin[position_ids]
+
+    # Expand dimensions to match query/key shape
+    # query/key: [batch, num_heads, seq_len, head_dim] or [seq_len, num_heads, head_dim]
+    if query.dim() == 4:
+        # [batch, num_heads, seq_len, head_dim]
+        # cos_selected: [batch, seq_len, rotary_dim] -> [batch, 1, seq_len, rotary_dim]
+        cos_selected = cos_selected.unsqueeze(1)
+        sin_selected = sin_selected.unsqueeze(1)
+    elif query.dim() == 3:
+        # [seq_len, num_heads, head_dim]
+        # cos_selected: [seq_len, rotary_dim] -> [seq_len, 1, rotary_dim]
+        cos_selected = cos_selected.unsqueeze(1)
+        sin_selected = sin_selected.unsqueeze(1)
+
+    # Check if we need to repeat cos/sin to match head_dim
+    rotary_dim = cos_selected.shape[-1]
+    head_dim = query.shape[-1]
+
+    if rotary_dim != head_dim:
+        # cos/sin only covers half of head_dim, need to repeat
+        # This handles the case where rotary is only applied to part of the dimensions
+        cos_selected = torch.cat([cos_selected, cos_selected], dim=-1)
+        sin_selected = torch.cat([sin_selected, sin_selected], dim=-1)
+
+    def rotate_half(x):
+        """Rotates half the hidden dims of the input."""
+        x1 = x[..., : x.shape[-1] // 2]
+        x2 = x[..., x.shape[-1] // 2 :]
+        return torch.cat((-x2, x1), dim=-1)
+
+    if rotary_interleaved:
+        # Interleaved rotary
+        def rotate_interleaved(x):
+            x1 = x[..., ::2]
+            x2 = x[..., 1::2]
+            return torch.stack((-x2, x1), dim=-1).flatten(-2)
+
+        q_embed = (query * cos_selected) + (rotate_interleaved(query) * sin_selected)
+        k_embed = (key * cos_selected) + (rotate_interleaved(key) * sin_selected)
+    else:
+        # Standard rotary (neox style)
+        q_embed = (query * cos_selected) + (rotate_half(query) * sin_selected)
+        k_embed = (key * cos_selected) + (rotate_half(key) * sin_selected)
+
+    return q_embed, k_embed