Skip to content

Commit 47b6e17

Browse files
PanAndyPanAndy
committed
deploy: 7f9d4d3 
1 parent 09ec3e4 commit 47b6e17

289 files changed

Lines changed: 2586 additions & 627 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎404.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
<link rel="preconnect" href="https://www.googletagmanager.com">
88
<script async src="https://www.googletagmanager.com/gtag/js?id=G-D6R4GXHVFP"></script>
99
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-D6R4GXHVFP",{anonymize_ip:!0})</script><link rel="stylesheet" href="/ROLL/assets/css/styles.3b87d9d1.css">
10-
<script src="/ROLL/assets/js/runtime~main.f0cc21be.js" defer="defer"></script>
11-
<script src="/ROLL/assets/js/main.27232a17.js" defer="defer"></script>
10+
<script src="/ROLL/assets/js/runtime~main.96bfdd95.js" defer="defer"></script>
11+
<script src="/ROLL/assets/js/main.41043644.js" defer="defer"></script>
1212
</head>
1313
<body>
1414
<svg style="display: none;"><defs>

‎assets/files/qwen3_moe_patch-6214c28fd60f14f33460a8ac151b63fb.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import torch
2+
import torch.nn.functional as F
3+
from torch.distributed.tensor import DTensor
4+
5+
6+
# force each expert to participate in computation graph so FSDP could gather all expert outputs
7+
def qwen3_moe_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
8+
batch_size, sequence_length, hidden_dim = hidden_states.shape
9+
hidden_states = hidden_states.view(-1, hidden_dim)
10+
router_logits = self.gate(hidden_states)
11+
12+
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
13+
routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
14+
if self.norm_topk_prob:
15+
routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
16+
routing_weights = routing_weights.to(hidden_states.dtype)
17+
18+
final_hidden_states = torch.zeros(
19+
(batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
20+
)
21+
22+
expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
23+
24+
for expert_idx in range(self.num_experts):
25+
expert_layer = self.experts[expert_idx]
26+
idx, top_x = torch.where(expert_mask[expert_idx])
27+
28+
if top_x.numel() > 0:
29+
current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
30+
current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]
31+
final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
32+
else:
33+
dummy_output = expert_layer(hidden_states[:1]) * 0.0
34+
final_hidden_states[:1] = final_hidden_states[:1] + dummy_output
35+
36+
final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
37+
return final_hidden_states, router_logits
38+
39+
40+
def _iter_convert_fsdps_moe_weights(named_params):
41+
"""
42+
Lazily convert FSDP MoE weights from FSDP format to HF/vLLM expected format.
43+
44+
Yields (name, tensor, full_tensor_size) one at a time so that the caller can
45+
control memory by batching into buffers. For expert parameters stored as
46+
DTensors (EP mode), full_tensor() is called here to gather the shards, but
47+
because this is a generator the full tensor is only materialized when the
48+
caller consumes the item.
49+
50+
FSDP format:
51+
- layers.0.mlp.experts.gate_up_proj: [num_experts, 2 * intermediate_dim, hidden_dim]
52+
- layers.0.mlp.experts.down_proj: [num_experts, hidden_dim, intermediate_dim]
53+
54+
Expected format:
55+
- layers.0.mlp.experts.0.gate_proj.weight: [intermediate_dim, hidden_dim]
56+
- layers.0.mlp.experts.0.up_proj.weight: [intermediate_dim, hidden_dim]
57+
- layers.0.mlp.experts.0.down_proj.weight: [hidden_dim, intermediate_dim]
58+
"""
59+
for name, param in named_params:
60+
if "experts.gate_up_proj" in name:
61+
if isinstance(param, DTensor):
62+
param = param.full_tensor()
63+
64+
gate_proj, up_proj = torch.chunk(param, 2, dim=1)
65+
num_experts = gate_proj.shape[0]
66+
67+
for expert_idx in range(num_experts):
68+
gate_name = name.replace("gate_up_proj", f"{expert_idx}.gate_proj.weight")
69+
up_name = name.replace("gate_up_proj", f"{expert_idx}.up_proj.weight")
70+
yield gate_name, gate_proj[expert_idx]
71+
yield up_name, up_proj[expert_idx]
72+
73+
# Allow GC to reclaim the full tensor once all slices are yielded
74+
del param, gate_proj, up_proj
75+
76+
elif "experts.down_proj" in name:
77+
if isinstance(param, DTensor):
78+
param = param.full_tensor()
79+
80+
num_experts = param.shape[0]
81+
82+
for expert_idx in range(num_experts):
83+
down_name = name.replace("down_proj", f"{expert_idx}.down_proj.weight")
84+
yield down_name, param[expert_idx]
85+
86+
del param
87+
88+
else:
89+
yield name, param

‎assets/js/0d7c724a.89ad15f7.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/js/0f6b141e.08780cf0.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/js/105abd28.ec82847a.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/js/1184df6d.b8bedcc3.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/js/1329d571.bdd645f4.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)