Skip to content

Commit fd2e247

Browse files
kkkky123kkkky123
committed
deploy: 777dad6 
1 parent 13ae495 commit fd2e247

279 files changed

Lines changed: 3700 additions & 440 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎404.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
<link rel="preconnect" href="https://www.googletagmanager.com">
88
<script async src="https://www.googletagmanager.com/gtag/js?id=G-D6R4GXHVFP"></script>
99
<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-D6R4GXHVFP",{anonymize_ip:!0})</script><link rel="stylesheet" href="/ROLL/assets/css/styles.4016bf18.css">
10-
<script src="/ROLL/assets/js/runtime~main.75553a17.js" defer="defer"></script>
11-
<script src="/ROLL/assets/js/main.bc7b6a14.js" defer="defer"></script>
10+
<script src="/ROLL/assets/js/runtime~main.f1a23ec3.js" defer="defer"></script>
11+
<script src="/ROLL/assets/js/main.d5b57526.js" defer="defer"></script>
1212
</head>
1313
<body class="navigation-with-keyboard">
1414
<svg style="display: none;"><defs>

‎assets/files/qwen3_moe_patch-83fd6b6c7a691229bf71364630c23b5f.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import torch
2+
import torch.nn.functional as F
3+
4+
5+
# force each expert to participate in computation graph so FSDP could gather all expert outputs
6+
def qwen3_moe_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
7+
batch_size, sequence_length, hidden_dim = hidden_states.shape
8+
hidden_states = hidden_states.view(-1, hidden_dim)
9+
router_logits = self.gate(hidden_states)
10+
11+
routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
12+
routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
13+
if self.norm_topk_prob:
14+
routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
15+
routing_weights = routing_weights.to(hidden_states.dtype)
16+
17+
final_hidden_states = torch.zeros(
18+
(batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
19+
)
20+
21+
expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
22+
23+
for expert_idx in range(self.num_experts):
24+
expert_layer = self.experts[expert_idx]
25+
idx, top_x = torch.where(expert_mask[expert_idx])
26+
27+
if top_x.numel() > 0:
28+
current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
29+
current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None]
30+
final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
31+
else:
32+
dummy_output = expert_layer(hidden_states[:1]) * 0.0
33+
final_hidden_states[:1] = final_hidden_states[:1] + dummy_output
34+
35+
final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
36+
return final_hidden_states, router_logits

‎assets/js/053bfa5a.a3ebb216.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)