Skip to content

Commit dbfe3be

Browse files
cyber-pioneerceci3
andauthored
Fix is_monolithic missing when setting VLLM_FL_PREFER_ENABLED=0 (#252)
### PR Category Core ### PR Type Bug Fixes ### Description fix is_monolithic missing when setting VLLM_FL_PREFER_ENABLED=0 ``` Root cause chain: is_out_of_tree() _ True (PlatformFL) _ select_unquantized_moe_backend() returns (OOT, None) _ experts_cls = None _ code calls experts_cls.is_monolithic _ AttributeError: NoneType has no attribute 'is_monolithic' The upstream function has this logic: if current_platform.is_out_of_tree(): return UnquantizedMoeBackend.OOT, None # _ expects OOT PluggableLayer to handle it When FusedMoEFL is registered, vLLM never reaches this code path (it uses FusedMoEFL's own method). But when FusedMoEFL is not registered (disabled/blacklisted), vLLM falls through to the built-in FusedMoE, which calls select_unquantized_moe_backend(). Since PlatformFL still reports is_out_of_tree() _ True, it returns (OOT, None) and blows up. ``` --------- Co-authored-by: ceci3 <ceci3@users.noreply.github.com>
1 parent 9f2d78f commit dbfe3be

2 files changed

Lines changed: 34 additions & 0 deletions

File tree

vllm_fl/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,9 @@ def register_router():
128128
# fused_moe import chain triggers cutlass_scaled_mm_supports_fp8 on MUSA
129129
if current_platform.device_type == "musa":
130130
return
131+
from vllm_fl.utils import is_oot_enabled
132+
if not is_oot_enabled():
133+
return
131134
from vllm_fl.ops.fused_moe.router import replace_router_with_fl
132135
replace_router_with_fl()
133136

vllm_fl/ops/custom_ops.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,26 @@
2929
),
3030
}
3131

32+
def _patch_unquantized_moe_oracle() -> None:
33+
"""
34+
Monkey-patch the upstream select_unquantized_moe_backend so it does not
35+
short-circuit to (OOT, None) on our platform. Instead it falls through
36+
to the normal CUDA/ROCm backend priority selection — the same logic that
37+
select_unquantized_moe_backend_oot uses.
38+
39+
This is needed when FusedMoEFL is NOT registered (PREFER_ENABLED=0 or
40+
fused_moe blacklisted): without the patch, the in-tree UnquantizedFusedMoEMethod
41+
would get (OOT, None), skip _setup_kernel, and crash at inference time.
42+
"""
43+
import vllm.model_executor.layers.fused_moe.oracle.unquantized as _oracle_mod
44+
from vllm_fl.ops.fused_moe.fused_moe_utils import select_unquantized_moe_backend_oot
45+
_oracle_mod.select_unquantized_moe_backend = select_unquantized_moe_backend_oot
46+
# Also patch the import in unquantized_fused_moe_method module
47+
import vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method as _method_mod
48+
_method_mod.select_unquantized_moe_backend = select_unquantized_moe_backend_oot
49+
logger.info("Patched select_unquantized_moe_backend to bypass OOT short-circuit")
50+
51+
3252
def register_oot_ops(whitelist: Optional[List[str]] = None) -> None:
3353
"""
3454
Register OOT (out-of-tree) custom operators.
@@ -40,11 +60,17 @@ def register_oot_ops(whitelist: Optional[List[str]] = None) -> None:
4060
4161
Operators in VLLM_FL_OOT_BLACKLIST or platform config oot_blacklist
4262
will be excluded from registration.
63+
64+
When fused_moe is not registered (PREFER_ENABLED=0 or blacklisted),
65+
the upstream select_unquantized_moe_backend oracle is monkey-patched
66+
so it picks native CUDA backends instead of returning (OOT, None).
4367
"""
4468
from vllm_fl.utils import get_oot_blacklist, get_oot_whitelist, is_oot_enabled, use_flaggems_op
4569

4670
# Check if OOT registration is enabled
4771
if not is_oot_enabled():
72+
# Patch the upstream oracle so in-tree FusedMoE works on this platform.
73+
_patch_unquantized_moe_oracle()
4874
return
4975

5076
# Get blacklist (from env var or platform config)
@@ -62,6 +88,11 @@ def register_oot_ops(whitelist: Optional[List[str]] = None) -> None:
6288
# Apply blacklist
6389
ops_to_register = [op for op in ops_to_register if op not in blacklist]
6490

91+
# If fused_moe is excluded (blacklisted or not in whitelist), patch the
92+
# upstream oracle so the in-tree FusedMoE doesn't crash on OOT platforms.
93+
if "fused_moe" not in ops_to_register:
94+
_patch_unquantized_moe_oracle()
95+
6596
for op_name in ops_to_register:
6697
if op_name not in OOT_OPS:
6798
logger.warning(f"OOT op '{op_name}' not found in OOT_OPS, skipping.")

0 commit comments

Comments
 (0)