(fix): Fix the position_ids issue for qwen_vl when uses megatron

Author: ShareLer

mcore_adapter/src/mcore_adapter/models/qwen2_5_vl/modeling_qwen2_5_vl.py

@@ -324,7 +324,19 @@ def forward(
         **kwargs,
     ) -> "torch.Tensor":
         force_vit_image = kwargs.pop("force_vit_image", False)
-        force_vit_video = kwargs.pop("force_vit_video", False)
+        force_vit_video = kwargs.pop("force_vit_video", False)       
+        
+        if position_ids is not None:
+            expected_shape = (3, input_ids.shape[0], input_ids.shape[1])  # (3, batch, seq_len)
+            if position_ids.shape != expected_shape:
+                if position_ids.shape == (input_ids.shape[0], input_ids.shape[1]):
+                    position_ids, _ = self.get_rope_index(
+                        input_ids, image_grid_thw, video_grid_thw, second_per_grid_ts, attention_mask
+                    )
+                else:
+                    raise ValueError(f"Unexpected position_ids shape: {position_ids.shape}, "
+                                     f"expected: {expected_shape} or {(input_ids.shape[0], input_ids.shape[1])}")
+
         if position_ids is None and input_ids is not None:
             position_ids, _ = self.get_rope_index(
                 input_ids, image_grid_thw, video_grid_thw, second_per_grid_ts, attention_mask

mcore_adapter/src/mcore_adapter/models/qwen2_vl/modeling_qwen2_vl.py

@@ -303,6 +303,16 @@ def forward(
     ) -> "torch.Tensor":
         force_vit_image = kwargs.pop("force_vit_image", False)
         force_vit_video = kwargs.pop("force_vit_video", False)
+
+        if position_ids is not None:
+            expected_shape = (3, input_ids.shape[0], input_ids.shape[1])  # (3, batch, seq_len)
+            if position_ids.shape != expected_shape:
+                if position_ids.shape == (input_ids.shape[0], input_ids.shape[1]):
+                    position_ids, _ = self.get_rope_index(input_ids, image_grid_thw, video_grid_thw)
+                else:
+                    raise ValueError(f"Unexpected position_ids shape: {position_ids.shape}, "
+                                     f"expected: {expected_shape} or {(input_ids.shape[0], input_ids.shape[1])}")
+
         if position_ids is None and input_ids is not None:
             position_ids, _ = self.get_rope_index(input_ids, image_grid_thw, video_grid_thw)