Fix

Author: lkhl

src/transformers/models/videollama3/image_processing_videollama3.py

@@ -445,12 +445,6 @@ def preprocess(
                 }
             )
 
-        if videos is not None:
-            raise ValueError(
-                "`Videollama3ImageProcessor` works only with image inputs and doesn't process videos anymore. "
-                "Your videos should be forwarded to `Videollama3VideoProcessor`. "
-            )
-
         return BatchFeature(data=data, tensor_type=return_tensors)
 
     def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):

src/transformers/models/videollama3/modeling_videollama3.py

@@ -409,7 +409,6 @@ class Videollama3PreTrainedModel(PreTrainedModel):
 class Videollama3VisionModel(Videollama3PreTrainedModel):
     config: Videollama3VisionConfig
     _no_split_modules = ["Videollama3VisionEncoderLayer"]
-    # _can_compile_fullgraph = False
     _can_record_outputs = {
         "hidden_states": Videollama3VisionEncoderLayer,
         "attentions": Videollama3VisionAttention,
@@ -514,7 +513,6 @@ def forward(
             hidden_states,
             cu_seqlens=cu_seqlens,
             position_embeddings=position_embeddings,
-            return_dict=True,
             **kwargs,
         )
 
@@ -1083,8 +1081,12 @@ def _repeat_interleave_samples(x, lengths, repeat_times):
                         dict_to_expand[key], lengths=lengths, repeat_times=expand_size
                     )
                 elif key == "video_compression_mask":
-                    samples = torch.split(video_compression_mask, list(video_nums))
-                    lengths = [mask.sum() for mask in samples]
+                    samples = torch.split(video_grid_thw, list(video_nums))
+                    merge_sizes = torch.split(video_merge_sizes, list(video_nums))
+                    lengths = [
+                        (torch.prod(sample, dim=1) // merge_size**2).sum()
+                        for sample, merge_size in zip(samples, merge_sizes)
+                    ]
                     dict_to_expand[key] = _repeat_interleave_samples(
                         dict_to_expand[key], lengths=lengths, repeat_times=expand_size
                     )

src/transformers/models/videollama3/modular_videollama3.py

@@ -481,7 +481,6 @@ class Videollama3PreTrainedModel(Qwen2VLPreTrainedModel):
 class Videollama3VisionModel(Videollama3PreTrainedModel):
     config: Videollama3VisionConfig
     _no_split_modules = ["Videollama3VisionEncoderLayer"]
-    # _can_compile_fullgraph = False
     _can_record_outputs = {
         "hidden_states": Videollama3VisionEncoderLayer,
         "attentions": Videollama3VisionAttention,
@@ -586,7 +585,6 @@ def forward(
             hidden_states,
             cu_seqlens=cu_seqlens,
             position_embeddings=position_embeddings,
-            return_dict=True,
             **kwargs,
         )
 
@@ -1082,8 +1080,12 @@ def _repeat_interleave_samples(x, lengths, repeat_times):
                         dict_to_expand[key], lengths=lengths, repeat_times=expand_size
                     )
                 elif key == "video_compression_mask":
-                    samples = torch.split(video_compression_mask, list(video_nums))
-                    lengths = [mask.sum() for mask in samples]
+                    samples = torch.split(video_grid_thw, list(video_nums))
+                    merge_sizes = torch.split(video_merge_sizes, list(video_nums))
+                    lengths = [
+                        (torch.prod(sample, dim=1) // merge_size**2).sum()
+                        for sample, merge_size in zip(samples, merge_sizes)
+                    ]
                     dict_to_expand[key] = _repeat_interleave_samples(
                         dict_to_expand[key], lengths=lengths, repeat_times=expand_size
                     )