Fix config and smart_resize

Author: lkhl

src/transformers/models/videollama3/configuration_videollama3.py

@@ -4,9 +4,9 @@
 #             the file from the modular. If any change should be done, please apply the change to the
 #                          modular_videollama3.py file directly. One of our CI enforces this.
 #                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+from transformers.configuration_utils import PretrainedConfig
 
-from ...configuration_utils import PretrainedConfig
-from ..qwen2.configuration_qwen2 import Qwen2Config
+from ..auto import CONFIG_MAPPING, AutoConfig
 
 
 class Videollama3VisionConfig(PretrainedConfig):
@@ -83,7 +83,7 @@ class Videollama3Config(PretrainedConfig):
     """
 
     model_type = "videollama3"
-    sub_configs = {"vision_config": Videollama3VisionConfig, "text_config": Qwen2Config}
+    sub_configs = {"vision_config": Videollama3VisionConfig, "text_config": AutoConfig}
     keys_to_ignore_at_inference = ["past_key_values"]
 
     def __init__(
@@ -95,21 +95,21 @@ def __init__(
         use_token_compression=False,
         **kwargs,
     ):
-        super().__init__(**kwargs)
         if isinstance(vision_config, dict):
             self.vision_config = self.sub_configs["vision_config"](**vision_config)
         elif vision_config is None:
             self.vision_config = self.sub_configs["vision_config"]()
 
         if isinstance(text_config, dict):
-            self.text_config = self.sub_configs["text_config"](**text_config)
+            self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
         elif text_config is None:
-            # For BC use all kwargs to init `TextConfig`
-            self.text_config = self.sub_configs["text_config"](**kwargs)
+            self.text_config = CONFIG_MAPPING["qwen2"]()
 
         self.image_token_id = image_token_id
         self.video_token_id = video_token_id
         self.use_token_compression = use_token_compression
 
+        super().__init__(**kwargs)
+
 
 __all__ = ["Videollama3VisionConfig", "Videollama3Config"]

src/transformers/models/videollama3/image_processing_videollama3.py

@@ -1,9 +1,6 @@
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
-#           This file was automatically generated from src/transformers/models/videollama3/modular_videollama3.py.
-#               Do NOT edit this file manually as any edits will be overwritten by the generation of
-#             the file from the modular. If any change should be done, please apply the change to the
-#                          modular_videollama3.py file directly. One of our CI enforces this.
-#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+"""Image processor class for VideoLLaMA3."""
+
+import math
 from typing import Optional, Union
 
 import numpy as np
@@ -26,15 +23,43 @@
 from ...image_processing_utils import BaseImageProcessor
 from ...image_transforms import convert_to_rgb, resize, to_channel_dimension_format
 from ...image_utils import infer_channel_dimension_format, is_scaled_image, make_list_of_images, to_numpy_array
-from .image_processing_videollama3_fast import smart_resize
 
 
 logger = logging.get_logger(__name__)
 
 
+def smart_resize(
+    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
+):
+    """Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+
+    """
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = max(factor, math.floor(height / beta / factor) * factor)
+        w_bar = max(factor, math.floor(width / beta / factor) * factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+    return h_bar, w_bar
+
+
 class Videollama3ImageProcessor(BaseImageProcessor):
     r"""
-    Constructs a Qwen2-VL image processor that dynamically resizes images based on the original images.
+    Constructs a VideoLLaMA3 image processor that dynamically resizes images based on the original images.
 
     Args:
         do_resize (`bool`, *optional*, defaults to `True`):

src/transformers/models/videollama3/image_processing_videollama3_fast.py

@@ -11,24 +11,20 @@
 import torch.nn.functional as F
 
 from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_processing_utils_fast import DefaultFastImageProcessorKwargs
 from transformers.image_utils import (
     IMAGENET_STANDARD_MEAN,
     IMAGENET_STANDARD_STD,
     ChannelDimension,
     ImageInput,
-    PILImageResampling,
     SizeDict,
 )
 from transformers.processing_utils import Unpack
 from transformers.utils import TensorType, auto_docstring, logging
 from transformers.video_utils import VideoInput, make_batched_videos
 
-from ...image_processing_utils_fast import (
-    BaseImageProcessorFast,
-    DefaultFastImageProcessorKwargs,
-    group_images_by_shape,
-    reorder_images,
-)
+from ...image_processing_utils_fast import BaseImageProcessorFast, group_images_by_shape, reorder_images
+from ...image_utils import PILImageResampling
 
 
 logger = logging.get_logger(__name__)