feat: optional deps handling for ONNX runtime

aminya · aminya · commit 4f5c268717a7 · 2026-07-11T05:07:23.000-07:00
This makes it possible to load docling with ONNX without needing torch/transformer deps:

- Moved the module-level torch and transformers imports down into the load methods.
- Fixed device selection to gracefully fall back to CPU when torch isn't around
- Wrapped engine registration in a guarded loader. If an optional backend (like TableFormer v2) is missing, we just skip it now instead of crashing the factory setup.
- Added a numpy image processor fallback for ONNX. This handles the RT-DETR layout presets without needing torch. Output is identical to the transformers version.

Signed-off-by: Amin Ya &lt;aminyahyaabadi74@gmail.com&gt;
diff --git a/docling/models/inference_engines/common/hf_vision_base.py b/docling/models/inference_engines/common/hf_vision_base.py
@@ -2,12 +2,14 @@
 
 from __future__ import annotations
 
+import json
 import logging
 from numbers import Integral, Real
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
+from PIL import Image
 
 from docling.datamodel.accelerator_options import AcceleratorOptions
 from docling.models.inference_engines.vlm._utils import resolve_model_artifacts_path
@@ -21,6 +23,82 @@
 _log = logging.getLogger(__name__)
 
 
+class NumpyImageProcessor:
+    """Dependency-light, torch-free stand-in for a transformers image processor.
+
+    Reproduces the resize/rescale/normalize preprocessing described by a
+    ``preprocessor_config.json`` using only numpy + Pillow and returns
+    ``{"pixel_values": np.ndarray}`` shaped ``(N, C, H, W)``. It exists so the ONNX
+    Runtime object-detection engine can preprocess inputs in environments without
+    torch/torchvision: transformers >= 5 gates its image-processor classes behind
+    those backends, so ``AutoImageProcessor.from_pretrained`` is unavailable there.
+
+    For the default docling layout presets (RT-DETR: resize to a fixed square, no
+    rescale/normalize) the output is byte-identical to the transformers slow image
+    processor.
+    """
+
+    # PIL resampling filter for the integer codes stored by transformers/PIL.
+    _RESAMPLE = {
+        0: Image.Resampling.NEAREST,
+        1: Image.Resampling.LANCZOS,
+        2: Image.Resampling.BILINEAR,
+        3: Image.Resampling.BICUBIC,
+        4: Image.Resampling.BOX,
+        5: Image.Resampling.HAMMING,
+    }
+
+    def __init__(self, config: Dict[str, Any]) -> None:
+        self.size: Optional[Dict[str, int]] = config.get("size")
+        self.do_resize: bool = bool(config.get("do_resize", False))
+        self.do_rescale: bool = bool(config.get("do_rescale", False))
+        self.do_normalize: bool = bool(config.get("do_normalize", False))
+        self.rescale_factor: float = float(config.get("rescale_factor", 1.0 / 255.0))
+        self.image_mean: Optional[List[float]] = config.get("image_mean")
+        self.image_std: Optional[List[float]] = config.get("image_std")
+        self.resample = self._RESAMPLE.get(
+            int(config.get("resample", 2)), Image.Resampling.BILINEAR
+        )
+
+    @classmethod
+    def from_config_file(cls, path: Path) -> "NumpyImageProcessor":
+        with path.open(encoding="utf-8") as handle:
+            return cls(json.load(handle))
+
+    def _target_hw(self) -> Optional[Tuple[int, int]]:
+        size = self.size or {}
+        if "height" in size and "width" in size:
+            return int(size["height"]), int(size["width"])
+        edge = size.get("shortest_edge") or size.get("longest_edge")
+        if edge is not None:
+            return int(edge), int(edge)
+        return None
+
+    def __call__(
+        self, images: Any, return_tensors: str = "np", **_: Any
+    ) -> Dict[str, np.ndarray]:
+        if not isinstance(images, (list, tuple)):
+            images = [images]
+        target = self._target_hw()
+        frames: List[np.ndarray] = []
+        for image in images:
+            pil = image.convert("RGB")
+            if self.do_resize and target is not None:
+                pil = pil.resize((target[1], target[0]), resample=self.resample)
+            frames.append(np.asarray(pil))
+        batch = np.stack(frames, axis=0)  # (N, H, W, C), uint8
+        if self.do_rescale or self.do_normalize:
+            batch = batch.astype(np.float32)
+        if self.do_rescale:
+            batch = batch * self.rescale_factor
+        if self.do_normalize and self.image_mean is not None and self.image_std is not None:
+            mean = np.asarray(self.image_mean, dtype=np.float32)
+            std = np.asarray(self.image_std, dtype=np.float32)
+            batch = (batch - mean) / std
+        pixel_values = np.ascontiguousarray(batch.transpose(0, 3, 1, 2))
+        return {"pixel_values": pixel_values}
+
+
 class HfVisionModelMixin(HuggingFaceModelDownloadMixin):
     """Shared utility mixin for HF vision model loading and label conversion."""
 
@@ -85,6 +163,17 @@ def _load_preprocessor(self, model_folder: Path) -> BaseImageProcessor:
 
             _log.debug("Loading image processor from %s", model_folder)
             return AutoImageProcessor.from_pretrained(str(model_folder))
+        except ImportError:
+            # transformers >= 5 gates its image-processor classes behind the
+            # torch/torchvision backends. In a torch-free environment the ONNX
+            # object-detection engine still needs preprocessing, so fall back to a
+            # dependency-light numpy processor built from preprocessor_config.json.
+            _log.info(
+                "transformers image processor unavailable (torch-free environment); "
+                "using numpy preprocessing fallback for %s",
+                model_folder,
+            )
+            return NumpyImageProcessor.from_config_file(preprocessor_config)
         except Exception as exc:
             raise RuntimeError(
                 f"Failed to load image processor from {model_folder}: {exc}"
diff --git a/docling/models/plugins/defaults.py b/docling/models/plugins/defaults.py
@@ -1,4 +1,21 @@
-def ocr_engines():
+from __future__ import annotations
+
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    # Experimental table crops layout models
+    from docling.experimental.models.table_crops_layout_model import (
+        TableCropsLayoutModel,
+    )
+
+    # Layout models
+    from docling.models.stages.layout.layout_model import LayoutModel
+    from docling.models.stages.layout.layout_object_detection_model import (
+        LayoutObjectDetectionModel,
+    )
+
+    # OCR models
     from docling.models.stages.ocr.auto_ocr_model import OcrAutoModel
     from docling.models.stages.ocr.easyocr_model import EasyOcrModel
     from docling.models.stages.ocr.kserve_v2_ocr_model import KserveV2OcrModel
@@ -7,60 +24,19 @@ def ocr_engines():
     from docling.models.stages.ocr.rapid_ocr_model import RapidOcrModel
     from docling.models.stages.ocr.tesseract_ocr_cli_model import TesseractOcrCliModel
     from docling.models.stages.ocr.tesseract_ocr_model import TesseractOcrModel
-
-    return {
-        "ocr_engines": [
-            OcrAutoModel,
-            EasyOcrModel,
-            KserveV2OcrModel,
-            NemotronOcrModel,
-            OcrMacModel,
-            RapidOcrModel,
-            TesseractOcrModel,
-            TesseractOcrCliModel,
-        ]
-    }
-
-
-def picture_description():
     from docling.models.stages.picture_description.picture_description_api_model import (
         PictureDescriptionApiModel,
     )
+
+    # Picture description models
     from docling.models.stages.picture_description.picture_description_vlm_engine_model import (
         PictureDescriptionVlmEngineModel,
     )
     from docling.models.stages.picture_description.picture_description_vlm_model import (
         PictureDescriptionVlmModel,
     )
 
-    return {
-        "picture_description": [
-            PictureDescriptionVlmEngineModel,  # New engine-based (preferred)
-            PictureDescriptionVlmModel,  # Legacy direct transformers
-            PictureDescriptionApiModel,  # API-based
-        ]
-    }
-
-
-def layout_engines():
-    from docling.experimental.models.table_crops_layout_model import (
-        TableCropsLayoutModel,
-    )
-    from docling.models.stages.layout.layout_model import LayoutModel
-    from docling.models.stages.layout.layout_object_detection_model import (
-        LayoutObjectDetectionModel,
-    )
-
-    return {
-        "layout_engines": [
-            LayoutObjectDetectionModel,
-            LayoutModel,
-            TableCropsLayoutModel,
-        ]
-    }
-
-
-def table_structure_engines():
+    # Table structure models
     from docling.models.stages.table_structure.table_structure_model import (
         TableStructureModel,
     )
@@ -71,10 +47,137 @@ def table_structure_engines():
         TableStructureModelV2,
     )
 
-    return {
-        "table_structure_engines": [
+
+def ocr_engines():
+    engines: list[
+        type[OcrAutoModel]
+        | type[EasyOcrModel]
+        | type[KserveV2OcrModel]
+        | type[NemotronOcrModel]
+        | type[OcrMacModel]
+        | type[RapidOcrModel]
+        | type[TesseractOcrModel]
+        | type[TesseractOcrCliModel]
+    ] = []
+
+    with suppress(ImportError):
+        from docling.models.stages.ocr.auto_ocr_model import OcrAutoModel
+
+        engines.append(OcrAutoModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.easyocr_model import EasyOcrModel
+
+        engines.append(EasyOcrModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.kserve_v2_ocr_model import KserveV2OcrModel
+
+        engines.append(KserveV2OcrModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.nemotron_ocr_model import NemotronOcrModel
+
+        engines.append(NemotronOcrModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.ocr_mac_model import OcrMacModel
+
+        engines.append(OcrMacModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.rapid_ocr_model import RapidOcrModel
+
+        engines.append(RapidOcrModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.tesseract_ocr_model import TesseractOcrModel
+
+        engines.append(TesseractOcrModel)
+    with suppress(ImportError):
+        from docling.models.stages.ocr.tesseract_ocr_cli_model import (
+            TesseractOcrCliModel,
+        )
+
+        engines.append(TesseractOcrCliModel)
+
+    return {"ocr_engines": engines}
+
+
+def picture_description():
+    engines: list[
+        type[PictureDescriptionVlmEngineModel]
+        | type[PictureDescriptionVlmModel]
+        | type[PictureDescriptionApiModel]
+    ] = []
+
+    with suppress(ImportError):
+        from docling.models.stages.picture_description.picture_description_vlm_engine_model import (
+            PictureDescriptionVlmEngineModel,
+        )
+
+        engines.append(PictureDescriptionVlmEngineModel)  # New engine-based (preferred)
+    with suppress(ImportError):
+        from docling.models.stages.picture_description.picture_description_vlm_model import (
+            PictureDescriptionVlmModel,
+        )
+
+        engines.append(PictureDescriptionVlmModel)  # Legacy direct transformers
+    with suppress(ImportError):
+        from docling.models.stages.picture_description.picture_description_api_model import (
+            PictureDescriptionApiModel,
+        )
+
+        engines.append(PictureDescriptionApiModel)  # API-based
+
+    return {"picture_description": engines}
+
+
+def layout_engines():
+    engines: list[
+        type[LayoutObjectDetectionModel]
+        | type[LayoutModel]
+        | type[TableCropsLayoutModel]
+    ] = []
+
+    with suppress(ImportError):
+        from docling.models.stages.layout.layout_object_detection_model import (
+            LayoutObjectDetectionModel,
+        )
+
+        engines.append(LayoutObjectDetectionModel)
+    with suppress(ImportError):
+        from docling.models.stages.layout.layout_model import LayoutModel
+
+        engines.append(LayoutModel)
+    with suppress(ImportError):
+        from docling.experimental.models.table_crops_layout_model import (
+            TableCropsLayoutModel,
+        )
+
+        engines.append(TableCropsLayoutModel)
+
+    return {"layout_engines": engines}
+
+
+def table_structure_engines():
+    engines: list[
+        type[TableStructureModel]
+        | type[TableStructureModelV2]
+        | type[GraniteVisionTableStructureModel]
+    ] = []
+
+    with suppress(ImportError):
+        from docling.models.stages.table_structure.table_structure_model import (
             TableStructureModel,
+        )
+
+        engines.append(TableStructureModel)
+    with suppress(ImportError):
+        from docling.models.stages.table_structure.table_structure_model_v2 import (
             TableStructureModelV2,
+        )
+
+        engines.append(TableStructureModelV2)
+    with suppress(ImportError):
+        from docling.models.stages.table_structure.table_structure_model_granite_vision import (
             GraniteVisionTableStructureModel,
-        ]
-    }
+        )
+
+        engines.append(GraniteVisionTableStructureModel)
+
+    return {"table_structure_engines": engines}
diff --git a/docling/models/stages/chart_extraction/granite_vision.py b/docling/models/stages/chart_extraction/granite_vision.py
@@ -8,7 +8,6 @@
 from typing import Any, List, Literal, Optional, cast
 
 import pandas as pd
-import torch
 from docling_core.types.doc import (
     CodeLanguageLabel,
     DescriptionMetaField,
@@ -24,7 +23,6 @@
 from docling_core.types.doc.document import CodeMetaField
 from PIL import Image
 from pydantic import BaseModel
-from transformers import AutoModelForImageTextToText, AutoProcessor
 
 from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions
 from docling.datamodel.base_models import ItemAndImageEnrichmentElement
@@ -200,6 +198,8 @@ class ChartExtractionModelGraniteVision(_BaseChartExtractionModelGraniteVision):
     _model_repo_revision = "6e1fbaae4604ecc85f4f371416d82154ca49ad67"
 
     def _load_model(self, artifacts_path: Path) -> None:
+        from transformers import AutoModelForImageTextToText, AutoProcessor
+
         self._processor = AutoProcessor.from_pretrained(
             artifacts_path,
             trust_remote_code=True,
@@ -346,6 +346,9 @@ class ChartExtractionModelGraniteVisionV4(_BaseChartExtractionModelGraniteVision
     _model_repo_revision = "dd48e97503de471803850df70843cf9eb5da8712"
 
     def _load_model(self, artifacts_path: Path) -> None:
+        import torch
+        from transformers import AutoModelForImageTextToText, AutoProcessor
+
         with warnings.catch_warnings():
             warnings.filterwarnings(
                 "ignore",
diff --git a/docling/utils/accelerator_utils.py b/docling/utils/accelerator_utils.py
