Support gemma-4-12B unified (encoder-free) multimodal model

The gemma-4-12B "unified" variant is encoder-free: it consumes raw 48px
merged pixel patches (patch_dim 6912) and raw 640-sample waveform frames
directly, rather than the SigLIP image / Conformer log-mel audio contract
of the standard gemma-4 (E2B/E4B) model.

Register "gemma4_unified" as an MMM model type and route it to
Gemma4MultiModalProcessor with a unified flag that adjusts two things:

* Vision: no pixel-value trimming. The unified vision graph strips
  padding patches internally (position == -1), so the full padded
  (max_soft_tokens, 6912) pixel_values and position_ids are fed as-is
  instead of being trimmed to actual_soft_tokens * pooling^2.

* Audio: each 640-sample frame is exactly one audio soft token, so the
  audio-token count equals the number of frames (no Conv2d stride-2
  subsampling as in the Conformer speech encoder).

Everything else (prompt image/audio token expansion, embedding fusion,
per-layer-input decoder wiring) is shared with the standard gemma4 path.
The paired preprocessing ops live in onnxruntime-extensions
(Gemma4ImageTransform at patch_size=48/pooling=1, Gemma4UnifiedAudioFrames).

Signed-off-by: Justin Chu <justinchuby@users.noreply.github.com>

Author: justinchuby

src/models/gemma4_multimodal_processor.cpp

@@ -167,6 +167,14 @@ ProcessGemma4Prompt(const Generators::Tokenizer& tokenizer, const std::string& p
 
 Gemma4MultiModalProcessor::Gemma4MultiModalProcessor(Config& config, const SessionInfo& session_info)
     : pixel_values_type_{session_info.GetInputDataType(config.model.vision.inputs.pixel_values)} {
+  // gemma-4-12B "unified" is encoder-free: it consumes raw 48px merged pixel
+  // patches (patch_dim 6912) and raw 640-sample waveform frames directly. The
+  // preprocessing ops (Gemma4ImageTransform at patch_size=48/pooling=1 and
+  // Gemma4UnifiedAudioFrames) already emit the final contract, so — unlike the
+  // SigLIP/log-mel "gemma4" path — no pixel trimming or conv-subsampling of the
+  // audio-token count is applied.
+  unified_ = config.model.type == "gemma4_unified";
+
   // Query pixel_position_ids type (int32 or int64) if the vision model has this input
   if (session_info.HasInput(config.model.vision.inputs.pixel_position_ids)) {
     pixel_position_ids_type_ = session_info.GetInputDataType(config.model.vision.inputs.pixel_position_ids);
@@ -285,10 +293,18 @@ std::unique_ptr<NamedTensors> Gemma4MultiModalProcessor::Process(const Tokenizer
                              std::make_shared<Tensor>(std::move(mask)));
     }
 
-    // Compute audio_sizes: the speech encoder uses 2-stage Conv2d with stride=2 each
-    int64_t t_after_1 = (time_dim - 1) / 2 + 1;
-    int64_t t_after_2 = (t_after_1 - 1) / 2 + 1;
-    num_audio_tokens = t_after_2;
+    // Compute audio_sizes / audio-token count.
+    // Unified: each 640-sample frame is exactly one audio soft token, so the
+    // token count equals the number of frames (time_dim) — no subsampling.
+    // Standard gemma4: the Conformer speech encoder applies 2-stage Conv2d with
+    // stride=2 each, so the token count is reduced accordingly.
+    if (unified_) {
+      num_audio_tokens = time_dim;
+    } else {
+      int64_t t_after_1 = (time_dim - 1) / 2 + 1;
+      int64_t t_after_2 = (t_after_1 - 1) / 2 + 1;
+      num_audio_tokens = t_after_2;
+    }
     std::array<int64_t, 1> audio_sizes_shape = {1};
     auto audio_sizes = OrtValue::CreateTensor<int64_t>(allocator, audio_sizes_shape);
     audio_sizes->GetTensorMutableData<int64_t>()[0] = num_audio_tokens;
@@ -320,7 +336,7 @@ std::unique_ptr<NamedTensors> Gemma4MultiModalProcessor::Process(const Tokenizer
     const int64_t num_padded_patches = (pv_dims == 3) ? pv_shape[1] : pv_shape[0];
     const int64_t patch_dim = (pv_dims == 3) ? pv_shape[2] : pv_shape[1];
 
-    if (actual_patches < num_padded_patches) {
+    if (!unified_ && actual_patches < num_padded_patches) {
       // Trim: copy only the first actual_patches from the padded tensor.
       // The preprocessor outputs float32 data, so we trim using float32 strides,
       // then cast to the model's pixel_values type (float, fp16, or bf16).
@@ -365,7 +381,7 @@ std::unique_ptr<NamedTensors> Gemma4MultiModalProcessor::Process(const Tokenizer
       const int64_t num_padded_pos = (pos_dims == 3) ? pos_shape[1] : pos_shape[0];
       const int64_t pos_last_dim = (pos_dims == 3) ? pos_shape[2] : pos_shape[1];
 
-      if (actual_patches < num_padded_pos) {
+      if (!unified_ && actual_patches < num_padded_pos) {
         // Trim position_ids: for 3D, copy per-batch with correct stride.
         // Detect the element type from the vision model's input to handle both int32 and int64.
         const int64_t pos_batch = (pos_dims == 3) ? pos_shape[0] : 1;

src/models/gemma4_multimodal_processor.h

@@ -20,6 +20,7 @@ struct Gemma4MultiModalProcessor : Processor {
   ONNXTensorElementDataType audio_features_type_;
 
   bool has_speech_{false};
+  bool unified_{false};  // gemma-4-12B encoder-free "unified" variant
   size_t vision_soft_tokens_per_image_{260};
 };
 

src/models/model.cpp

@@ -974,6 +974,7 @@ MultiModalProcessor::MultiModalProcessor(Config& config, const SessionInfo& sess
           {"phi4mm", Processor::Create<PhiMultiModalProcessor>},
           {"gemma3", Processor::Create<GemmaImageProcessor>},
           {"gemma4", Processor::Create<Gemma4MultiModalProcessor>},
+          {"gemma4_unified", Processor::Create<Gemma4MultiModalProcessor>},
           {"mistral3", Processor::Create<Mistral3ImageProcessor>},
           {"fara", Processor::Create<QwenImageProcessor>},
           {"qwen2_5_vl", Processor::Create<QwenImageProcessor>},

src/models/model_type.h

@@ -63,7 +63,7 @@ struct ModelType {
 
   inline static bool IsMMM(const std::string& model_type) {
     // Multi-modal model (MMM)
-    static constexpr std::array<std::string_view, 2> MMM = {"gemma4", "phi4mm"};
+    static constexpr std::array<std::string_view, 3> MMM = {"gemma4", "gemma4_unified", "phi4mm"};
     return std::find(MMM.begin(), MMM.end(), model_type) != MMM.end();
   }