Add Moonshine streaming ASR model support

examples/python/nemotron_speech.py → examples/python/streaming_asr.py

@@ -1,5 +1,18 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+"""Streaming ASR example using the onnxruntime-genai StreamingProcessor API.
+
+Drives any streaming encoder/decoder ASR model exposed through onnxruntime-genai:
+  * NVIDIA Nemotron streaming RNN-T (`nemotron_speech`, multilingual)
+  * Moonshine streaming encoder-decoder (`streaming_enc_dec_asr`, English only)
+
+The runtime pipeline is identical for both: the StreamingProcessor factory
+dispatches per `model.type`, and the Generator routes both through the
+TransducerState path under the hood.
+
+The `--language` flag and the LANG_TO_ID table below are Nemotron-specific;
+Moonshine has no lang_id runtime knob, so leave `--language` unset for it.
+"""
 
 import argparse
 import json
@@ -11,6 +24,8 @@
 from common import get_config
 
 # Maps short language codes / locale tags to (lang_id, human-readable name).
+# Nemotron-specific: ignored when running a Moonshine model (which is English-only
+# and exposes no lang_id runtime option).
 # Restricted to the languages officially supported by
 # NVIDIA-Nemotron-3.5-ASR-Streaming-Multilingual-0.6b (per model card).
 # IDs follow the canonical prompt_dictionary in model_config.yaml.

src/config.cpp

@@ -1038,6 +1038,49 @@ struct VAD_Element : JSON::Element {
   std::unique_ptr<RunOptions_Element> run_options_;
 };
 
+struct Moonshine_Element : JSON::Element {
+  explicit Moonshine_Element(Config::Model::Moonshine& v) : v_{v} {}
+
+  void OnValue(std::string_view name, JSON::Value value) override {
+    if (name == "frontend_filename") {
+      v_.frontend_filename = JSON::Get<std::string_view>(value);
+    } else if (name == "encoder_filename") {
+      v_.encoder_filename = JSON::Get<std::string_view>(value);
+    } else if (name == "adapter_filename") {
+      v_.adapter_filename = JSON::Get<std::string_view>(value);
+    } else if (name == "cross_kv_filename") {
+      v_.cross_kv_filename = JSON::Get<std::string_view>(value);
+    } else if (name == "decoder_kv_filename") {
+      v_.decoder_kv_filename = JSON::Get<std::string_view>(value);
+    } else if (name == "sample_buffer_size") {
+      v_.sample_buffer_size = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "conv1_buffer_size") {
+      v_.conv1_buffer_size = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "conv2_buffer_size") {
+      v_.conv2_buffer_size = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "total_lookahead") {
+      v_.total_lookahead = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "left_context_frames") {
+      v_.left_context_frames = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "max_seq_len") {
+      v_.max_seq_len = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "tokens_per_second") {
+      v_.tokens_per_second = static_cast<float>(JSON::Get<double>(value));
+    } else if (name == "seconds_per_memory_frame") {
+      v_.seconds_per_memory_frame = static_cast<float>(JSON::Get<double>(value));
+    } else if (name == "max_segment_memory_frames") {
+      v_.max_segment_memory_frames = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "min_segment_memory_frames") {
+      v_.min_segment_memory_frames = static_cast<int>(JSON::Get<double>(value));
+    } else {
+      throw JSON::unknown_value_error{};
+    }
+  }
+
+ private:
+  Config::Model::Moonshine& v_;
+};
+
 struct EmbeddingInputs_Element : JSON::Element {
   explicit EmbeddingInputs_Element(Config::Model::Embedding::Inputs& v) : v_{v} {}
 
@@ -1173,6 +1216,8 @@ struct Model_Element : JSON::Element {
       v_.sample_rate = SafeDoubleToInt(JSON::Get<double>(value), name);
     } else if (name == "chunk_samples") {
       v_.chunk_samples = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "overlap_samples") {
+      v_.overlap_samples = SafeDoubleToInt(JSON::Get<double>(value), name);
     } else if (name == "blank_id") {
       v_.blank_id = SafeDoubleToInt(JSON::Get<double>(value), name);
     } else if (name == "max_symbols_per_step") {
@@ -1216,6 +1261,9 @@ struct Model_Element : JSON::Element {
     if (name == "vad") {
       return vad_;
     }
+    if (name == "moonshine") {
+      return moonshine_;
+    }
     throw JSON::unknown_value_error{};
   }
 
@@ -1230,6 +1278,7 @@ struct Model_Element : JSON::Element {
   Speech_Element speech_{v_.speech};
   Joiner_Element joiner_{v_.joiner};
   VAD_Element vad_{v_.vad};
+  Moonshine_Element moonshine_{v_.moonshine};
 };
 
 int SafeDoubleToInt(double x, std::string_view name) {
@@ -1701,7 +1750,7 @@ fs::path Config::ResolvePath(std::string_view value) const {
 Config::Config(const fs::path& path, std::string_view json_overlay) : config_path{path} {
   ParseConfig(path / "genai_config.json", json_overlay, *this);
 
-  if (model.context_length == 0 && !ModelType::IsRNNT(model.type)) {
+  if (model.context_length == 0 && !ModelType::IsRNNT(model.type) && !ModelType::IsStreamingEncDecASR(model.type)) {
     throw std::runtime_error("model context_length is 0 or was not set. It must be greater than 0");
   }
 

src/config.h

@@ -162,6 +162,7 @@ struct Config {
     int pre_encode_cache_size{};
     int sample_rate{};
     int chunk_samples{};
+    int overlap_samples{};
     int blank_id{};
     int max_symbols_per_step{};
 
@@ -314,6 +315,31 @@ struct Config {
       std::optional<RunOptions> run_options;
     } vad;
 
+    struct Moonshine {
+      std::string frontend_filename;
+      std::string encoder_filename;
+      std::string adapter_filename;
+      std::string cross_kv_filename;
+      std::string decoder_kv_filename;
+
+      int sample_buffer_size{};
+      int conv1_buffer_size{};
+      int conv2_buffer_size{};
+
+      // Encoder sliding-window geometry.
+      int total_lookahead{};
+      int left_context_frames{};
+
+      // Decoder token-emission cap & frame rate.
+      int max_seq_len{};
+      float tokens_per_second{};
+      float seconds_per_memory_frame{};
+
+      // Segmentation limits (hard cap + VAD-silence min duration), in memory frames.
+      int max_segment_memory_frames{};
+      int min_segment_memory_frames{};
+    } moonshine;
+
     struct Decoder {
       std::string filename;
       SessionOptions session_options;

src/generators.cpp

@@ -371,9 +371,11 @@ std::unique_ptr<Search> CreateSearch(const GeneratorParams& params) {
 }
 
 Generator::Generator(const Model& model, const GeneratorParams& params) : model_{model.shared_from_this()} {
-  // RNNT and TDT models don't use the traditional search/logits pipeline,
-  // so skip the standard validations and just create the state.
-  if (ModelType::IsTransducer(model.config_->model.type)) {
+  // RNNT/TDT/streaming-enc-dec-ASR models don't use the traditional
+  // search/logits pipeline; skip the standard validations and create the
+  // TransducerState-derived state directly.
+  const auto& model_type = model.config_->model.type;
+  if (ModelType::IsTransducer(model_type) || ModelType::IsStreamingEncDecASR(model_type)) {
     state_ = model.CreateState({}, params);
     transducer_state_ = dynamic_cast<TransducerState*>(state_.get());
     return;

src/models/model.cpp

@@ -23,6 +23,7 @@
 #include "parakeet.h"
 #include "parakeet_processor.h"
 #include "nemotron_speech.h"
+#include "moonshine_streaming.h"
 #include "multi_modal.h"
 #include "lfm2.h"
 #include "marian.h"
@@ -888,6 +889,8 @@ std::shared_ptr<Model> CreateModel(OrtEnv& ort_env, std::unique_ptr<Config> conf
     return std::make_shared<DecoderOnly_Model>(std::move(config), ort_env);
   if (ModelType::IsRNNT(config->model.type))
     return std::make_shared<NemotronSpeechModel>(std::move(config), ort_env);
+  if (ModelType::IsStreamingEncDecASR(config->model.type))
+    return std::make_shared<MoonshineStreamingModel>(std::move(config), ort_env);
   if (ModelType::IsTDT(config->model.type))
     return std::make_shared<ParakeetTdtModel>(std::move(config), ort_env);
   if (ModelType::IsALM(config->model.type))

src/models/model_type.h

@@ -45,7 +45,7 @@ struct ModelType {
   }
 
   inline static bool IsRNNT(const std::string& model_type) {
-    // RNNT models bypass the search/logits pipeline entirely.
+    // RNN-Transducer encoder/decoder/joiner models.
     static constexpr std::array<std::string_view, 1> rnnt_types = {"nemotron_speech"};
     return std::find(rnnt_types.begin(), rnnt_types.end(), model_type) != rnnt_types.end();
   }
@@ -61,6 +61,16 @@ struct ModelType {
     return IsRNNT(model_type) || IsTDT(model_type);
   }
 
+  // Streaming encoder-decoder ASR models (e.g. Moonshine). Architecturally
+  // distinct from transducers (audio encoder + auto-regressive transformer
+  // decoder with self+cross KV cache) but share the same runtime contract:
+  // bypass the search/logits pipeline and drive token emission via
+  // TransducerState::StepToken().
+  inline static bool IsStreamingEncDecASR(const std::string& model_type) {
+    static constexpr std::array<std::string_view, 1> streaming_enc_dec = {"streaming_enc_dec_asr"};
+    return std::find(streaming_enc_dec.begin(), streaming_enc_dec.end(), model_type) != streaming_enc_dec.end();
+  }
+
   inline static bool IsMMM(const std::string& model_type) {
     // Multi-modal model (MMM)
     static constexpr std::array<std::string_view, 2> MMM = {"gemma4", "phi4mm"};