litert-lm: fix null backends, suppress logs, patch GPU factory method

Three fixes that together make end-to-end text generation work on CPU
and unblock GPU:

1. Null vision/audio backend: the C API (engine.cc:299) checks
   `if (vision_backend_str)` before parsing — null means "auto-detect
   from model metadata, skip if absent". Passing non-null ("GPU"/"CPU")
   forced the engine to require encoder sections, failing on text-only
   models. EngineSettings now defaults vision/audio to None (= null).

2. Log suppression: Engine::new calls litert_lm_set_min_log_level(3)
   (ERROR-only) to suppress the wall of absl INFO/WARNING output that
   litert-lm-sys dumps during init. Clean output now shows only the
   prompt + response.

3. SamplerParams::top_p() now implies Sampler::TopP (was leaving the
   default TopK, which the CPU backend rejects as unimplemented).

4. GPU fix (build workflow): patch engine.cc to call
   EngineFactory::CreateAny instead of CreateDefault. The CLI
   (litert_lm_main.cc) uses CreateAny which picks an optimized engine
   type; CreateDefault forces kLiteRTCompiledModel which hangs during
   Metal shader compilation. Rebuild required — triggered separately.

Verified: Qwen3-0.6B generates text on CPU via litert-lm safe API.

Author: darmie

.github/workflows/build-litert-lm.yml

@@ -32,6 +32,14 @@ jobs:
           git clone --depth=1 --branch=${{ env.LITERT_LM_TAG }} \
             https://github.com/google-ai-edge/litert-lm.git /tmp/litert-lm
 
+      # Patch engine.cc: CreateDefault → CreateAny so the GPU path uses the
+      # optimized engine type (same as the upstream litert-lm CLI).
+      - name: patch engine.cc (CreateDefault → CreateAny)
+        run: |
+          sed -i 's/EngineFactory::CreateDefault/EngineFactory::CreateAny/g' \
+            /tmp/litert-lm/c/engine.cc
+          grep -n 'EngineFactory::Create' /tmp/litert-lm/c/engine.cc
+
       # Inject a cc_binary(linkshared=True) target so Bazel itself produces
       # the .so with full dep resolution — no manual post-link needed.
       - name: create shared lib BUILD target
@@ -83,6 +91,12 @@ jobs:
           git clone --depth=1 --branch=${{ env.LITERT_LM_TAG }} \
             https://github.com/google-ai-edge/litert-lm.git /tmp/litert-lm
 
+      - name: patch engine.cc (CreateDefault → CreateAny)
+        run: |
+          sed -i '' 's/EngineFactory::CreateDefault/EngineFactory::CreateAny/g' \
+            /tmp/litert-lm/c/engine.cc
+          grep -n 'EngineFactory::Create' /tmp/litert-lm/c/engine.cc
+
       - name: create shared lib BUILD target
         run: |
           mkdir -p /tmp/litert-lm/shared_build

litert-lm/examples/llm_generate.rs

@@ -47,27 +47,31 @@ fn main() -> Result<(), Box<dyn Error>> {
     println!("model: {} ({})", model_path.display(), model_info.label);
     println!("loading engine...");
 
-    let use_cpu = args.iter().any(|a| a == "--cpu");
-    let backend = if use_cpu { Backend::Cpu } else { Backend::Gpu };
+    let backend = if args.iter().any(|a| a == "--cpu") {
+        Backend::Cpu
+    } else {
+        Backend::Gpu
+    };
     println!("backend: {backend:?}");
     let cache_dir = std::env::temp_dir().join("litert-lm-cache");
     fs::create_dir_all(&cache_dir)?;
-    let mut settings = EngineSettings::new(&model_path)
-        .backend(backend)
-        .max_num_tokens(512)
-        .cache_dir(&cache_dir);
-    // For text-only models on CPU: vision/audio backends must be GPU
-    // (upstream quirk — CPU path fatally requires encoder sections).
-    if use_cpu {
-        settings = settings
-            .vision_backend(Backend::Gpu)
-            .audio_backend(Backend::Gpu);
-    }
-    let engine = Engine::new(settings)?;
+    let engine = Engine::new(
+        EngineSettings::new(&model_path)
+            .backend(backend)
+            .max_num_tokens(512)
+            .cache_dir(&cache_dir),
+    )?;
 
     println!("creating session...");
-    let mut session =
-        engine.create_session(SamplerParams::default().top_k(40).temperature(0.7).seed(42))?;
+    // Use TopP (nucleus) sampling — matches the model's own metadata and
+    // is supported on both CPU and GPU backends. TopK is not yet implemented
+    // on CPU upstream.
+    let mut session = engine.create_session(
+        SamplerParams::default()
+            .top_p(0.95)
+            .temperature(0.7)
+            .seed(42),
+    )?;
 
     let prompt = "Explain Rust lifetimes in one sentence.";
     println!("prompt: {prompt}");

litert-lm/src/engine.rs

@@ -49,21 +49,25 @@ impl Backend {
 pub struct EngineSettings {
     model_path: PathBuf,
     backend: Backend,
-    vision_backend: Backend,
-    audio_backend: Backend,
+    /// None = auto-detect from model metadata (null in C API).
+    vision_backend: Option<Backend>,
+    /// None = auto-detect from model metadata (null in C API).
+    audio_backend: Option<Backend>,
     max_num_tokens: Option<i32>,
     cache_dir: Option<PathBuf>,
 }
 
 impl EngineSettings {
     /// Create settings for a model file at the given path.
-    /// Defaults to [`Backend::Gpu`] for all modalities.
+    /// Main backend defaults to [`Backend::Gpu`]; vision/audio auto-detect
+    /// from model metadata (null in C API → `EngineSettings::CreateDefault`
+    /// decides).
     pub fn new(model_path: impl Into<PathBuf>) -> Self {
         Self {
             model_path: model_path.into(),
             backend: Backend::default(),
-            vision_backend: Backend::default(),
-            audio_backend: Backend::default(),
+            vision_backend: None,
+            audio_backend: None,
             max_num_tokens: None,
             cache_dir: None,
         }
@@ -76,19 +80,19 @@ impl EngineSettings {
         self
     }
 
-    /// Set the vision encoder backend. Models without a vision encoder
-    /// ignore this setting.
+    /// Set the vision encoder backend. `None` (default) auto-detects from
+    /// model metadata; text-only models skip vision entirely.
     #[must_use]
     pub fn vision_backend(mut self, backend: Backend) -> Self {
-        self.vision_backend = backend;
+        self.vision_backend = Some(backend);
         self
     }
 
-    /// Set the audio encoder backend. Models without an audio encoder
-    /// ignore this setting.
+    /// Set the audio encoder backend. `None` (default) auto-detects from
+    /// model metadata; text-only models skip audio entirely.
     #[must_use]
     pub fn audio_backend(mut self, backend: Backend) -> Self {
-        self.audio_backend = backend;
+        self.audio_backend = Some(backend);
         self
     }
 
@@ -141,17 +145,31 @@ impl Engine {
     /// # Ok::<(), litert_lm::Error>(())
     /// ```
     pub fn new(settings: EngineSettings) -> Result<Self> {
+        // Suppress the wall of INFO/WARNING logs from absl + LiteRT internals.
+        // 2 = WARNING, 3 = ERROR. We set to ERROR-only by default.
+        unsafe { sys::litert_lm_set_min_log_level(3) };
+
         let model_str = path_to_cstring(&settings.model_path)?;
         let backend_str = CString::new(settings.backend.as_str()).unwrap();
-        let vision_str = CString::new(settings.vision_backend.as_str()).unwrap();
-        let audio_str = CString::new(settings.audio_backend.as_str()).unwrap();
+
+        // Vision/audio: None → null → C API auto-detects from model metadata.
+        // Only pass a non-null string if the user explicitly set a backend
+        // (e.g., for multimodal models with vision/audio encoder sections).
+        let vision_cstr = settings
+            .vision_backend
+            .map(|b| CString::new(b.as_str()).unwrap());
+        let audio_cstr = settings
+            .audio_backend
+            .map(|b| CString::new(b.as_str()).unwrap());
 
         let raw_settings = unsafe {
             sys::litert_lm_engine_settings_create(
                 model_str.as_ptr(),
                 backend_str.as_ptr(),
-                vision_str.as_ptr(),
-                audio_str.as_ptr(),
+                vision_cstr
+                    .as_ref()
+                    .map_or(std::ptr::null(), |s| s.as_ptr()),
+                audio_cstr.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
             )
         };
         if raw_settings.is_null() {

litert-lm/src/sampler.rs

@@ -66,10 +66,11 @@ impl SamplerParams {
         self
     }
 
-    /// Set the nucleus probability threshold.
+    /// Set the nucleus probability threshold. Implies [`Sampler::TopP`].
     #[must_use]
     pub fn top_p(mut self, v: f32) -> Self {
         self.top_p = v;
+        self.sampler = Sampler::TopP;
         self
     }