feat: merge upstream PR sigoden#1508 (audio transcription)

Co-Authored-By: Federico Simoncelli <federico.simoncelli@gmail.com>

Author: blob42

models.yaml

@@ -38,6 +38,7 @@ impl_client_trait!(
     ),
     (prepare_embeddings, openai_embeddings),
     (noop_prepare_rerank, noop_rerank),
+    (noop_audio_transcriptions),
 );
 
 fn prepare_chat_completions(

src/client/azure_openai.rs

@@ -36,6 +36,7 @@ impl_client_trait!(
     ),
     (noop_prepare_embeddings, noop_embeddings),
     (noop_prepare_rerank, noop_rerank),
+    (noop_audio_transcriptions),
 );
 
 fn prepare_chat_completions(

src/client/claude.rs

@@ -36,6 +36,7 @@ impl_client_trait!(
     ),
     (prepare_embeddings, embeddings),
     (prepare_rerank, generic_rerank),
+    (noop_audio_transcriptions),
 );
 
 fn prepare_chat_completions(

src/client/cohere.rs

@@ -148,6 +148,21 @@ pub trait Client: Sync + Send {
         bail!("The client doesn't support rerank api")
     }
 
+    async fn audio_transcriptions(&self, data: TranscriptionData) -> Result<String> {
+        let client = self.build_client()?;
+        self.audio_transcriptions_inner(&client, data)
+            .await
+            .context("Failed to call audio-transcriptions api")
+    }
+
+    async fn audio_transcriptions_inner(
+        &self,
+        _client: &ReqwestClient,
+        _data: TranscriptionData,
+    ) -> Result<String> {
+        bail!("The client doesn't support audio transcriptions api")
+    }
+
     fn request_builder(
         &self,
         client: &reqwest::Client,
@@ -343,6 +358,19 @@ pub struct RerankResult {
     pub relevance_score: f64,
 }
 
+pub struct TranscriptionData {
+    pub path: std::path::PathBuf,
+    pub prompt: Option<String>,
+}
+
+pub async fn noop_audio_transcriptions<T: Client>(
+    _self: &T,
+    _client: &ReqwestClient,
+    _data: TranscriptionData,
+) -> Result<String> {
+    bail!("The client doesn't support audio transcriptions api")
+}
+
 pub type PromptAction<'a> = (&'a str, &'a str, Option<&'a str>);
 
 pub async fn create_config(

src/client/common.rs

@@ -35,6 +35,7 @@ impl_client_trait!(
     ),
     (prepare_embeddings, embeddings),
     (noop_prepare_rerank, noop_rerank),
+    (noop_audio_transcriptions),
 );
 
 fn prepare_chat_completions(

src/client/gemini.rs

@@ -173,6 +173,7 @@ macro_rules! impl_client_trait {
         ($prepare_chat_completions:path, $chat_completions:path, $chat_completions_streaming:path),
         ($prepare_embeddings:path, $embeddings:path),
         ($prepare_rerank:path, $rerank:path),
+        ($audio_transcriptions:path),
     ) => {
         #[async_trait::async_trait]
         impl $crate::client::Client for $crate::client::$client {
@@ -218,6 +219,14 @@ macro_rules! impl_client_trait {
                 let builder = self.request_builder(client, request_data);
                 $rerank(builder, self.model()).await
             }
+
+            async fn audio_transcriptions_inner(
+                &self,
+                client: &reqwest::Client,
+                data: $crate::client::TranscriptionData,
+            ) -> Result<String> {
+                $audio_transcriptions(self, client, data).await
+            }
         }
     };
 }

src/client/macros.rs

@@ -37,6 +37,7 @@ impl_client_trait!(
     ),
     (prepare_embeddings, openai_embeddings),
     (noop_prepare_rerank, noop_rerank),
+    (openai_audio_transcriptions),
 );
 
 fn prepare_chat_completions(
@@ -406,3 +407,90 @@ fn normalize_function_id(value: &str) -> Option<String> {
         Some(value.to_string())
     }
 }
+
+pub async fn openai_audio_transcriptions(
+    self_: &OpenAIClient,
+    client: &reqwest::Client,
+    data: TranscriptionData,
+) -> Result<String> {
+    let api_key = self_.get_api_key()?;
+    let api_base = self_
+        .get_api_base()
+        .unwrap_or_else(|_| API_BASE.to_string());
+    let model_name = self_.model.real_name().to_string();
+    openai_compatible_audio_transcriptions(client, &api_base, Some(&api_key), &model_name, data)
+        .await
+}
+
+pub async fn openai_compatible_audio_transcriptions(
+    client: &reqwest::Client,
+    api_base: &str,
+    api_key: Option<&str>,
+    model_name: &str,
+    data: TranscriptionData,
+) -> Result<String> {
+    let url = format!("{}/audio/transcriptions", api_base.trim_end_matches('/'));
+
+    let file_bytes = tokio::fs::read(&data.path)
+        .await
+        .with_context(|| format!("Failed to read audio file '{}'", data.path.display()))?;
+
+    let file_name = data
+        .path
+        .file_name()
+        .and_then(|n| n.to_str())
+        .unwrap_or("audio")
+        .to_string();
+
+    let mime_type = audio_mime_type(&data.path);
+
+    let file_part = reqwest::multipart::Part::bytes(file_bytes)
+        .file_name(file_name)
+        .mime_str(&mime_type)
+        .context("Invalid MIME type for audio file")?;
+
+    let mut form = reqwest::multipart::Form::new()
+        .part("file", file_part)
+        .text("model", model_name.to_string());
+
+    if let Some(prompt) = data.prompt {
+        form = form.text("prompt", prompt);
+    }
+
+    let mut request = client.post(&url).multipart(form);
+    if let Some(key) = api_key {
+        request = request.bearer_auth(key);
+    }
+
+    let res = request.send().await?;
+    let status = res.status();
+    let body: serde_json::Value = res.json().await?;
+
+    if !status.is_success() {
+        catch_error(&body, status.as_u16())?;
+    }
+
+    body["text"]
+        .as_str()
+        .map(|s| s.to_string())
+        .ok_or_else(|| anyhow::anyhow!("Invalid transcription response: {body}"))
+}
+
+fn audio_mime_type(path: &std::path::Path) -> String {
+    let ext = path
+        .extension()
+        .and_then(|e| e.to_str())
+        .unwrap_or("")
+        .to_lowercase();
+    match ext.as_str() {
+        "mp3" | "mpeg" | "mpga" => "audio/mpeg",
+        "mp4" => "audio/mp4",
+        "m4a" => "audio/m4a",
+        "ogg" | "oga" => "audio/ogg",
+        "wav" => "audio/wav",
+        "webm" => "audio/webm",
+        "flac" => "audio/flac",
+        _ => "application/octet-stream",
+    }
+    .to_string()
+}

src/client/openai.rs

@@ -33,6 +33,7 @@ impl_client_trait!(
     ),
     (prepare_embeddings, openai_embeddings),
     (prepare_rerank, generic_rerank),
+    (openai_compatible_client_audio_transcriptions),
 );
 
 fn prepare_chat_completions(
@@ -141,6 +142,24 @@ pub struct GenericRerankResBody {
     pub results: RerankOutput,
 }
 
+pub async fn openai_compatible_client_audio_transcriptions(
+    self_: &OpenAICompatibleClient,
+    client: &reqwest::Client,
+    data: TranscriptionData,
+) -> Result<String> {
+    let api_key = self_.get_api_key().ok();
+    let api_base = get_api_base_ext(self_)?;
+    let model_name = self_.model.real_name().to_string();
+    openai_compatible_audio_transcriptions(
+        client,
+        &api_base,
+        api_key.as_deref(),
+        &model_name,
+        data,
+    )
+    .await
+}
+
 pub fn generic_build_rerank_body(data: &RerankData, model: &Model) -> Value {
     let RerankData {
         query,

src/client/openai_compatible.rs

@@ -6,11 +6,12 @@ use self::completer::ReplCompleter;
 use self::highlighter::ReplHighlighter;
 use self::prompt::ReplPrompt;
 
-use crate::client::{call_chat_completions, call_chat_completions_streaming};
+use crate::client::{call_chat_completions, call_chat_completions_streaming, init_client, TranscriptionData};
 use crate::config::{
     macro_execute, AgentVariables, AssertState, Config, GlobalConfig, Input, LastMessage,
     StateFlags,
 };
+use crate::function::ToolResult;
 use crate::render::render_error;
 use crate::run_command;
 use crate::utils::{
@@ -32,7 +33,7 @@ use std::{env, fs, process};
 
 const MENU_NAME: &str = "completion_menu";
 
-static REPL_COMMANDS: LazyLock<[ReplCommand; 37]> = LazyLock::new(|| {
+static REPL_COMMANDS: LazyLock<[ReplCommand; 38]> = LazyLock::new(|| {
     [
         ReplCommand::new(".help", "Show this help guide", AssertState::pass()),
         ReplCommand::new(".info", "Show system info", AssertState::pass()),
@@ -172,6 +173,11 @@ static REPL_COMMANDS: LazyLock<[ReplCommand; 37]> = LazyLock::new(|| {
             "Include files, directories, URLs or commands",
             AssertState::pass(),
         ),
+        ReplCommand::new(
+            ".transcript",
+            "Transcribe an audio file",
+            AssertState::pass(),
+        ),
         ReplCommand::new(
             ".continue",
             "Continue previous response",
@@ -655,6 +661,37 @@ pub async fn run_repl_command(
 .file %% -- translate last reply to english"#
                 ),
             },
+            ".transcript" => match args {
+                Some(args) => {
+                    let (files, text) = split_args_text(args, cfg!(windows));
+                    if files.is_empty() {
+                        println!("Usage: .transcript <audio-file> [-- <prompt>]");
+                    } else {
+                        let path = std::path::PathBuf::from(&files[0]);
+                        let prompt = if text.is_empty() { None } else { Some(text.to_string()) };
+                        let data = TranscriptionData { path, prompt };
+                        let model = config.read().current_model().clone();
+                        let client = init_client(config, Some(model))?;
+                        let transcript = abortable_run_with_spinner(
+                            client.audio_transcriptions(data),
+                            "Transcribing",
+                            abort_signal.clone(),
+                        )
+                        .await?;
+                        config.read().print_markdown(&transcript)?;
+                        let input = Input::from_str(config, &files[0], None);
+                        config
+                            .write()
+                            .after_chat_completion(&input, &transcript, &[] as &[ToolResult])?;
+                    }
+                }
+                None => println!(
+                    r#"Usage: .transcript <audio-file> [-- <prompt>]
+
+.transcript meeting.mp3
+.transcript interview.wav -- Interview about Rust programming"#
+                ),
+            },
             ".continue" => {
                 let LastMessage {
                     mut input, output, ..