feat(runtime): bridge Python image and audio providers

Author: georgi

packages/runtime/src/providers/python-provider.ts

@@ -15,6 +15,7 @@ import type {
   ASRModel,
   EmbeddingModel,
   VideoModel,
+  MusicModel,
   Message,
   ProviderTool,
   ProviderStreamItem,
@@ -23,10 +24,13 @@ import type {
   ImageToImageParams,
   TextToSpeechParams,
   TextToVideoParams,
-  ImageToVideoParams
+  ImageToVideoParams,
+  TextToMusicParams,
+  EncodedAudioResult
 } from "./types.js";
 import type { PythonBridgeBase } from "../python-bridge-base.js";
 import { isRecord, isString } from "@nodetool-ai/protocol";
+import { sniffAudioMime } from "./audio-mime.js";
 
 type PythonProviderOptions = Record<string, unknown> & {
   _id: string;
@@ -160,6 +164,10 @@ export class PythonProvider extends BaseProvider {
     return this._getModels("video") as Promise<VideoModel[]>;
   }
 
+  async getAvailableMusicModels(): Promise<MusicModel[]> {
+    return this._getModels("music") as Promise<MusicModel[]>;
+  }
+
   private async _getModels(modelType: string): Promise<unknown[]> {
     const models = await this._bridge.getProviderModels(
       this._pythonProviderId,
@@ -333,6 +341,26 @@ export class PythonProvider extends BaseProvider {
     }
   }
 
+  async textToSpeechEncoded(
+    args: TextToSpeechParams
+  ): Promise<EncodedAudioResult | null> {
+    const data = await this._bridge.providerTTSEncoded(
+      this._pythonProviderId,
+      { ...args },
+      this._secrets
+    );
+    return { data, mimeType: sniffAudioMime(data) };
+  }
+
+  async textToMusic(params: TextToMusicParams): Promise<EncodedAudioResult> {
+    const data = await this._bridge.providerTextToAudio(
+      this._pythonProviderId,
+      { ...params, model: params.model.id },
+      this._secrets
+    );
+    return { data, mimeType: sniffAudioMime(data) };
+  }
+
   async automaticSpeechRecognition(args: {
     audio: Uint8Array;
     model: string;

packages/runtime/src/python-bridge-base.ts

@@ -1150,6 +1150,32 @@ export abstract class PythonBridgeBase
     return result.blobs.video;
   }
 
+  async providerTextToAudio(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array> {
+    const result = await this._providerBlobCall("provider.text_to_audio", {
+      provider: providerId,
+      params,
+      secrets: secrets ?? {}
+    });
+    return result.blobs.audio;
+  }
+
+  async providerTTSEncoded(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array> {
+    const result = await this._providerBlobCall("provider.tts_encoded", {
+      provider: providerId,
+      params,
+      secrets: secrets ?? {}
+    });
+    return result.blobs.audio;
+  }
+
   async *providerTTS(
     providerId: string,
     text: string,

packages/runtime/src/python-bridge-types.ts

@@ -588,6 +588,16 @@ export interface PythonBridge extends EventEmitter {
     secrets?: Record<string, string>,
     signal?: AbortSignal
   ): Promise<Uint8Array>;
+  providerTextToAudio(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array>;
+  providerTTSEncoded(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array>;
   providerASR(
     providerId: string,
     audio: Uint8Array,

packages/runtime/src/swappable-python-bridge.ts

@@ -275,6 +275,22 @@ export class SwappableBridge extends EventEmitter implements PythonBridge {
     );
   }
 
+  providerTextToAudio(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array> {
+    return this._target.providerTextToAudio(providerId, params, secrets);
+  }
+
+  providerTTSEncoded(
+    providerId: string,
+    params: Record<string, unknown>,
+    secrets?: Record<string, string>
+  ): Promise<Uint8Array> {
+    return this._target.providerTTSEncoded(providerId, params, secrets);
+  }
+
   providerASR(
     providerId: string,
     audio: Uint8Array,

packages/runtime/tests/providers/provider-registry-extended.test.ts

@@ -212,4 +212,50 @@ describe("provider-registry — extended coverage", () => {
       undefined
     );
   });
+
+  it("discovers music and routes encoded audio through the Python bridge", async () => {
+    const wav = new Uint8Array([
+      0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x41, 0x56, 0x45
+    ]);
+    const getProviderModels = vi.fn(async () => [
+      { id: "ace", name: "ACE-Step", provider: "wangp" }
+    ]);
+    const textToAudio = vi.fn(async () => wav);
+    const ttsEncoded = vi.fn(async () => wav);
+    const provider = new PythonProvider({
+      _id: "wangp",
+      _bridge: {
+        getProviderModels,
+        providerTextToAudio: textToAudio,
+        providerTTSEncoded: ttsEncoded
+      }
+    } as any);
+
+    await expect(provider.getAvailableMusicModels()).resolves.toEqual([
+      { id: "ace", name: "ACE-Step", provider: "wangp" }
+    ]);
+    await expect(
+      provider.textToMusic({
+        model: { id: "ace", name: "ACE-Step", provider: "wangp" },
+        prompt: "ambient"
+      })
+    ).resolves.toEqual({ data: wav, mimeType: "audio/wav" });
+    await expect(
+      provider.textToSpeechEncoded({ text: "hello", model: "qwen3" })
+    ).resolves.toEqual({ data: wav, mimeType: "audio/wav" });
+    expect(getProviderModels).toHaveBeenCalledWith("wangp", "music", {});
+    expect(textToAudio).toHaveBeenCalledWith(
+      "wangp",
+      {
+        model: "ace",
+        prompt: "ambient"
+      },
+      {}
+    );
+    expect(ttsEncoded).toHaveBeenCalledWith(
+      "wangp",
+      { text: "hello", model: "qwen3" },
+      {}
+    );
+  });
 });

packages/runtime/tests/python-bridge-base-coverage.test.ts

@@ -714,6 +714,40 @@ describe("PythonBridgeBase — provider RPCs", () => {
     await expect(p).resolves.toBe(output);
   });
 
+  it("providerTextToAudio returns the encoded audio blob", async () => {
+    const output = new Uint8Array([4, 5, 6]);
+    const p = bridge.providerTextToAudio("wangp", {
+      model: "ace_step_1.5",
+      prompt: "ambient"
+    });
+    const frame = bridge.sent.find((f) => f.type === "provider.text_to_audio")!;
+    expect(frame.data).toEqual({
+      provider: "wangp",
+      params: { model: "ace_step_1.5", prompt: "ambient" },
+      secrets: {},
+      blob_transfer: "chunked-v1"
+    });
+    reply("provider.text_to_audio", { blobs: { audio: output } });
+    await expect(p).resolves.toBe(output);
+  });
+
+  it("providerTTSEncoded returns the encoded audio blob", async () => {
+    const output = new Uint8Array([7, 8, 9]);
+    const p = bridge.providerTTSEncoded("wangp", {
+      model: "qwen3_tts",
+      text: "hello"
+    });
+    const frame = bridge.sent.find((f) => f.type === "provider.tts_encoded")!;
+    expect(frame.data).toEqual({
+      provider: "wangp",
+      params: { model: "qwen3_tts", text: "hello" },
+      secrets: {},
+      blob_transfer: "chunked-v1"
+    });
+    reply("provider.tts_encoded", { blobs: { audio: output } });
+    await expect(p).resolves.toBe(output);
+  });
+
   it("cancels an in-flight video provider request with its abort signal", async () => {
     const controller = new AbortController();
     const p = bridge.providerTextToVideo(

packages/runtime/tests/swappable-python-bridge.test.ts

@@ -93,6 +93,12 @@ class FakeBridge extends EventEmitter {
   providerImageToVideo(): Promise<Uint8Array> {
     return Promise.resolve(new Uint8Array());
   }
+  providerTextToAudio(): Promise<Uint8Array> {
+    return Promise.resolve(new Uint8Array());
+  }
+  providerTTSEncoded(): Promise<Uint8Array> {
+    return Promise.resolve(new Uint8Array());
+  }
   providerASR(): Promise<{ text: string }> {
     return Promise.resolve({ text: "" });
   }