feat(groqcloud): add audio transcription and translation actions

src/providers/groqcloud/actions.ts

@@ -5,7 +5,12 @@ import { defineProviderAction } from "../../core/provider-definition.ts";
 
 const service = "groqcloud";
 
-export type GroqcloudActionName = "list_models" | "get_model" | "create_chat_completion";
+export type GroqcloudActionName =
+  | "list_models"
+  | "get_model"
+  | "create_chat_completion"
+  | "create_audio_transcription"
+  | "create_audio_translation";
 
 const nullSchema: JsonSchema = { type: "null", description: "Null value." };
 const unknownJsonValueSchema = s.unknown("Any JSON value accepted by the upstream API.");
@@ -119,6 +124,93 @@ const chatCompletionOutputSchema = s.looseObject("The response payload for a Gro
   system_fingerprint: s.string("The backend system fingerprint for the completion."),
 });
 
+const audioModelSchema = s.stringEnum("The GroqCloud speech-to-text model identifier.", [
+  "whisper-large-v3",
+  "whisper-large-v3-turbo",
+]);
+const translationModelSchema = s.stringEnum(
+  "The GroqCloud speech-to-text model identifier. Only whisper-large-v3 supports translation.",
+  ["whisper-large-v3"],
+);
+const audioFileSchema: JsonSchema = {
+  ...s.object(
+    "The audio source. Provide url for GroqCloud to fetch the audio, or content_base64 to upload the bytes inline.",
+    {
+      name: s.nonEmptyString(
+        "The file name reported to GroqCloud, including the audio file extension. Required with content_base64.",
+      ),
+      mimetype: s.string("The MIME type of the audio file, such as audio/mpeg."),
+      url: s.nonEmptyString("A public URL that GroqCloud downloads the audio from."),
+      content_base64: s.nonEmptyString("The base64-encoded audio content to upload."),
+    },
+    { optional: ["name", "mimetype", "url", "content_base64"] },
+  ),
+  anyOf: [{ required: ["url"] }, { required: ["content_base64", "name"] }],
+  not: { required: ["url", "content_base64"] },
+};
+const audioResponseFormatSchema = s.stringEnum(
+  "The transcript format to return. This connector returns structured payloads, so the plain text format is not offered.",
+  ["json", "verbose_json"],
+);
+const audioTemperatureSchema = s.number("The sampling temperature applied to the transcription.", {
+  minimum: 0,
+  maximum: 1,
+});
+const audioPromptSchema = s.string("Optional context or style guidance for the transcript, limited to 224 tokens.");
+const transcriptionInputSchema = s.object(
+  "The input payload for transcribing audio with GroqCloud.",
+  {
+    model: audioModelSchema,
+    file: audioFileSchema,
+    language: s.string("The ISO-639-1 code of the spoken language, such as en, which improves accuracy and latency."),
+    prompt: audioPromptSchema,
+    response_format: audioResponseFormatSchema,
+    temperature: audioTemperatureSchema,
+    timestamp_granularities: s.array(
+      "The timestamp detail to include. Requires response_format to be verbose_json.",
+      s.stringEnum("A timestamp granularity.", ["word", "segment"]),
+      { minItems: 1 },
+    ),
+  },
+  { required: ["model", "file"] },
+);
+const translationInputSchema = s.object(
+  "The input payload for translating audio into English with GroqCloud.",
+  {
+    model: translationModelSchema,
+    file: audioFileSchema,
+    prompt: audioPromptSchema,
+    response_format: audioResponseFormatSchema,
+    temperature: audioTemperatureSchema,
+  },
+  { required: ["model", "file"] },
+);
+const audioSegmentSchema = s.looseObject("A transcribed segment of the audio.", {
+  id: s.integer("The segment index."),
+  seek: s.integer("The seek offset of the segment."),
+  start: s.number("The segment start time in seconds."),
+  end: s.number("The segment end time in seconds."),
+  text: s.string("The transcribed text for the segment."),
+  tokens: s.array("The token identifiers for the segment.", s.integer("A token identifier.")),
+  temperature: s.number("The sampling temperature used for the segment."),
+  avg_logprob: s.number("The average log probability of the segment."),
+  compression_ratio: s.number("The compression ratio of the segment."),
+  no_speech_prob: s.number("The probability that the segment contains no speech."),
+});
+const audioWordSchema = s.looseObject("A transcribed word with timestamps.", {
+  word: s.string("The transcribed word."),
+  start: s.number("The word start time in seconds."),
+  end: s.number("The word end time in seconds."),
+});
+const audioTranscriptOutputSchema = s.looseObject("The transcript payload returned by GroqCloud.", {
+  text: s.string("The full transcript text."),
+  language: s.string("The detected or requested language of the audio."),
+  duration: s.number("The audio duration in seconds."),
+  segments: s.array("The transcribed segments, returned for the verbose_json format.", audioSegmentSchema),
+  words: s.array("The transcribed words, returned when word timestamp granularity is requested.", audioWordSchema),
+  x_groq: jsonObjectSchema,
+});
+
 export const groqcloudActions: ActionDefinition[] = [
   defineProviderAction(service, {
     name: "list_models",
@@ -138,4 +230,18 @@ export const groqcloudActions: ActionDefinition[] = [
     inputSchema: chatCompletionInputSchema,
     outputSchema: chatCompletionOutputSchema,
   }),
+  defineProviderAction(service, {
+    name: "create_audio_transcription",
+    description:
+      "Transcribe an audio file into text in its original language using a GroqCloud Whisper model. Supply the audio inline as base64 or as a public URL that GroqCloud downloads.",
+    inputSchema: transcriptionInputSchema,
+    outputSchema: audioTranscriptOutputSchema,
+  }),
+  defineProviderAction(service, {
+    name: "create_audio_translation",
+    description:
+      "Translate an audio file into English text using a GroqCloud Whisper model. Supply the audio inline as base64 or as a public URL that GroqCloud downloads.",
+    inputSchema: translationInputSchema,
+    outputSchema: audioTranscriptOutputSchema,
+  }),
 ];

src/providers/groqcloud/executors.test.ts

@@ -0,0 +1,212 @@
+import { describe, expect, it, vi } from "vitest";
+import { validateActionInput } from "../../core/validation.ts";
+import { groqcloudActions } from "./actions.ts";
+import { groqcloudActionHandlers } from "./executors.ts";
+
+const apiBaseUrl = "https://api.groq.com/openai/v1";
+const audioBase64 = Buffer.from("fake-audio-bytes").toString("base64");
+
+function createContext(fetcher: typeof fetch) {
+  return {
+    apiKey: "test-key",
+    fetcher,
+  };
+}
+
+function jsonFetcher(payload: Record<string, unknown> = {}): typeof fetch {
+  return vi.fn(async () => Response.json(payload)) as typeof fetch;
+}
+
+function requestInit(fetcher: typeof fetch): RequestInit {
+  return vi.mocked(fetcher).mock.calls[0]![1] as RequestInit;
+}
+
+function requestForm(fetcher: typeof fetch): FormData {
+  return requestInit(fetcher).body as FormData;
+}
+
+describe("GroqCloud audio transcription", () => {
+  it("uploads inline base64 audio as multipart form data", async () => {
+    const fetcher = jsonFetcher({ text: "hello" });
+
+    await groqcloudActionHandlers.create_audio_transcription(
+      {
+        model: "whisper-large-v3-turbo",
+        file: { name: "meeting.mp3", mimetype: "audio/mpeg", content_base64: audioBase64 },
+        language: "en",
+        temperature: 0,
+      },
+      createContext(fetcher),
+    );
+
+    expect(fetcher).toHaveBeenCalledWith(
+      `${apiBaseUrl}/audio/transcriptions`,
+      expect.objectContaining({ method: "POST" }),
+    );
+
+    const form = requestForm(fetcher);
+    const uploaded = form.get("file") as File;
+    expect(uploaded.name).toBe("meeting.mp3");
+    expect(uploaded.type).toBe("audio/mpeg");
+    expect(await uploaded.text()).toBe("fake-audio-bytes");
+    expect(form.get("model")).toBe("whisper-large-v3-turbo");
+    expect(form.get("language")).toBe("en");
+    expect(form.get("temperature")).toBe("0");
+  });
+
+  it("omits the JSON content type so the multipart boundary is preserved", async () => {
+    const fetcher = jsonFetcher({ text: "hello" });
+
+    await groqcloudActionHandlers.create_audio_transcription(
+      { model: "whisper-large-v3", file: { name: "a.mp3", content_base64: audioBase64 } },
+      createContext(fetcher),
+    );
+
+    const headers = requestInit(fetcher).headers as Record<string, string>;
+    expect(headers["content-type"]).toBeUndefined();
+    expect(headers.authorization).toBe("Bearer test-key");
+  });
+
+  it("forwards a public url instead of downloading the audio", async () => {
+    const fetcher = jsonFetcher({ text: "hello" });
+
+    await groqcloudActionHandlers.create_audio_translation(
+      { model: "whisper-large-v3", file: { url: "https://example.com/clip.mp3" } },
+      createContext(fetcher),
+    );
+
+    expect(vi.mocked(fetcher).mock.calls).toHaveLength(1);
+    expect(String(vi.mocked(fetcher).mock.calls[0]![0])).toBe(`${apiBaseUrl}/audio/translations`);
+    const form = requestForm(fetcher);
+    expect(form.get("url")).toBe("https://example.com/clip.mp3");
+    expect(form.get("file")).toBeNull();
+  });
+
+  it("rejects private and loopback audio urls before forwarding them", () => {
+    const fetcher = jsonFetcher();
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        { model: "whisper-large-v3", file: { url: "http://127.0.0.1/internal.mp3" } },
+        createContext(fetcher),
+      ),
+    ).toThrow();
+
+    expect(fetcher).not.toHaveBeenCalled();
+  });
+
+  it("requires exactly one audio source", () => {
+    const fetcher = jsonFetcher();
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription({ model: "whisper-large-v3" }, createContext(fetcher)),
+    ).toThrow("file is required");
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        { model: "whisper-large-v3", file: { name: "a.mp3" } },
+        createContext(fetcher),
+      ),
+    ).toThrow("file must include url or content_base64");
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        {
+          model: "whisper-large-v3",
+          file: { name: "a.mp3", content_base64: audioBase64, url: "https://example.com/clip.mp3" },
+        },
+        createContext(fetcher),
+      ),
+    ).toThrow("provide only one of file.url or file.content_base64");
+
+    expect(fetcher).not.toHaveBeenCalled();
+  });
+
+  it("rejects malformed base64 audio content", () => {
+    const fetcher = jsonFetcher();
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        { model: "whisper-large-v3", file: { name: "a.mp3", content_base64: "not*base64" } },
+        createContext(fetcher),
+      ),
+    ).toThrow("file.content_base64 must be valid base64");
+  });
+
+  it("rejects inline audio above GroqCloud's attachment limit", () => {
+    const fetcher = jsonFetcher();
+    const attachmentMaxBytes = 25 * 1024 * 1024;
+    const oversizedAudioBase64 = Buffer.alloc(attachmentMaxBytes + 1).toString("base64");
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        { model: "whisper-large-v3", file: { name: "a.mp3", content_base64: oversizedAudioBase64 } },
+        createContext(fetcher),
+      ),
+    ).toThrow(`file.content_base64 exceeds ${attachmentMaxBytes} bytes`);
+
+    expect(fetcher).not.toHaveBeenCalled();
+  });
+
+  it("repeats timestamp granularities as an array field and requires verbose_json", async () => {
+    const fetcher = jsonFetcher({ text: "hello" });
+
+    await groqcloudActionHandlers.create_audio_transcription(
+      {
+        model: "whisper-large-v3",
+        file: { name: "a.mp3", content_base64: audioBase64 },
+        response_format: "verbose_json",
+        timestamp_granularities: ["segment", "word"],
+      },
+      createContext(fetcher),
+    );
+
+    expect(requestForm(fetcher).getAll("timestamp_granularities[]")).toEqual(["segment", "word"]);
+    expect(requestForm(fetcher).get("timestamp_granularities")).toBeNull();
+
+    expect(() =>
+      groqcloudActionHandlers.create_audio_transcription(
+        {
+          model: "whisper-large-v3",
+          file: { name: "a.mp3", content_base64: audioBase64 },
+          timestamp_granularities: ["word"],
+        },
+        createContext(jsonFetcher()),
+      ),
+    ).toThrow("timestamp_granularities requires response_format=verbose_json");
+  });
+
+  it("offers turbo for transcription but not for translation", () => {
+    const transcription = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
+    const translation = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_translation")!;
+    const file = { url: "https://example.com/a.mp3" };
+
+    expect(validateActionInput(transcription, { model: "whisper-large-v3-turbo", file }).valid).toBe(true);
+    expect(validateActionInput(translation, { model: "whisper-large-v3", file }).valid).toBe(true);
+    expect(validateActionInput(translation, { model: "whisper-large-v3-turbo", file }).valid).toBe(false);
+  });
+
+  it("requires a name alongside inline audio content", () => {
+    const action = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
+
+    expect(
+      validateActionInput(action, { model: "whisper-large-v3", file: { name: "a.mp3", content_base64: audioBase64 } })
+        .valid,
+    ).toBe(true);
+    expect(
+      validateActionInput(action, { model: "whisper-large-v3", file: { content_base64: audioBase64 } }).valid,
+    ).toBe(false);
+    expect(
+      validateActionInput(action, {
+        model: "whisper-large-v3",
+        file: { name: "a.mp3", content_base64: audioBase64, url: "https://example.com/a.mp3" },
+      }).valid,
+    ).toBe(false);
+  });
+
+  it("rejects empty audio urls in the action schema", () => {
+    const action = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
+
+    expect(validateActionInput(action, { model: "whisper-large-v3", file: { url: "" } }).valid).toBe(false);
+  });
+});