Skip to content

Commit 36a1d4b

Browse files
rupaut98l1shen
andauthored
feat(groqcloud): add audio transcription and translation actions (oomol-lab#256)
The `groqcloud` provider currently exposes chat completions and model listing only, so Groq's speech-to-text endpoints are unreachable — the provider has no proxy fallback either. | Action | Endpoint | |---|---| | `create_audio_transcription` | `POST /audio/transcriptions` | | `create_audio_translation` | `POST /audio/translations` | Audio is supplied inline as base64, or as a public URL that GroqCloud fetches itself through its native `url` field. Caller URLs are checked with `assertPublicHttpUrl` before being forwarded. The size cap applies to inline uploads; GroqCloud enforces its own limit on URLs it fetches. Translation is restricted to `whisper-large-v3`. Requesting `whisper-large-v3-turbo` there returns `The model 'whisper-large-v3-turbo' does not support 'translate'`, so the restriction is in the schema rather than surfacing as a runtime error. Exercised against the live API with a real key: inline base64 upload, forwarded URL, `verbose_json` with word-level `timestamp_granularities[]`, `language`/`prompt`/`temperature` passthrough, and both endpoints. --------- Co-authored-by: l1shen <648952316@qq.com>
1 parent 368eba8 commit 36a1d4b

3 files changed

Lines changed: 422 additions & 5 deletions

File tree

src/providers/groqcloud/actions.ts

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,12 @@ import { defineProviderAction } from "../../core/provider-definition.ts";
55

66
const service = "groqcloud";
77

8-
export type GroqcloudActionName = "list_models" | "get_model" | "create_chat_completion";
8+
export type GroqcloudActionName =
9+
| "list_models"
10+
| "get_model"
11+
| "create_chat_completion"
12+
| "create_audio_transcription"
13+
| "create_audio_translation";
914

1015
const nullSchema: JsonSchema = { type: "null", description: "Null value." };
1116
const unknownJsonValueSchema = s.unknown("Any JSON value accepted by the upstream API.");
@@ -119,6 +124,93 @@ const chatCompletionOutputSchema = s.looseObject("The response payload for a Gro
119124
system_fingerprint: s.string("The backend system fingerprint for the completion."),
120125
});
121126

127+
const audioModelSchema = s.stringEnum("The GroqCloud speech-to-text model identifier.", [
128+
"whisper-large-v3",
129+
"whisper-large-v3-turbo",
130+
]);
131+
const translationModelSchema = s.stringEnum(
132+
"The GroqCloud speech-to-text model identifier. Only whisper-large-v3 supports translation.",
133+
["whisper-large-v3"],
134+
);
135+
const audioFileSchema: JsonSchema = {
136+
...s.object(
137+
"The audio source. Provide url for GroqCloud to fetch the audio, or content_base64 to upload the bytes inline.",
138+
{
139+
name: s.nonEmptyString(
140+
"The file name reported to GroqCloud, including the audio file extension. Required with content_base64.",
141+
),
142+
mimetype: s.string("The MIME type of the audio file, such as audio/mpeg."),
143+
url: s.nonEmptyString("A public URL that GroqCloud downloads the audio from."),
144+
content_base64: s.nonEmptyString("The base64-encoded audio content to upload."),
145+
},
146+
{ optional: ["name", "mimetype", "url", "content_base64"] },
147+
),
148+
anyOf: [{ required: ["url"] }, { required: ["content_base64", "name"] }],
149+
not: { required: ["url", "content_base64"] },
150+
};
151+
const audioResponseFormatSchema = s.stringEnum(
152+
"The transcript format to return. This connector returns structured payloads, so the plain text format is not offered.",
153+
["json", "verbose_json"],
154+
);
155+
const audioTemperatureSchema = s.number("The sampling temperature applied to the transcription.", {
156+
minimum: 0,
157+
maximum: 1,
158+
});
159+
const audioPromptSchema = s.string("Optional context or style guidance for the transcript, limited to 224 tokens.");
160+
const transcriptionInputSchema = s.object(
161+
"The input payload for transcribing audio with GroqCloud.",
162+
{
163+
model: audioModelSchema,
164+
file: audioFileSchema,
165+
language: s.string("The ISO-639-1 code of the spoken language, such as en, which improves accuracy and latency."),
166+
prompt: audioPromptSchema,
167+
response_format: audioResponseFormatSchema,
168+
temperature: audioTemperatureSchema,
169+
timestamp_granularities: s.array(
170+
"The timestamp detail to include. Requires response_format to be verbose_json.",
171+
s.stringEnum("A timestamp granularity.", ["word", "segment"]),
172+
{ minItems: 1 },
173+
),
174+
},
175+
{ required: ["model", "file"] },
176+
);
177+
const translationInputSchema = s.object(
178+
"The input payload for translating audio into English with GroqCloud.",
179+
{
180+
model: translationModelSchema,
181+
file: audioFileSchema,
182+
prompt: audioPromptSchema,
183+
response_format: audioResponseFormatSchema,
184+
temperature: audioTemperatureSchema,
185+
},
186+
{ required: ["model", "file"] },
187+
);
188+
const audioSegmentSchema = s.looseObject("A transcribed segment of the audio.", {
189+
id: s.integer("The segment index."),
190+
seek: s.integer("The seek offset of the segment."),
191+
start: s.number("The segment start time in seconds."),
192+
end: s.number("The segment end time in seconds."),
193+
text: s.string("The transcribed text for the segment."),
194+
tokens: s.array("The token identifiers for the segment.", s.integer("A token identifier.")),
195+
temperature: s.number("The sampling temperature used for the segment."),
196+
avg_logprob: s.number("The average log probability of the segment."),
197+
compression_ratio: s.number("The compression ratio of the segment."),
198+
no_speech_prob: s.number("The probability that the segment contains no speech."),
199+
});
200+
const audioWordSchema = s.looseObject("A transcribed word with timestamps.", {
201+
word: s.string("The transcribed word."),
202+
start: s.number("The word start time in seconds."),
203+
end: s.number("The word end time in seconds."),
204+
});
205+
const audioTranscriptOutputSchema = s.looseObject("The transcript payload returned by GroqCloud.", {
206+
text: s.string("The full transcript text."),
207+
language: s.string("The detected or requested language of the audio."),
208+
duration: s.number("The audio duration in seconds."),
209+
segments: s.array("The transcribed segments, returned for the verbose_json format.", audioSegmentSchema),
210+
words: s.array("The transcribed words, returned when word timestamp granularity is requested.", audioWordSchema),
211+
x_groq: jsonObjectSchema,
212+
});
213+
122214
export const groqcloudActions: ActionDefinition[] = [
123215
defineProviderAction(service, {
124216
name: "list_models",
@@ -138,4 +230,18 @@ export const groqcloudActions: ActionDefinition[] = [
138230
inputSchema: chatCompletionInputSchema,
139231
outputSchema: chatCompletionOutputSchema,
140232
}),
233+
defineProviderAction(service, {
234+
name: "create_audio_transcription",
235+
description:
236+
"Transcribe an audio file into text in its original language using a GroqCloud Whisper model. Supply the audio inline as base64 or as a public URL that GroqCloud downloads.",
237+
inputSchema: transcriptionInputSchema,
238+
outputSchema: audioTranscriptOutputSchema,
239+
}),
240+
defineProviderAction(service, {
241+
name: "create_audio_translation",
242+
description:
243+
"Translate an audio file into English text using a GroqCloud Whisper model. Supply the audio inline as base64 or as a public URL that GroqCloud downloads.",
244+
inputSchema: translationInputSchema,
245+
outputSchema: audioTranscriptOutputSchema,
246+
}),
141247
];
Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
import { describe, expect, it, vi } from "vitest";
2+
import { validateActionInput } from "../../core/validation.ts";
3+
import { groqcloudActions } from "./actions.ts";
4+
import { groqcloudActionHandlers } from "./executors.ts";
5+
6+
const apiBaseUrl = "https://api.groq.com/openai/v1";
7+
const audioBase64 = Buffer.from("fake-audio-bytes").toString("base64");
8+
9+
function createContext(fetcher: typeof fetch) {
10+
return {
11+
apiKey: "test-key",
12+
fetcher,
13+
};
14+
}
15+
16+
function jsonFetcher(payload: Record<string, unknown> = {}): typeof fetch {
17+
return vi.fn(async () => Response.json(payload)) as typeof fetch;
18+
}
19+
20+
function requestInit(fetcher: typeof fetch): RequestInit {
21+
return vi.mocked(fetcher).mock.calls[0]![1] as RequestInit;
22+
}
23+
24+
function requestForm(fetcher: typeof fetch): FormData {
25+
return requestInit(fetcher).body as FormData;
26+
}
27+
28+
describe("GroqCloud audio transcription", () => {
29+
it("uploads inline base64 audio as multipart form data", async () => {
30+
const fetcher = jsonFetcher({ text: "hello" });
31+
32+
await groqcloudActionHandlers.create_audio_transcription(
33+
{
34+
model: "whisper-large-v3-turbo",
35+
file: { name: "meeting.mp3", mimetype: "audio/mpeg", content_base64: audioBase64 },
36+
language: "en",
37+
temperature: 0,
38+
},
39+
createContext(fetcher),
40+
);
41+
42+
expect(fetcher).toHaveBeenCalledWith(
43+
`${apiBaseUrl}/audio/transcriptions`,
44+
expect.objectContaining({ method: "POST" }),
45+
);
46+
47+
const form = requestForm(fetcher);
48+
const uploaded = form.get("file") as File;
49+
expect(uploaded.name).toBe("meeting.mp3");
50+
expect(uploaded.type).toBe("audio/mpeg");
51+
expect(await uploaded.text()).toBe("fake-audio-bytes");
52+
expect(form.get("model")).toBe("whisper-large-v3-turbo");
53+
expect(form.get("language")).toBe("en");
54+
expect(form.get("temperature")).toBe("0");
55+
});
56+
57+
it("omits the JSON content type so the multipart boundary is preserved", async () => {
58+
const fetcher = jsonFetcher({ text: "hello" });
59+
60+
await groqcloudActionHandlers.create_audio_transcription(
61+
{ model: "whisper-large-v3", file: { name: "a.mp3", content_base64: audioBase64 } },
62+
createContext(fetcher),
63+
);
64+
65+
const headers = requestInit(fetcher).headers as Record<string, string>;
66+
expect(headers["content-type"]).toBeUndefined();
67+
expect(headers.authorization).toBe("Bearer test-key");
68+
});
69+
70+
it("forwards a public url instead of downloading the audio", async () => {
71+
const fetcher = jsonFetcher({ text: "hello" });
72+
73+
await groqcloudActionHandlers.create_audio_translation(
74+
{ model: "whisper-large-v3", file: { url: "https://example.com/clip.mp3" } },
75+
createContext(fetcher),
76+
);
77+
78+
expect(vi.mocked(fetcher).mock.calls).toHaveLength(1);
79+
expect(String(vi.mocked(fetcher).mock.calls[0]![0])).toBe(`${apiBaseUrl}/audio/translations`);
80+
const form = requestForm(fetcher);
81+
expect(form.get("url")).toBe("https://example.com/clip.mp3");
82+
expect(form.get("file")).toBeNull();
83+
});
84+
85+
it("rejects private and loopback audio urls before forwarding them", () => {
86+
const fetcher = jsonFetcher();
87+
88+
expect(() =>
89+
groqcloudActionHandlers.create_audio_transcription(
90+
{ model: "whisper-large-v3", file: { url: "http://127.0.0.1/internal.mp3" } },
91+
createContext(fetcher),
92+
),
93+
).toThrow();
94+
95+
expect(fetcher).not.toHaveBeenCalled();
96+
});
97+
98+
it("requires exactly one audio source", () => {
99+
const fetcher = jsonFetcher();
100+
101+
expect(() =>
102+
groqcloudActionHandlers.create_audio_transcription({ model: "whisper-large-v3" }, createContext(fetcher)),
103+
).toThrow("file is required");
104+
105+
expect(() =>
106+
groqcloudActionHandlers.create_audio_transcription(
107+
{ model: "whisper-large-v3", file: { name: "a.mp3" } },
108+
createContext(fetcher),
109+
),
110+
).toThrow("file must include url or content_base64");
111+
112+
expect(() =>
113+
groqcloudActionHandlers.create_audio_transcription(
114+
{
115+
model: "whisper-large-v3",
116+
file: { name: "a.mp3", content_base64: audioBase64, url: "https://example.com/clip.mp3" },
117+
},
118+
createContext(fetcher),
119+
),
120+
).toThrow("provide only one of file.url or file.content_base64");
121+
122+
expect(fetcher).not.toHaveBeenCalled();
123+
});
124+
125+
it("rejects malformed base64 audio content", () => {
126+
const fetcher = jsonFetcher();
127+
128+
expect(() =>
129+
groqcloudActionHandlers.create_audio_transcription(
130+
{ model: "whisper-large-v3", file: { name: "a.mp3", content_base64: "not*base64" } },
131+
createContext(fetcher),
132+
),
133+
).toThrow("file.content_base64 must be valid base64");
134+
});
135+
136+
it("rejects inline audio above GroqCloud's attachment limit", () => {
137+
const fetcher = jsonFetcher();
138+
const attachmentMaxBytes = 25 * 1024 * 1024;
139+
const oversizedAudioBase64 = Buffer.alloc(attachmentMaxBytes + 1).toString("base64");
140+
141+
expect(() =>
142+
groqcloudActionHandlers.create_audio_transcription(
143+
{ model: "whisper-large-v3", file: { name: "a.mp3", content_base64: oversizedAudioBase64 } },
144+
createContext(fetcher),
145+
),
146+
).toThrow(`file.content_base64 exceeds ${attachmentMaxBytes} bytes`);
147+
148+
expect(fetcher).not.toHaveBeenCalled();
149+
});
150+
151+
it("repeats timestamp granularities as an array field and requires verbose_json", async () => {
152+
const fetcher = jsonFetcher({ text: "hello" });
153+
154+
await groqcloudActionHandlers.create_audio_transcription(
155+
{
156+
model: "whisper-large-v3",
157+
file: { name: "a.mp3", content_base64: audioBase64 },
158+
response_format: "verbose_json",
159+
timestamp_granularities: ["segment", "word"],
160+
},
161+
createContext(fetcher),
162+
);
163+
164+
expect(requestForm(fetcher).getAll("timestamp_granularities[]")).toEqual(["segment", "word"]);
165+
expect(requestForm(fetcher).get("timestamp_granularities")).toBeNull();
166+
167+
expect(() =>
168+
groqcloudActionHandlers.create_audio_transcription(
169+
{
170+
model: "whisper-large-v3",
171+
file: { name: "a.mp3", content_base64: audioBase64 },
172+
timestamp_granularities: ["word"],
173+
},
174+
createContext(jsonFetcher()),
175+
),
176+
).toThrow("timestamp_granularities requires response_format=verbose_json");
177+
});
178+
179+
it("offers turbo for transcription but not for translation", () => {
180+
const transcription = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
181+
const translation = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_translation")!;
182+
const file = { url: "https://example.com/a.mp3" };
183+
184+
expect(validateActionInput(transcription, { model: "whisper-large-v3-turbo", file }).valid).toBe(true);
185+
expect(validateActionInput(translation, { model: "whisper-large-v3", file }).valid).toBe(true);
186+
expect(validateActionInput(translation, { model: "whisper-large-v3-turbo", file }).valid).toBe(false);
187+
});
188+
189+
it("requires a name alongside inline audio content", () => {
190+
const action = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
191+
192+
expect(
193+
validateActionInput(action, { model: "whisper-large-v3", file: { name: "a.mp3", content_base64: audioBase64 } })
194+
.valid,
195+
).toBe(true);
196+
expect(
197+
validateActionInput(action, { model: "whisper-large-v3", file: { content_base64: audioBase64 } }).valid,
198+
).toBe(false);
199+
expect(
200+
validateActionInput(action, {
201+
model: "whisper-large-v3",
202+
file: { name: "a.mp3", content_base64: audioBase64, url: "https://example.com/a.mp3" },
203+
}).valid,
204+
).toBe(false);
205+
});
206+
207+
it("rejects empty audio urls in the action schema", () => {
208+
const action = groqcloudActions.find((action) => action.id === "groqcloud.create_audio_transcription")!;
209+
210+
expect(validateActionInput(action, { model: "whisper-large-v3", file: { url: "" } }).valid).toBe(false);
211+
});
212+
});

0 commit comments

Comments
 (0)