Skip to content

Commit e63f7fa

Browse files
nicremonicremo
committed
feat: cloud LLM rewrite via Groq (GPT-OSS 20B at 1000 t/s)
- Cloud rewrite as primary, Ollama as fallback (same pattern as transcription) - Default model: openai/gpt-oss-20b (1000 tokens/sec, practically free) - 5 cloud models available: GPT-OSS 20B/120B, Qwen3 32B, Llama 3.3 70B, Llama 3.1 8B - Cloud/Local toggle on Models page under Text Enhancement - Auto-fallback to local Ollama when cloud is unavailable - Ollama no longer required when rewrite mode is Cloud - Same API key and base URL as transcription (Groq)
1 parent cf3e7b3 commit e63f7fa

5 files changed

Lines changed: 200 additions & 35 deletions

File tree

‎src/main/cloud-rewrite.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
const REWRITE_TIMEOUT_MS = 30_000;
2+
3+
function buildChatUrl(baseUrl: string): string {
4+
const base = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
5+
return `${base}/v1/chat/completions`;
6+
}
7+
8+
function stripThinkingTags(text: string): string {
9+
return text
10+
.replace(/<think>[\s\S]*?<\/think>/g, '')
11+
.replace(/<think>[\s\S]*$/g, '')
12+
.trim();
13+
}
14+
15+
function cleanRewriteOutput(content: string | undefined, rawText: string): string {
16+
const trimmed = stripThinkingTags(content ?? '').trim();
17+
if (!trimmed) {
18+
return rawText;
19+
}
20+
21+
const stripped =
22+
trimmed.startsWith('"') && trimmed.endsWith('"')
23+
? trimmed.slice(1, -1).trim()
24+
: trimmed.startsWith("'") && trimmed.endsWith("'")
25+
? trimmed.slice(1, -1).trim()
26+
: trimmed;
27+
28+
const metaLeadPattern =
29+
/^(sure|here(?:'s| is)|rewritten|revised|updated|i changed|i rewrote|i made|this is)/i;
30+
31+
if (!metaLeadPattern.test(stripped)) {
32+
return stripped;
33+
}
34+
35+
const colonIndex = stripped.indexOf(':');
36+
if (colonIndex >= 0) {
37+
const afterColon = stripped.slice(colonIndex + 1).trim();
38+
if (afterColon) {
39+
return afterColon;
40+
}
41+
}
42+
43+
return stripped;
44+
}
45+
46+
export async function rewriteWithCloud(
47+
baseUrl: string,
48+
apiKey: string,
49+
model: string,
50+
systemPrompt: string,
51+
rawText: string,
52+
): Promise<string> {
53+
const controller = new AbortController();
54+
const timeout = setTimeout(() => controller.abort(), REWRITE_TIMEOUT_MS);
55+
56+
try {
57+
const response = await fetch(buildChatUrl(baseUrl), {
58+
method: 'POST',
59+
headers: {
60+
'Content-Type': 'application/json',
61+
Authorization: `Bearer ${apiKey}`,
62+
},
63+
body: JSON.stringify({
64+
model,
65+
temperature: 0,
66+
max_tokens: 2048,
67+
messages: [
68+
{ role: 'system', content: systemPrompt },
69+
{
70+
role: 'user',
71+
content: [
72+
'Rewrite the dictated text below.',
73+
'If the speaker corrected themselves or changed their mind, use only their final intent.',
74+
'Reply with only the final rewritten text — no preface, explanation, labels, or quotation marks.',
75+
'',
76+
'<dictation>',
77+
rawText,
78+
'</dictation>',
79+
].join('\n'),
80+
},
81+
],
82+
}),
83+
signal: controller.signal,
84+
});
85+
86+
if (!response.ok) {
87+
const body = await response.text().catch(() => '');
88+
throw new Error(`Cloud rewrite failed (${response.status}): ${body}`);
89+
}
90+
91+
const payload = (await response.json()) as {
92+
choices?: Array<{ message?: { content?: string } }>;
93+
};
94+
95+
const content = payload.choices?.[0]?.message?.content;
96+
return cleanRewriteOutput(content, rawText);
97+
} finally {
98+
clearTimeout(timeout);
99+
}
100+
}

‎src/main/defaults.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ export function createDefaultSettings(): AppSettings {
2929
whisperLabel: RECOMMENDED_WHISPER_LABEL,
3030
ollamaBaseUrl: 'http://127.0.0.1:11434',
3131
textModel: RECOMMENDED_TEXT_MODEL,
32+
rewriteMode: 'cloud',
33+
cloudRewriteModel: 'openai/gpt-oss-20b',
3234
styleMode: 'conversation',
3335
enhancementLevel: 'medium',
3436
transcriptionMode: 'auto',

‎src/main/dictation.ts‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { getApiKey, isApiKeySet } from './api-key';
55
import { resolveStyleForApp } from './app-rules';
66
import { CloudTranscriptionError, transcribeWithCloud } from './cloud-transcription';
77
import { buildDictionaryContext, buildWhisperPrompt } from './dictionary';
8+
import { rewriteWithCloud } from './cloud-rewrite';
89
import { getEnhancementPrompt } from './prompts';
910
import { rewriteWithOllama } from './ollama';
1011
import { getFocusInfo, triggerPaste } from './native-helper';
@@ -171,36 +172,66 @@ export async function processDictationAudio({
171172
throw new Error('No speech was detected in the recording.');
172173
}
173174

174-
setStatus({
175-
phase: 'rewriting',
176-
title: 'Polishing',
177-
detail: `${settings.textModel} is applying the selected rewrite level. The first request can take a moment while Ollama warms the model.`,
178-
preview: rawText,
179-
rawText,
180-
});
181-
175+
const enhancementPrompt = getEnhancementPrompt(resolved.styleMode, resolved.enhancementLevel, dictionaryContext, settings.cloudLanguage);
182176
let finalText = rawText;
183177
let usedRewriteFallback = false;
184178

185-
try {
186-
finalText = await rewriteWithOllama(
187-
settings.ollamaBaseUrl,
188-
settings.textModel,
189-
getEnhancementPrompt(resolved.styleMode, resolved.enhancementLevel, dictionaryContext, settings.cloudLanguage),
179+
if (settings.rewriteMode === 'cloud' && isApiKeySet(settings)) {
180+
setStatus({
181+
phase: 'rewriting',
182+
title: 'Polishing',
183+
detail: `${settings.cloudRewriteModel} is polishing your text via cloud.`,
184+
preview: rawText,
190185
rawText,
191-
);
192-
} catch (error) {
193-
usedRewriteFallback = true;
186+
});
187+
188+
const apiKey = getApiKey(settings);
189+
if (apiKey) {
190+
try {
191+
finalText = await rewriteWithCloud(
192+
settings.cloudApiBaseUrl,
193+
apiKey,
194+
settings.cloudRewriteModel,
195+
enhancementPrompt,
196+
rawText,
197+
);
198+
} catch (cloudError) {
199+
console.warn('[openwhisp] Cloud rewrite failed, falling back to Ollama:', cloudError instanceof Error ? cloudError.message : cloudError);
200+
try {
201+
setStatus({
202+
phase: 'rewriting',
203+
title: 'Polishing locally',
204+
detail: `Cloud unavailable, using ${settings.textModel} via Ollama.`,
205+
preview: rawText,
206+
rawText,
207+
});
208+
finalText = await rewriteWithOllama(settings.ollamaBaseUrl, settings.textModel, enhancementPrompt, rawText);
209+
} catch {
210+
usedRewriteFallback = true;
211+
}
212+
}
213+
}
214+
} else {
194215
setStatus({
195-
phase: 'error',
196-
title: 'Rewrite unavailable',
197-
detail:
198-
error instanceof Error
199-
? `${error.message} OpenWhisp will use the raw transcription for now.`
200-
: 'OpenWhisp could not finish the rewrite pass, so it will use the raw transcription.',
216+
phase: 'rewriting',
217+
title: 'Polishing',
218+
detail: `${settings.textModel} is applying the selected rewrite level.`,
201219
preview: rawText,
202220
rawText,
203221
});
222+
223+
try {
224+
finalText = await rewriteWithOllama(settings.ollamaBaseUrl, settings.textModel, enhancementPrompt, rawText);
225+
} catch (error) {
226+
usedRewriteFallback = true;
227+
setStatus({
228+
phase: 'error',
229+
title: 'Rewrite unavailable',
230+
detail: error instanceof Error ? error.message : 'Could not rewrite.',
231+
preview: rawText,
232+
rawText,
233+
});
234+
}
204235
}
205236

206237
console.log('[openwhisp:dictation] final', { rewriteFallback: usedRewriteFallback, text: finalText });

‎src/renderer/App.tsx‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,9 @@ export function App() {
219219
await window.openWhisp.showMainWindow();
220220
return;
221221
}
222-
if (current.settings.enhancementLevel !== 'none' && (!current.ollamaReachable || !modelInstalled)) {
223-
pushStatus({ phase: 'error', title: 'Text model unavailable', detail: 'Start Ollama and install the rewrite model, or set enhancement to No Filter.' });
222+
const needsOllama = current.settings.enhancementLevel !== 'none' && current.settings.rewriteMode === 'local';
223+
if (needsOllama && (!current.ollamaReachable || !modelInstalled)) {
224+
pushStatus({ phase: 'error', title: 'Text model unavailable', detail: 'Start Ollama, install the rewrite model, or switch to Cloud rewrite.' });
224225
await window.openWhisp.showMainWindow();
225226
return;
226227
}
@@ -846,18 +847,44 @@ function ModelsPage({ bootstrap, busyAction, onAction }: { bootstrap: BootstrapS
846847
<button className="btn btn-link" onClick={() => void onAction('refresh', () => window.openWhisp.refreshOllama())}>Refresh</button>
847848
</div>
848849
<div className="setting-row">
849-
<label className="setting-label" htmlFor="model-select">Rewrite model</label>
850-
<select id="model-select" className="setting-select" value={bootstrap.settings.textModel} onChange={(e) => void onAction('settings', () => window.openWhisp.updateSettings({ textModel: e.target.value }))}>
851-
{bootstrap.ollamaModels.length === 0
852-
? <option value={bootstrap.settings.textModel}>{bootstrap.settings.textModel}</option>
853-
: bootstrap.ollamaModels.map((m) => <option key={m.name} value={m.name}>{m.name} ({formatBytes(m.size)})</option>)
854-
}
855-
</select>
850+
<label className="setting-label">Rewrite via</label>
851+
<div className="source-selector">
852+
<button className={`source-btn${bootstrap.settings.rewriteMode === 'cloud' ? ' source-btn-active' : ''}`} onClick={() => void onAction('settings', () => window.openWhisp.updateSettings({ rewriteMode: 'cloud' }))}>Cloud</button>
853+
<button className={`source-btn${bootstrap.settings.rewriteMode === 'local' ? ' source-btn-active' : ''}`} onClick={() => void onAction('settings', () => window.openWhisp.updateSettings({ rewriteMode: 'local' }))}>Local</button>
854+
</div>
856855
</div>
857-
{!bootstrap.recommendedModelInstalled && bootstrap.ollamaReachable && (
858-
<button className="btn btn-sm btn-primary" style={{ marginTop: 10 }} disabled={busyAction === 'model'} onClick={() => void onAction('model', () => window.openWhisp.pullRecommendedModel())}>
859-
{busyAction === 'model' ? 'Downloading...' : `Download ${RECOMMENDED_TEXT_MODEL}`}
860-
</button>
856+
<div className="setting-hint">
857+
{bootstrap.settings.rewriteMode === 'cloud' ? 'Fast rewrite via Groq cloud API. Falls back to Ollama when offline.' : 'Local rewrite via Ollama. No data leaves your Mac.'}
858+
</div>
859+
{bootstrap.settings.rewriteMode === 'cloud' && (
860+
<div className="setting-row" style={{ marginTop: 12 }}>
861+
<label className="setting-label" htmlFor="cloud-rewrite-model">Cloud model</label>
862+
<select id="cloud-rewrite-model" className="setting-select" value={bootstrap.settings.cloudRewriteModel} onChange={(e) => void onAction('settings', () => window.openWhisp.updateSettings({ cloudRewriteModel: e.target.value }))}>
863+
<option value="openai/gpt-oss-20b">GPT-OSS 20B (1000 t/s)</option>
864+
<option value="openai/gpt-oss-120b">GPT-OSS 120B (500 t/s)</option>
865+
<option value="qwen/qwen3-32b">Qwen3 32B (400 t/s)</option>
866+
<option value="llama-3.3-70b-versatile">Llama 3.3 70B (280 t/s)</option>
867+
<option value="llama-3.1-8b-instant">Llama 3.1 8B (560 t/s)</option>
868+
</select>
869+
</div>
870+
)}
871+
{bootstrap.settings.rewriteMode === 'local' && (
872+
<>
873+
<div className="setting-row" style={{ marginTop: 12 }}>
874+
<label className="setting-label" htmlFor="model-select">Ollama model</label>
875+
<select id="model-select" className="setting-select" value={bootstrap.settings.textModel} onChange={(e) => void onAction('settings', () => window.openWhisp.updateSettings({ textModel: e.target.value }))}>
876+
{bootstrap.ollamaModels.length === 0
877+
? <option value={bootstrap.settings.textModel}>{bootstrap.settings.textModel}</option>
878+
: bootstrap.ollamaModels.map((m) => <option key={m.name} value={m.name}>{m.name} ({formatBytes(m.size)})</option>)
879+
}
880+
</select>
881+
</div>
882+
{!bootstrap.recommendedModelInstalled && bootstrap.ollamaReachable && (
883+
<button className="btn btn-sm btn-primary" style={{ marginTop: 10 }} disabled={busyAction === 'model'} onClick={() => void onAction('model', () => window.openWhisp.pullRecommendedModel())}>
884+
{busyAction === 'model' ? 'Downloading...' : `Download ${RECOMMENDED_TEXT_MODEL}`}
885+
</button>
886+
)}
887+
</>
861888
)}
862889
</div>
863890

‎src/shared/types.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
export type EnhancementLevel = 'none' | 'soft' | 'medium' | 'high';
22
export type StyleMode = 'conversation' | 'vibe-coding';
33
export type TranscriptionMode = 'auto' | 'cloud' | 'local';
4+
export type RewriteMode = 'cloud' | 'local';
45
export type CloudTranscriptionModel = 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | 'whisper-1' | 'whisper-large-v3' | 'whisper-large-v3-turbo' | 'distil-whisper-large-v3-en';
56

67
export type OverlayPhase =
@@ -36,6 +37,8 @@ export interface AppSettings {
3637
whisperLabel: string;
3738
ollamaBaseUrl: string;
3839
textModel: string;
40+
rewriteMode: RewriteMode;
41+
cloudRewriteModel: string;
3942
styleMode: StyleMode;
4043
enhancementLevel: EnhancementLevel;
4144
transcriptionMode: TranscriptionMode;
@@ -119,6 +122,8 @@ export interface UpdateSettingsInput {
119122
cloudLanguage?: string;
120123
openaiApiKey?: string;
121124
textModel?: string;
125+
rewriteMode?: RewriteMode;
126+
cloudRewriteModel?: string;
122127
ollamaBaseUrl?: string;
123128
storageDirectory?: string;
124129
autoPaste?: boolean;

0 commit comments

Comments
 (0)