-
Notifications
You must be signed in to change notification settings - Fork 78
Expand file tree
/
Copy pathscripts.specs.ts
More file actions
371 lines (347 loc) · 13.7 KB
/
Copy pathscripts.specs.ts
File metadata and controls
371 lines (347 loc) · 13.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
/**
* The `scripts` module's specs — data only, no implementation.
*
* Split out so a belt can be assembled synchronously: the registry's eager
* spec table imports this file, never `scripts.ts`, so nothing the
* implementations pull in reaches the entry graph. `scripts.ts` imports these
* back and attaches each to its implementation, so there is one spec object
* behind both halves.
*/
import type { CapabilitySpec } from "./types.js";
import type { JsonSchema } from "@nodetool-ai/runtime";
export const DEFAULT_CONCURRENCY = 3;
export const MAX_CONCURRENCY = 8;
export const DEFAULT_ASR_PROVIDER = "openai";
export const DEFAULT_ASR_MODEL = "whisper-1";
export const LINE_TARGETS_SCHEMA = {
type: "array" as const,
items: { type: "string" as const },
description:
"Lines to voice, each a line id, 0-based reading-order index, or the " +
"line's exact text. Omit to voice every line that is unvoiced or stale."
};
export const LIST_SCRIPTS_SCHEMA: JsonSchema = {
type: "object",
properties: {
limit: {
type: "number",
description: "Max scripts to return (default 20)."
}
}
};
export const GET_SCRIPT_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." }
},
required: ["script_id"]
};
export const VOICE_SCRIPT_LINES_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." },
targets: LINE_TARGETS_SCHEMA,
provider: {
type: "string",
description:
"TTS provider id (from find_model, capability=text_to_speech). Overrides every line's own voice."
},
model: {
type: "string",
description:
"TTS model id (from find_model). Overrides every line's own voice."
},
voice: {
type: "string",
description:
"Voice id for the override. Required when provider/model are passed."
},
speed: {
type: "number",
description:
"Speech speed multiplier passed to the provider (e.g. 0.25–4.0)."
},
transcribe: {
type: "boolean",
description:
"Transcribe each take for word timings (default true). Set false to skip the extra ASR call."
},
asr_provider: {
type: "string",
description: `Provider for word timings (default ${DEFAULT_ASR_PROVIDER}).`
},
asr_model: {
type: "string",
description: `Model for word timings (default ${DEFAULT_ASR_MODEL}).`
},
concurrency: {
type: "number",
description: `Lines voiced in parallel (default ${DEFAULT_CONCURRENCY}, max ${MAX_CONCURRENCY}).`
}
},
required: ["script_id"]
};
export const ASSEMBLE_SCRIPT_TIMELINE_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." },
name: {
type: "string",
description: "Name for the sequence. Defaults to the script's name."
},
fps: { type: "number", description: "Frame rate (default 30)." }
},
required: ["script_id"]
};
export const EDIT_SCRIPT_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." },
ops: {
type: "array",
description:
'Operations in order. Each is {"op": <name>, ...arguments}: ' +
"set_setup {stage?, brief?, format?, length_seconds?, pace?, language?}, " +
"add_speaker {name, color?, provider?, model?, voice?, entityId?}, " +
"set_speaker {target, name?, color?, entityId?}, " +
"set_speaker_voice {target, provider, model, voice, settings?}, " +
"remove_speaker {target}, add_section {title?}, " +
"add_line {text, speaker?, section?, direction?, pause_after_ms?, index?}, " +
"set_line_text {target, text}, set_line_speaker {target, speaker}, " +
"remove_line {target}. `set_setup` writes the guided flow's own state — " +
"the stage the creator is standing at (idea, format, review, voices, " +
"done), the brief the script is written from, the format (voiceover, " +
"dialogue, interview, ad-read, tutorial), the length in seconds, the " +
"reading pace and the language; write_script reads them. `entityId` is an asset id from the ingredient " +
"library (list_entities) — when set the speaker is that character/entity " +
"and storyboard renders can season prompts with it. A line `target` is " +
"its id, its 0-based index across the script, or its exact text; a " +
"speaker `target` is its id or name. `set_speaker` renames a cast " +
"member; the op that gives a LINE its speaker is `set_line_speaker`. An " +
"argument an op does not take is refused rather than ignored, so a " +
"misspelled key is reported instead of silently dropping its value.",
items: { type: "object" }
}
},
required: ["script_id", "ops"]
};
export const WRITE_SCRIPT_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." },
provider: {
type: "string",
description:
"Language-model provider id (from find_model) for the writer."
},
model: {
type: "string",
description: "Language model id (from find_model) for the writer."
},
rewrite: {
type: "boolean",
description:
"Rewrite the script that is already there instead of writing a new " +
"one. Lines the rewrite keeps come back with their own ids, so their " +
"takes and storyboard links survive."
},
imported_text: {
type: "string",
description:
"Words the user supplied — pasted, or extracted from a file. They are " +
"split into lines and given speakers, never rewritten. Overrides the " +
"brief as the source of the words."
}
},
required: ["script_id"]
};
export const DERIVE_STORYBOARD_SCHEMA: JsonSchema = {
type: "object",
properties: {
script_id: { type: "string", description: "Script id." },
provider: {
type: "string",
description:
"Language-model provider id (from find_model) for the director pass " +
"that writes each shot's visuals. Omit provider and model together to " +
"get the deterministic scaffold only."
},
model: {
type: "string",
description:
"Language model id (from find_model) for the director pass. Omit " +
"provider and model together to get the deterministic scaffold only."
},
name: {
type: "string",
description:
"Name for the created storyboard. Defaults to the script's name."
}
},
required: ["script_id"]
};
export const CREATE_SCRIPT_SCHEMA: JsonSchema = {
type: "object",
properties: {
name: { type: "string", description: "Name of the new script." },
project_id: {
type: "string",
description: "Project to create the script in (default 'default')."
},
id: {
type: "string",
description:
"Id to create it under. An id that already exists is returned as-is, " +
"so a retry does not duplicate the script."
}
},
required: ["name"]
};
export const createScriptSpec: CapabilitySpec = {
name: "create_script",
description:
"Create an empty script and return its id. This is the first step of " +
"writing one headlessly: create it, then add speakers and lines with " +
"edit_script, and voice them with voice_script_lines. Use " +
"extract_script_from_storyboard instead when the words already exist as " +
"shot dialogue on a board.",
inputSchema: CREATE_SCRIPT_SCHEMA,
category: "write",
userMessage: (params) => `Creating script ${String(params["name"])}`
};
export const listScriptsSpec: CapabilitySpec = {
name: "list_scripts",
description:
"List the caller's scripts, newest first: id, name, cast size, line count, " +
"how many lines are voiced, and the timeline it was assembled into. Start " +
"here when the user names a script but not its id.",
inputSchema: LIST_SCRIPTS_SCHEMA,
category: "read",
userMessage: () => "Listing scripts"
};
export const getScriptSpec: CapabilitySpec = {
name: "get_script",
description:
"Read one script: its cast with each speaker's voice, and every line in " +
"reading order with its id, index, speaker, text, direction, pause, " +
"effective voice, and voicing status ('draft' = never voiced, 'stale' = " +
"text or voice changed since the take, 'voiced' = up to date, 'no_voice' = " +
"no speaker voice assigned). Call this before voicing — the other tools " +
"address lines by these ids.",
inputSchema: GET_SCRIPT_SCHEMA,
category: "read",
userMessage: (params) => `Reading script ${String(params["script_id"])}`
};
export const voiceScriptLinesSpec: CapabilitySpec = {
name: "voice_script_lines",
description:
"Synthesize speech for a script's lines by calling the TTS provider " +
"directly — no workflow is created or run. Each take is saved as an audio " +
"asset, appended to its line, and made the line's current take (earlier " +
"takes are kept). Omit `targets` to voice every line that is unvoiced or " +
"stale, so a whole script is one call. Each line uses its own voice (its " +
"override, else its speaker's) unless you pass provider/model/voice to " +
"override them all. Word timings are transcribed best-effort so the " +
"assembled timeline carries captions.",
inputSchema: VOICE_SCRIPT_LINES_SCHEMA,
category: "write",
userMessage: (params) => {
const targets = params["targets"];
const count = Array.isArray(targets) ? `${targets.length} ` : "";
return `Voicing ${count}script lines`;
}
};
export const assembleScriptTimelineSpec: CapabilitySpec = {
name: "assemble_script_timeline",
description:
"Lay a script's voiced takes end to end into a voiceover timeline sequence " +
"and save it, without building a workflow. Each clip carries its take's " +
"word timings, the speaker label, and a link back to its script line, so a " +
"later re-voice can round-trip into the cut. Lines with no voiced take are " +
"skipped and listed. Re-running rebuilds the same sequence in place, " +
"keeping clips other surfaces added. Validate the result with " +
"validate_timeline before rendering.",
inputSchema: ASSEMBLE_SCRIPT_TIMELINE_SCHEMA,
category: "write",
userMessage: (params) =>
`Assembling script ${String(params["script_id"])} into a timeline`
};
export const editScriptSpec: CapabilitySpec = {
name: "edit_script",
description:
"Edit a saved script headlessly: write its guided setup (brief, format, " +
"length, pace, language, stage), add cast members and assign their " +
"voices, add sections, and add, rewrite, reassign and remove lines. " +
"Operations run in order against the stored document and the result is " +
"saved; an open editor picks the change up live. Rewriting a line leaves " +
"its takes in place as stale, so voice_script_lines re-records exactly " +
"those. Call get_script first for line and speaker ids.",
inputSchema: EDIT_SCRIPT_SCHEMA,
category: "write",
userMessage: (params) => {
const count = Array.isArray(params["ops"]) ? params["ops"].length : 0;
return `Editing script ${String(params["script_id"])} (${count} ops)`;
}
};
export const writeScriptSpec: CapabilitySpec = {
name: "write_script",
description:
"Write a script's lines from its brief, format and length — set them " +
"first with edit_script's set_setup op — and save the cast and lines it " +
"produces, replacing what was there. Pass rewrite: true to rewrite an " +
"existing script; the lines it keeps keep their ids, so their takes and " +
"storyboard links survive. Words passed as imported_text are only split " +
"into lines and attributed to speakers, never reworded. Records no take: " +
"voice the result with voice_script_lines.",
inputSchema: WRITE_SCRIPT_SCHEMA,
category: "write",
userMessage: (params) =>
`${params["rewrite"] === true ? "Rewriting" : "Writing"} script ${String(params["script_id"])}`
};
export const deriveStoryboardFromScriptSpec: CapabilitySpec = {
name: "derive_storyboard_from_script",
description:
"Create a storyboard whose shots cover this script's lines, linked to it: " +
"one shot per line in reading order, each carrying the line ids it covers " +
"and the projected dialogue/narration. With provider + model, a director " +
"pass writes each shot's action, slug, camera and motion over that " +
"scaffold; a response that drops or reassigns the line links is refused " +
"and retried, and the scaffold is the floor. Without them the shots are " +
"the scaffold alone, status 'planned'. Render it with " +
"render_storyboard_stills.",
inputSchema: DERIVE_STORYBOARD_SCHEMA,
category: "write",
userMessage: (params) =>
`Deriving a storyboard from script ${String(params["script_id"])}`
};
export const deleteScriptSpec: CapabilitySpec = {
name: "delete_script",
description:
"Delete a script you own, together with its saved version history. " +
"This cannot be undone. A script belonging to another user is reported " +
"as missing.",
inputSchema: {
type: "object",
properties: {
script_id: {
type: "string",
description: "The script to delete. You must own it."
}
},
required: ["script_id"]
},
category: "write",
userMessage: (params) => `Deleting script ${params["script_id"]}`
};
/** Every spec this module declares, in declaration order. */
export const scriptsSpecs: readonly CapabilitySpec[] = [
listScriptsSpec,
createScriptSpec,
getScriptSpec,
voiceScriptLinesSpec,
assembleScriptTimelineSpec,
editScriptSpec,
writeScriptSpec,
deriveStoryboardFromScriptSpec,
deleteScriptSpec
];