feat(timeline): let an agent replace a whole timeline document [T18]

An agent could edit a timeline only through `edit_timeline`'s op list, so
authoring a sequence from scratch, or applying a document it had built
elsewhere, meant expressing the whole thing as incremental ops.

`set_timeline_document` replaces the document in one call, in an order
chosen so a refused call leaves nothing behind: ownership, then the
`expected_updated_at` precheck, then validation, then the snapshot, then the
CAS write, then a re-validation of what was actually stored. A conflict or a
validation error returns before the snapshot — a refused call that leaves a
version row behind is still a write. The write itself is a single attempt
with no retry loop: `edit_timeline` retries because its ops re-apply against
a newer document, but a whole-document replace does not compose, so
retrying would clobber exactly the change CAS just caught.

The return carries the post-write validation rather than the pre-write one,
so a caller sees what it created instead of what it intended, plus the
version id to undo it.

`duration_ms` is restamped from the clips, because a replaced document that
keeps the old sequence duration makes `preview_timeline_frame` truncate its
sampling window.

Resolves F18.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdzG3a3jN3Dzx6QV8xh4Bo

Author: claude

packages/agents/src/capabilities/timelines.specs.ts

@@ -353,6 +353,72 @@ export const editTimelineSpec: CapabilitySpec = {
   }
 };
 
+export const SET_TIMELINE_DOCUMENT_SCHEMA: JsonSchema = {
+  type: "object",
+  properties: {
+    timeline_id: {
+      type: "string",
+      description: "Timeline sequence id (from list_timelines)."
+    },
+    document: {
+      type: "object",
+      description:
+        "The whole document to store: {tracks, clips, markers, transcript?, " +
+        "scriptEnabled?}. It replaces the stored one field for field, so " +
+        "anything you leave out is dropped — read the current document with " +
+        "get_timeline and send it back changed, rather than sending only the " +
+        "part you edited. `markers` may be omitted and defaults to an empty " +
+        "list."
+    },
+    fps: {
+      type: "number",
+      description:
+        "Frame rate to store with the sequence. Omit to keep the current one."
+    },
+    width: {
+      type: "number",
+      description: "Render width. Omit to keep the current one."
+    },
+    height: {
+      type: "number",
+      description: "Render height. Omit to keep the current one."
+    },
+    expected_updated_at: {
+      type: "string",
+      description:
+        "The `updated_at` the document was read at. When it no longer " +
+        "matches the stored row the write is refused as a conflict instead " +
+        "of overwriting whoever changed it in between."
+    },
+    snapshot_name: {
+      type: "string",
+      description:
+        "Label for the snapshot taken before the write, e.g. 'before the " +
+        "title pass'."
+    }
+  },
+  required: ["timeline_id", "document"]
+};
+
+export const setTimelineDocumentSpec: CapabilitySpec = {
+  name: "set_timeline_document",
+  description:
+    "Write a whole timeline document at once — every track, clip, marker and " +
+    "animation in one call, instead of a script of edit_timeline ops. Reach " +
+    "for it when you are authoring a cut from scratch or restructuring one " +
+    "wholesale; edit_timeline stays the better tool for a few targeted " +
+    "changes to a cut that already exists. The document is validated before " +
+    "anything is written: errors refuse the write and come back as issues, " +
+    "so a document that would not render never reaches the sequence. The " +
+    "state it replaces is snapshotted as a manual version first, so the " +
+    "write is undoable with restore_timeline_version, and the validation of " +
+    "what actually landed is returned with the result.",
+  inputSchema: SET_TIMELINE_DOCUMENT_SCHEMA,
+  category: "write",
+  userMessage: (params) =>
+    `Writing the document of timeline ${String(params["timeline_id"])}`
+};
+
 export const validateTimelineSpec: CapabilitySpec = {
   name: "validate_timeline",
   description:
@@ -473,6 +539,7 @@ export const timelinesSpecs: readonly CapabilitySpec[] = [
   deleteTimelineVersionSpec,
   editTimelineSpec,
   validateTimelineSpec,
+  setTimelineDocumentSpec,
   previewTimelineFrameSpec,
   deleteTimelineSpec
 ];

packages/agents/src/capabilities/timelines.ts

@@ -48,6 +48,7 @@ import {
   deleteTimelineVersionSpec,
   editTimelineSpec,
   validateTimelineSpec,
+  setTimelineDocumentSpec,
   previewTimelineFrameSpec,
   DEFAULT_PREVIEW_COUNT,
   MAX_PREVIEW_TIMES,
@@ -64,6 +65,7 @@ import {
   DELETE_TIMELINE_VERSION_SCHEMA,
   EDIT_TIMELINE_SCHEMA,
   VALIDATE_TIMELINE_SCHEMA,
+  SET_TIMELINE_DOCUMENT_SCHEMA,
   deleteTimelineSpec
 } from "./timelines.specs.js";
 import { isFiniteNumber, isRecord, isString } from "../utils/type-guards.js";
@@ -81,7 +83,8 @@ export {
   RESTORE_TIMELINE_VERSION_SCHEMA,
   DELETE_TIMELINE_VERSION_SCHEMA,
   EDIT_TIMELINE_SCHEMA,
-  VALIDATE_TIMELINE_SCHEMA
+  VALIDATE_TIMELINE_SCHEMA,
+  SET_TIMELINE_DOCUMENT_SCHEMA
 } from "./timelines.specs.js";
 import { resolveProjectId } from "./project-scope.js";
 
@@ -905,6 +908,192 @@ const validateTimeline: CapabilityExport = {
   }
 };
 
+
+// ---------------------------------------------------------------------------
+// set_timeline_document
+// ---------------------------------------------------------------------------
+
+/**
+ * A document the caller sent, ready to validate.
+ *
+ * `markers` is the one field an agent authoring a cut has no reason to think
+ * about, and the Zod schema requires it — so an omitted one becomes an empty
+ * list rather than a `schema_invalid` refusal about a field nobody meant to
+ * leave out. Everything else is stored as given: the whole point of this
+ * capability is that what the caller sends is what the sequence becomes.
+ */
+function documentToStore(raw: Record<string, unknown>): Record<string, unknown> {
+  return raw["markers"] === undefined ? { ...raw, markers: [] } : raw;
+}
+
+/** The end of the last clip — what the sequence's stored duration means. */
+function documentDurationMs(document: Record<string, unknown>): number {
+  const clips = document["clips"];
+  if (!Array.isArray(clips)) return 0;
+  return clips.reduce((end: number, clip: unknown) => {
+    if (!isRecord(clip)) return end;
+    const start = Number(clip["startMs"]) || 0;
+    const length = Number(clip["durationMs"]) || 0;
+    return Math.max(end, start + length);
+  }, 0);
+}
+
+/**
+ * Replace a sequence's whole document in one call.
+ *
+ * `edit_timeline` is a script of ops against what is already stored, which is
+ * the wrong shape for authoring a cut from nothing: every clip costs an op,
+ * and the ops only reach the fields the bridge exposes. This takes the
+ * document itself.
+ *
+ * The order is validate → snapshot → CAS → validate again, and each step
+ * exists for a failure the one before it cannot catch:
+ *
+ * 1. **Validate first.** A document that would not render must not reach the
+ *    database — a refusal that still wrote is worse than no capability at all.
+ *    Errors return the issues and nothing else happens, snapshot included: a
+ *    refused call that left a version row behind is still a write.
+ * 2. **Snapshot.** The state being replaced becomes a manual version, so the
+ *    write is undoable with `restore_timeline_version` the way a restore is.
+ * 3. **CAS.** The write lands only while the row still reads as it did, so a
+ *    concurrent edit is reported instead of overwritten. There is no retry
+ *    loop here, unlike `edit_timeline`: ops re-apply against a newer document
+ *    and compose, a whole-document replace does not — retrying it would clobber
+ *    exactly the change CAS caught.
+ * 4. **Validate again.** The caller sees what it created rather than what it
+ *    intended. The two validations differ: the second runs against the stored
+ *    fps/width/height and the document as the row now holds it.
+ */
+const setTimelineDocument: CapabilityExport = {
+  spec: setTimelineDocumentSpec,
+  impl: async (run, params) => {
+    const sequence = await loadTimeline(run, params["timeline_id"]);
+    if (isError(sequence)) return sequence;
+
+    const raw = params["document"];
+    if (!isRecord(raw)) {
+      return {
+        error:
+          "document is required and must be an object ({tracks, clips, markers}). Read the current one with get_timeline."
+      };
+    }
+
+    const expected = params["expected_updated_at"];
+    if (expected !== undefined && expected !== null) {
+      if (!isString(expected)) {
+        return { error: "expected_updated_at must be a string timestamp." };
+      }
+      if (expected !== sequence.updated_at) {
+        return {
+          error: `Timeline ${sequence.id} was modified since it was read at ${expected} (it now reads ${sequence.updated_at}); nothing was written. Read it again with get_timeline and re-apply your changes.`,
+          written: false,
+          conflict: true,
+          timeline_id: sequence.id
+        };
+      }
+    }
+
+    const fps = sequenceSetting(params["fps"], sequence.fps, "fps");
+    if (isError(fps)) return fps;
+    const width = sequenceSetting(params["width"], sequence.width, "width");
+    if (isError(width)) return width;
+    const height = sequenceSetting(params["height"], sequence.height, "height");
+    if (isError(height)) return height;
+
+    const document = documentToStore(raw);
+    const { validateTimelineSequence } =
+      await import("@nodetool-ai/execution/timeline-debug");
+    const before: TimelineValidation = validateTimelineSequence(document, {
+      fps,
+      width,
+      height
+    });
+    if (!before.ok) {
+      return {
+        error: `The document has ${before.errors.length} error${
+          before.errors.length === 1 ? "" : "s"
+        } and was not written to timeline ${sequence.id}. Fix them and call again.`,
+        written: false,
+        timeline_id: sequence.id,
+        validation: before,
+        summary: validationSummary(before)
+      };
+    }
+
+    const { TimelineSequence, TimelineSequenceVersion } =
+      await import("@nodetool-ai/models");
+    const snapshotName =
+      isString(params["snapshot_name"]) && params["snapshot_name"]
+        ? params["snapshot_name"]
+        : "Before set_timeline_document";
+    const undo = await TimelineSequenceVersion.snapshot(sequence, {
+      saveType: "manual",
+      name: snapshotName
+    });
+
+    const durationMs = documentDurationMs(document);
+    let saved: TimelineSequence | null;
+    try {
+      saved = await TimelineSequence.updateFieldsIfUnchanged(
+        sequence.id,
+        sequence.updated_at,
+        {
+          document: JSON.stringify(document),
+          fps,
+          width,
+          height,
+          duration_ms: durationMs
+        }
+      );
+    } catch (error) {
+      // The model rejects a document without tracks/clips/markers arrays. The
+      // validation above already covers that, so reaching here means the two
+      // disagree — report it rather than throwing out of the capability.
+      return {
+        error: `The document was refused by the store: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+        written: false,
+        timeline_id: sequence.id,
+        undo_version: undo.version
+      };
+    }
+    if (!saved) {
+      return {
+        error: `Timeline ${sequence.id} was modified concurrently; nothing was written. Read it again with get_timeline and re-apply your changes.`,
+        written: false,
+        conflict: true,
+        timeline_id: sequence.id,
+        undo_version: undo.version
+      };
+    }
+
+    const after: TimelineValidation = validateTimelineSequence(
+      saved.toDocument(),
+      { fps: saved.fps, width: saved.width, height: saved.height }
+    );
+    return {
+      ok: true,
+      written: true,
+      timeline_id: saved.id,
+      updated_at: saved.updated_at,
+      undo_version: undo.version,
+      fps: saved.fps,
+      width: saved.width,
+      height: saved.height,
+      duration_ms: saved.duration_ms,
+      track_count: Array.isArray(document["tracks"])
+        ? document["tracks"].length
+        : 0,
+      clip_count: Array.isArray(document["clips"])
+        ? document["clips"].length
+        : 0,
+      validation: after,
+      summary: validationSummary(after)
+    };
+  }
+};
+
 /**
  * The timecodes to render when the caller names none: evenly spaced across the
  * sequence, avoiding both ends — the first and last frame of a cut are the two
@@ -1126,6 +1315,7 @@ export const TIMELINE_CAPABILITIES: readonly CapabilityExport[] = [
   deleteTimelineVersion,
   editTimeline,
   validateTimeline,
+  setTimelineDocument,
   previewTimelineFrame,
   deleteTimeline
 ];
@@ -1146,6 +1336,7 @@ export {
   deleteTimelineVersion,
   editTimeline,
   validateTimeline,
+  setTimelineDocument,
   previewTimelineFrame,
   deleteTimeline
 };

packages/agents/src/codeact/nodetool-api.ts

@@ -124,7 +124,8 @@ export const NODETOOL_API_NAMESPACE_TOOLS: Record<string, readonly string[]> = {
     "delete_timeline_version",
     "validate_timeline",
     "preview_timeline_frame",
-    "edit_timeline"
+    "edit_timeline",
+    "set_timeline_document"
   ],
   sketches: [
     "list_sketches",
@@ -1159,7 +1160,18 @@ const nodetool = (() => {
               __merge(opts, { document: target })
             ),
       /** Apply document edits to a saved sequence, server-side. */
-      edit: (id, ops) => __need("edit_timeline")({ timeline_id: id, ops: ops })
+      edit: (id, ops) => __need("edit_timeline")({ timeline_id: id, ops: ops }),
+      /**
+       * Write a whole document at once. The document replaces the stored one,
+       * so send back what get() returned, changed. Validated before anything
+       * is written and snapshotted first, so a bad document is refused and a
+       * good one is undoable. Options: {fps, width, height,
+       * expected_updated_at, snapshot_name}.
+       */
+      setDocument: (id, document, opts) =>
+        __need("set_timeline_document")(
+          __merge(opts, { timeline_id: id, document: document })
+        )
     },
 
     sketches: {
@@ -1598,7 +1610,10 @@ const NAMESPACE_DOCS: PromptEntry[] = [
   chosen timecodes — read the picture back instead of guessing at it),
   \`versions(id)\`,
   \`getVersion(id, n)\`, \`snapshot(id, {name})\`, \`restore(id, n)\`,
-  \`deleteVersion(id, n)\`, and
+  \`deleteVersion(id, n)\`,
+  \`setDocument(id, document, {fps, width, height, expected_updated_at,
+  snapshot_name})\` — the whole document in one call, validated before it is
+  written and snapshotted first, for authoring a cut from scratch — and
   \`edit(id, ops)\` — the cut itself, server-side: \`[{op: "add_track", type:
   "audio"}, {op: "add_text_clip", text: "Hi"}, {op: "split_clip", target:
   "shot", atMs: 3000}, {op: "animate_clip", target: "Hi", animations:

packages/agents/src/tools/tool-permissions.ts

@@ -268,6 +268,8 @@ export const TOOL_PERMISSION_CATEGORIES: Readonly<
   // Document edits: each rewrites a stored document under a CAS.
   edit_timeline: "write",
   edit_sketch: "write",
+  // Replaces a timeline's whole document, after snapshotting what it replaces.
+  set_timeline_document: "write",
   // Writes the glTF back over the asset it came from.
   edit_model3d: "write",
   create_model3d: "write",

packages/agents/tests/capabilities-registry.test.ts

@@ -239,6 +239,7 @@ const CAPABILITY_CATEGORY_SNAPSHOT: Record<string, PermissionCategory> = {
   segment_image: "write",
   serpapi_search: "read",
   set_setting: "write",
+  set_timeline_document: "write",
   set_workflow_access: "external",
   share_result: "read",
   start_background_job: "execute",