Add reference-to-video support to providers and storyboards

packages/agents/scripts/dump-creative-run.ts

@@ -79,7 +79,7 @@ function createFalMediaBackend(fal: {
       return save(label, "png", bytes);
     },
     async video(from, prompt, label, durationSeconds) {
-      const bytes = await fal.imageToVideo([from], {
+      const bytes = await fal.imageToVideo(from, {
         model: videoModel,
         prompt,
         durationSeconds,

packages/agents/src/capabilities/models.specs.ts

@@ -17,6 +17,7 @@ export const SUPPORTED_CAPABILITIES = [
   "segment_image",
   "text_to_video",
   "image_to_video",
+  "reference_to_video",
   "text_to_speech",
   "text_to_music",
   "automatic_speech_recognition",

packages/agents/src/capabilities/models.ts

@@ -153,6 +153,7 @@ const CAPABILITY_REF_TYPE = {
   segment_image: "image_model",
   text_to_video: "video_model",
   image_to_video: "video_model",
+  reference_to_video: "video_model",
   text_to_speech: "tts_model",
   // A music-typed node property takes a `music_model`, never a `tts_model`.
   // Handing back the wrong tag made every music ref unassignable: the property
@@ -201,6 +202,8 @@ function capabilityToRecommendedTasks(
       return new Set(["text_to_video"]);
     case "image_to_video":
       return new Set(["image_to_video"]);
+    case "reference_to_video":
+      return new Set(["reference_to_video"]);
     case "generate_embedding":
       return new Set(["embedding"]);
     case "generate_message":
@@ -222,6 +225,7 @@ function capabilityToRecommendedModalities(
       return new Set(["image"]);
     case "text_to_video":
     case "image_to_video":
+    case "reference_to_video":
       return new Set(["video"]);
     case "text_to_speech":
       return new Set(["tts"]);
@@ -246,6 +250,7 @@ async function fetchModelsForCapability(
       return await provider.getAvailableImageModels();
     case "text_to_video":
     case "image_to_video":
+    case "reference_to_video":
       return await provider.getAvailableVideoModels();
     case "text_to_speech":
       return await provider.getAvailableTTSModels();
@@ -287,6 +292,7 @@ function capabilityTask(capability: SupportedCapability): string | null {
     case "image_to_image":
     case "text_to_video":
     case "image_to_video":
+    case "reference_to_video":
     case "text_to_speech":
     case "text_to_music":
       return capability;

packages/agents/src/capabilities/storyboards.specs.ts

@@ -124,11 +124,12 @@ export const RENDER_CLIPS_SCHEMA: JsonSchema = {
     },
     mode: {
       type: "string",
-      enum: ["keyframe", "direct"],
+      enum: ["keyframe", "direct", "reference"],
       description:
         "Override how the selected shots render, for this call only. " +
         "'keyframe' animates each shot's still (image_to_video); 'direct' " +
-        "generates from the prompt with no still (text_to_video). Defaults " +
+        "generates from the prompt with no still (text_to_video); 'reference' " +
+        "uses entity reference images (reference_to_video). Defaults " +
         "to each shot's own render_mode, which defaults to 'keyframe'. Set " +
         "the shot's render_mode with edit_storyboard to make it stick."
     },
@@ -338,15 +339,16 @@ export const renderStoryboardClipsSpec: CapabilitySpec = {
     "directly — no workflow is created or run. A shot renders the way its " +
     "render_mode says: 'keyframe' (the default) animates its selected still " +
     "with image_to_video, 'direct' generates from the prompt with " +
-    "text_to_video and needs no still. Pass `mode` to override both for this " +
+    "text_to_video and needs no still; 'reference' uses entity reference images " +
+    "with reference_to_video. Pass `mode` to override for this " +
     "call. Each clip is saved as an asset and attached to its shot (previous " +
     "takes are kept as versions), leaving the shot 'rendered' and ready for " +
     "assemble_storyboard_timeline. Omit `targets` to render every shot that " +
     "still needs a clip and can render one — a shot covered by another " +
     "shot's generation already has its picture and is skipped. " +
     "A keyframe-mode shot with no " +
     "still is reported, not rendered — run render_storyboard_stills first, or " +
-    "set its render_mode to 'direct'. This is the expensive step.",
+    "set its render_mode to 'direct' or 'reference'. This is the expensive step.",
   inputSchema: RENDER_CLIPS_SCHEMA,
   category: "write",
   userMessage: (params) => {

packages/agents/src/capabilities/storyboards.ts

@@ -438,7 +438,7 @@ interface ShotOutcome {
   index: number;
   slug?: string;
   /** How the clip was rendered. Absent on a stills outcome. */
-  render_mode?: "keyframe" | "direct";
+  render_mode?: "keyframe" | "direct" | "reference";
   ok: boolean;
   asset_id?: string;
   asset_uri?: string;
@@ -790,36 +790,54 @@ const renderStoryboardClips: CapabilityExport = {
     const { row, doc } = board;
 
     const override = params["mode"];
-    if (override !== undefined && override !== "keyframe" && override !== "direct") {
-      return { error: 'mode must be "keyframe" or "direct".' };
+    if (override !== undefined && override !== "keyframe" && override !== "direct" && override !== "reference") {
+      return { error: 'mode must be "keyframe", "direct", or "reference".' };
     }
     // The call's override wins over the shot's own setting, for this call only.
     const { shotRenderMode } = await import("@nodetool-ai/protocol");
-    const modeOf = (shot: Shot): "keyframe" | "direct" =>
-      override === "keyframe" || override === "direct"
+    const modeOf = (shot: Shot): "keyframe" | "direct" | "reference" =>
+      override === "keyframe" || override === "direct" || override === "reference"
         ? override
         : shotRenderMode(shot);
 
-    const model = resolveModel(
-      params,
-      doc.videoModel,
-      "clip",
-      // A board renders one way or the other far more often than both, so name
-      // the capability the selection actually needs.
-      doc.shots.every((s) => modeOf(s) === "direct")
-        ? "text_to_video"
-        : "image_to_video"
-    );
-    if (isError(model)) return model;
-
     const selected = selectShots(
       doc.shots,
       params["targets"],
       (s) =>
         !shotHasPicture(s, doc.shots) &&
-        (modeOf(s) === "direct" || !!s.keyframe)
+        (modeOf(s) === "direct" || modeOf(s) === "reference" || !!s.keyframe)
     );
     if (isError(selected)) return selected;
+    const requiredCapabilities = new Set(
+      selected.map((shot) =>
+        modeOf(shot) === "direct"
+          ? "text_to_video"
+          : modeOf(shot) === "reference"
+            ? "reference_to_video"
+            : "image_to_video"
+      )
+    );
+    const capability = requiredCapabilities.has("reference_to_video")
+      ? "reference_to_video"
+      : requiredCapabilities.has("image_to_video")
+        ? "image_to_video"
+        : "text_to_video";
+    const model = resolveModel(params, doc.videoModel, "clip", capability);
+    if (isError(model)) return model;
+    const declaredTasks = doc.videoModel?.supported_tasks;
+    if (
+      Array.isArray(declaredTasks) && declaredTasks.length > 0 &&
+      model.model === doc.videoModel?.id && model.provider === doc.videoModel?.provider
+    ) {
+      const unsupported = [...requiredCapabilities].find(
+        (task) => !declaredTasks.includes(task)
+      );
+      if (unsupported) {
+        return {
+          error: `The selected clip model does not support ${unsupported}; choose a model that supports every selected shot mode.`
+        };
+      }
+    }
     const entities = await loadBoardEntities(context, doc);
     const fresh = await filterStale(selected, params, doc, entities, "clip");
     const skipped = fresh.skipped;
@@ -1684,8 +1702,8 @@ function applyShotFields(
   }
   if (args["render_mode"] !== undefined) {
     const mode = String(args["render_mode"]);
-    if (mode !== "keyframe" && mode !== "direct") {
-      throw new Error('render_mode must be "keyframe" or "direct".');
+    if (mode !== "keyframe" && mode !== "direct" && mode !== "reference") {
+      throw new Error('render_mode must be "keyframe", "direct", or "reference".');
     }
     next.render_mode = mode;
   }

packages/agents/src/prompts/workflow-authoring-knowledge.ts

@@ -23,6 +23,7 @@ export type GenericNodeCapability =
   | "image_to_image"
   | "text_to_video"
   | "image_to_video"
+  | "reference_to_video"
   | "text_to_speech"
   | "automatic_speech_recognition"
   | "generate_embedding"
@@ -79,6 +80,14 @@ export const GENERIC_AI_NODES: readonly GenericAINode[] = [
       "Animate a source image into a video. Required: image, prompt, model.",
     acceptsModel: true
   },
+  {
+    type: "nodetool.video.ReferenceToVideo",
+    capability: "reference_to_video",
+    task: "Reference → Video",
+    summary:
+      "Generate a video guided by ordered reference images and videos. Required: at least one reference image or video, prompt, model.",
+    acceptsModel: true
+  },
   {
     type: "nodetool.audio.TextToSpeech",
     capability: "text_to_speech",

packages/agents/tests/capabilities-models-and-storyboards-reference-video.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";
+import { BaseProvider } from "@nodetool-ai/runtime";
+import type { ProcessingContext, ProviderId, VideoModel } from "@nodetool-ai/runtime";
+import { Asset, ModelObserver, Storyboard, initTestDb } from "@nodetool-ai/models";
+import type { Shot } from "@nodetool-ai/protocol";
+import { toolForCapabilityName } from "../src/capabilities/lazy-tool.js";
+import { createCapabilityRun, UNGATED } from "../src/capabilities/invoke.js";
+import { withGenerationSeam } from "./_helpers/generation-seam.js";
+
+class ReferenceVideoProvider extends BaseProvider {
+  constructor(private readonly models: VideoModel[]) {
+    super("fal_ai" as ProviderId);
+  }
+  override async referenceToVideo(): Promise<Uint8Array> { return new Uint8Array(); }
+  override async getAvailableVideoModels(): Promise<VideoModel[]> {
+    return this.models;
+  }
+}
+
+const ctx = { userId: "u1" } as ProcessingContext;
+const shot = (overrides: Partial<Shot> & { id: string; index: number }): Shot => ({
+  type: "shot",
+  action: `action ${overrides.index}`,
+  status: "planned",
+  ...overrides
+});
+
+function renderContext(referenceIds: [string, string]) {
+  return withGenerationSeam({
+    userId: "u1",
+    runProviderPrediction: vi.fn(async () => new Uint8Array([0, 0, 0, 24, 102, 116, 121, 112])),
+    hasModelInterface: () => true,
+    createAsset: vi.fn(async (args: { name: string; contentType: string; content: Uint8Array }) => {
+      return Asset.create<Asset>({ user_id: "u1", name: args.name, content_type: args.contentType });
+    }),
+    resolveAssetBytes: vi.fn(async (uri: string) => ({
+      bytes: uri.includes(referenceIds[0]) ? new Uint8Array([1, 2]) : new Uint8Array([3, 4])
+    }))
+  }) as ProcessingContext & { runProviderPrediction: ReturnType<typeof vi.fn> };
+}
+
+describe("reference_to_video capability contracts", () => {
+  beforeEach(() => initTestDb());
+  afterEach(() => ModelObserver.clear());
+
+  it("find_model filters video models by reference_to_video", async () => {
+    const provider = new ReferenceVideoProvider([
+      { id: "image-model", name: "Image model", provider: "fal_ai", supportedTasks: ["image_to_video"] },
+      { id: "reference-model", name: "Reference model", provider: "fal_ai", supportedTasks: ["reference_to_video"] }
+    ]);
+    const tool = toolForCapabilityName("find_model", (context) =>
+      createCapabilityRun({ context, gate: UNGATED, providers: { fal_ai: provider } })
+    );
+    const result = (await tool.process(ctx, { capability: "reference_to_video" })) as {
+      results: Array<{ model_id: string }>;
+    };
+    expect(result.results.map((model) => model.model_id)).toEqual(["reference-model"]);
+  });
+
+  it("render_storyboard_clips dispatches a reference shot through reference_to_video", async () => {
+    const first = await Asset.create<Asset>({
+      user_id: "u1", name: "ref-a.png", content_type: "image/png",
+      metadata: { nodetool_entity: { kind: "character", name: "A", descriptor: "first" } }
+    });
+    const second = await Asset.create<Asset>({
+      user_id: "u1", name: "ref-b.png", content_type: "image/png",
+      metadata: { nodetool_entity: { kind: "character", name: "B", descriptor: "second" } }
+    });
+    const board = await Storyboard.create<Storyboard>({
+      user_id: "u1", project_id: "default", name: "Reference board",
+      document: JSON.stringify({
+        screenplay: null,
+        shots: [shot({ id: "s1", index: 0, render_mode: "reference", entity_ids: [first.id, second.id] })],
+        brief: "", style: "", entityIds: [first.id, second.id], aspectRatio: "16:9",
+        directorModel: null, imageModel: null,
+        videoModel: { type: "video_model", id: "reference-model", provider: "fal_ai" }
+      })
+    });
+    const context = renderContext([first.id, second.id]);
+    const result = (await toolForCapabilityName("render_storyboard_clips").process(context, {
+      storyboard_id: board.id
+    })) as { rendered: number };
+    expect(result.rendered).toBe(1);
+    expect(context.runProviderPrediction.mock.calls[0][0]).toMatchObject({
+      capability: "reference_to_video",
+      model: "reference-model",
+      params: { reference_images: [new Uint8Array([1, 2]), new Uint8Array([3, 4])] }
+    });
+  });
+});

packages/agents/tests/generic-ai-nodes.test.ts

@@ -11,6 +11,7 @@ const VALID_CAPABILITIES: GenericNodeCapability[] = [
   "image_to_image",
   "text_to_video",
   "image_to_video",
+  "reference_to_video",
   "text_to_speech",
   "automatic_speech_recognition",
   "generate_embedding",

packages/agents/tests/storyboard-render-tools.test.ts

@@ -327,7 +327,7 @@ describe("storyboard render tools", () => {
       ops: [{ op: "update_shot", target: "s1", render_mode: "sideways" }]
     })) as { failed: number; ops: Array<{ error?: string }> };
     expect(rejected.failed).toBe(1);
-    expect(rejected.ops[0].error).toContain('"keyframe" or "direct"');
+    expect(rejected.ops[0].error).toContain('"keyframe", "direct", or "reference"');
   });
 
   it("revises a clip in place, keeping the previous take as a version", async () => {

packages/base-nodes/src/index.ts

@@ -226,6 +226,7 @@ export {
 export {
   TextToVideoNode,
   ImageToVideoNode,
+  ReferenceToVideoNode,
   LoadVideoFileNode,
   SaveVideoFileVideoNode,
   LoadVideoAssetsNode,

packages/cli/src/harness/capability-table.ts

@@ -315,11 +315,11 @@ export const CAPABILITY_COVERAGE: readonly CapabilityCoverageEntry[] = [
     name: "find_model",
     module: "models",
     impl: "packages/agents/src/capabilities/models.ts",
-    contract: "fdb5f2e60161",
+    contract: "cc6866ced0fd",
     selfcheck: "capability-suites",
     suites: [
       "packages/agents/tests/capabilities-models.test.ts",
-      "packages/agents/tests/capabilities-models-rankings.test.ts",
+      "packages/agents/tests/capabilities-models-and-storyboards-reference-video.test.ts",
     ],
     evals: [
       {
@@ -2473,11 +2473,11 @@ export const CAPABILITY_COVERAGE: readonly CapabilityCoverageEntry[] = [
     name: "render_storyboard_clips",
     module: "storyboards",
     impl: "packages/agents/src/capabilities/storyboards.ts",
-    contract: "99052baaa53f",
+    contract: "568457bf299c",
     selfcheck: "capability-suites",
     suites: [
       "packages/agents/tests/capabilities-storyboards.test.ts",
-      "packages/agents/tests/capabilities-storyboard-board-ops.test.ts",
+      "packages/agents/tests/capabilities-models-and-storyboards-reference-video.test.ts",
     ],
     evals: [
       {

packages/dsl/src/flow/generated/nodetool.video.ts

@@ -46,6 +46,29 @@ export function imageToVideo(inputs: ImageToVideoInputs): Promise<ImageToVideoOu
   return callNode<ImageToVideoOutputs>("nodetool.video.ImageToVideo", inputs);
 }
 
+// Reference To Video — nodetool.video.ReferenceToVideo
+export type ReferenceToVideoInputs = {
+  reference_images?: ImageRef[];
+  reference_videos?: VideoRef[];
+  model?: unknown;
+  prompt?: string;
+  use_reference_video_audio?: boolean;
+  negative_prompt?: string;
+  entities?: Entity[];
+  aspect_ratio?: string;
+  resolution?: string;
+  duration?: number;
+  timeout_seconds?: number;
+};
+
+export interface ReferenceToVideoOutputs {
+  output: VideoRef;
+}
+
+export function referenceToVideo(inputs: ReferenceToVideoInputs): Promise<ReferenceToVideoOutputs> {
+  return callNode<ReferenceToVideoOutputs>("nodetool.video.ReferenceToVideo", inputs);
+}
+
 // Load Video File — nodetool.video.LoadVideoFile
 export type LoadVideoFileInputs = {
   path?: string;