Support reference-to-video generation through WanGP

Dockerfile.combined

@@ -5,7 +5,7 @@ ARG CORE_IMAGE=ghcr.io/nodetool-ai/nodetool:latest
 FROM ${CORE_IMAGE}
 
 ARG NODETOOL_CORE_REPOSITORY=https://github.com/nodetool-ai/nodetool-core.git
-ARG NODETOOL_CORE_COMMIT=2f5c2519d9e46b38975c349c8467e41e053ed0e8
+ARG NODETOOL_CORE_COMMIT=e53b334aa3b659c92baa2307c11ed8a7308f19c3
 ARG WANGP_REPOSITORY=https://github.com/deepbeepmeep/Wan2GP.git
 ARG WANGP_COMMIT=362c3467a70e1136ceb52eec95907205a8f88543
 ARG MMGP_REPOSITORY=https://github.com/deepbeepmeep/mmgp.git
@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.title="NodeTool WanGP non-commercial GPU worker"
       org.opencontainers.image.description="Authenticated direct-Python WanGP provider worker; free non-monetized evaluation only" \
       org.opencontainers.image.licenses="AGPL-3.0-or-later AND LicenseRef-WanGP-Community-2.0 AND LicenseRef-mmGP-NonCommercial" \
       org.opencontainers.image.source="https://github.com/nodetool-ai/nodetool-wan2gp" \
-      io.nodetool.core.commit="2f5c2519d9e46b38975c349c8467e41e053ed0e8" \
+      io.nodetool.core.commit="e53b334aa3b659c92baa2307c11ed8a7308f19c3" \
       io.nodetool.wangp.commit="362c3467a70e1136ceb52eec95907205a8f88543" \
       io.nodetool.mmgp.commit="589ba050d320c4df879f48d11b23d6a88e6c68c8" \
       io.nodetool.commercial-use="prohibited-without-separate-written-license"
@@ -89,7 +89,7 @@ ENV WANGP_CONFIG_DIR=/workspace/wan2gp/config \
     WANGP_ROOT=/opt/Wan2GP \
     NODETOOL_MODEL_PREPARE_COMMAND_WANGP="/opt/wan2gp-venv/bin/python /opt/nodetool-wan2gp-combined/prepare_model.py" \
     NODETOOL_PROVIDER_ADAPTER_COMMAND_WANGP="/opt/wan2gp-venv/bin/python /opt/nodetool-wan2gp-combined/provider_adapter.py" \
-    NODETOOL_PROVIDER_ADAPTER_CAPABILITIES_WANGP="text_to_video,image_to_video,text_to_image,image_to_image,text_to_audio,text_to_speech_encoded" \
+    NODETOOL_PROVIDER_ADAPTER_CAPABILITIES_WANGP="text_to_video,image_to_video,reference_to_video,text_to_image,image_to_image,text_to_audio,text_to_speech_encoded" \
     NODETOOL_PROVIDER_ADAPTER_DISPLAY_NAME_WANGP="WanGP" \
     HF_HOME=/workspace/cache/huggingface \
     PYTHONUNBUFFERED=1 \

combined/provider_adapter.py

@@ -31,9 +31,7 @@ def __init__(self) -> None:
     def send(self, event_type: str, data: dict[str, Any]) -> None:
         with self._lock:
             self._stream.write(
-                json.dumps(
-                    {"type": event_type, "data": data}, separators=(",", ":")
-                )
+                json.dumps({"type": event_type, "data": data}, separators=(",", ":"))
                 + "\n"
             )
             self._stream.flush()
@@ -98,12 +96,86 @@ def _choice_values(definition: Any) -> list[str]:
         return []
     values: list[str] = []
     for choice in definition.get("choices", []):
-        value = choice.get("value") if isinstance(choice, dict) else choice
+        value = (
+            choice.get("value")
+            if isinstance(choice, dict)
+            else choice[1]
+            if isinstance(choice, (list, tuple)) and len(choice) > 1
+            else choice
+        )
+        if value is not None and str(value):
+            values.append(str(value))
+    for value in definition.get("selection", []):
         if value is not None and str(value):
             values.append(str(value))
     return values
 
 
+def _setting_definition(metadata: dict[str, Any] | None, name: str) -> Any:
+    values = (metadata or {}).get("setting_values", {})
+    if not isinstance(values, dict):
+        return None
+    direct = values.get(name)
+    if direct is not None:
+        return direct
+    video = values.get("video_prompt_type")
+    return video.get(name) if isinstance(video, dict) else None
+
+
+def _media_roles(metadata: dict[str, Any] | None, kind: str) -> dict[str, Any]:
+    media = (metadata or {}).get("media_inputs", {})
+    roles = media.get(kind, {}) if isinstance(media, dict) else {}
+    return roles if isinstance(roles, dict) else {}
+
+
+def _reference_modes(metadata: dict[str, Any] | None) -> list[str]:
+    image_modes = _choice_values(_setting_definition(metadata, "image_ref_choices"))
+    image_modes += _choice_values(_setting_definition(metadata, "guide_custom_choices"))
+    return [value for value in image_modes if "I" in value and "F" not in value]
+
+
+def _video_reference_modes(metadata: dict[str, Any] | None) -> list[str]:
+    # WanGP's guide choices are authoritative. H3 declares V-U and V+-U for
+    # one and two reference videos. GV is generic control video and excluded.
+    values = _choice_values(_setting_definition(metadata, "guide_custom_choices"))
+    return [value for value in values if value in {"V-U", "V+-U"}]
+
+
+def _audio_video_modes(metadata: dict[str, Any] | None) -> list[str]:
+    audio = (metadata or {}).get("setting_values", {}).get("audio_prompt_type", {})
+    definition = audio.get("sources") if isinstance(audio, dict) else None
+    values = _choice_values(definition)
+    return [value for value in values if "K" in value]
+
+
+def _combine_prompt_modes(*modes: str) -> str:
+    result = ""
+    for mode in modes:
+        for flag in mode:
+            if flag not in result:
+                result += flag
+    return result
+
+
+def _supports_reference_video(metadata: dict[str, Any] | None) -> bool:
+    if not metadata:
+        return False
+    image_roles = _media_roles(metadata, "image")
+    video_roles = _media_roles(metadata, "video")
+    image_support = bool(
+        (
+            image_roles.get("reference")
+            or image_roles.get("multiple_references")
+            or image_roles.get("single_reference")
+        )
+        and _reference_modes(metadata)
+    )
+    video_support = bool(
+        video_roles.get("control") and _video_reference_modes(metadata)
+    )
+    return bool(image_support or video_support)
+
+
 def _models(session: Any, model_kind: str) -> list[dict[str, Any]]:
     """Translate WanGP model metadata into NodeTool provider models."""
     if model_kind not in {"video", "image", "tts", "music"}:
@@ -123,6 +195,8 @@ def _models(session: Any, model_kind: str) -> list[dict[str, Any]]:
             supported = [
                 task for task in _MODEL_TASKS[model_kind] if capabilities.get(task)
             ]
+            if model_kind == "video" and _supports_reference_video(metadata):
+                supported.append("reference_to_video")
         elif model_kind == "music":
             supported = ["text_to_music"] if capabilities.get("text_to_audio") else []
         else:
@@ -195,22 +269,23 @@ def _model_id(request: dict[str, Any]) -> str:
 
 def _input_path(request: dict[str, Any], name: str) -> Path:
     path = Path(str(request.get(name) or "")).resolve()
-    if not path.is_file():
+    if not path.is_file() or path.stat().st_size == 0:
         raise ValueError(f"{request.get('operation')} requires a readable {name}")
     return path
 
 
 def _setting_choice_with_flag(
     metadata: dict[str, Any] | None, setting: str, flag: str
 ) -> str:
-    definitions = (metadata or {}).get("setting_values", {}).get(
-        "video_prompt_type", {}
-    )
-    choice_def = definitions.get(setting)
-    if not isinstance(choice_def, dict):
-        return ""
-    for choice in choice_def.get("choices", []):
-        value = choice.get("value", "") if isinstance(choice, dict) else ""
+    choice_def = _setting_definition(metadata, setting)
+    for choice in choice_def.get("choices", []) if isinstance(choice_def, dict) else []:
+        value = (
+            choice.get("value", "")
+            if isinstance(choice, dict)
+            else choice[1]
+            if isinstance(choice, (list, tuple)) and len(choice) > 1
+            else choice
+        )
         if flag in str(value):
             return str(value)
     return ""
@@ -255,6 +330,11 @@ def _settings(
         settings["image_mode"] = 1
 
     if operation in {"image_to_image", "image_to_video"}:
+        if operation == "image_to_video" and metadata is not None:
+            capabilities = metadata.get("capabilities", {})
+            image_roles = _media_roles(metadata, "image")
+            if not capabilities.get("image_to_video") or not image_roles.get("start"):
+                raise ValueError(f"Model {model_type} does not support image_to_video")
         image_path = str(_input_path(request, "image_path"))
         image_inputs = (metadata or {}).get("media_inputs", {}).get("image", {})
         if image_inputs.get("start") or operation == "image_to_video":
@@ -272,6 +352,83 @@ def _settings(
         else:
             raise ValueError(f"Model {model_type} does not accept an input image")
 
+    if operation == "reference_to_video":
+        if (
+            metadata is None
+            or str(metadata.get("model_type") or model_type) != model_type
+        ):
+            raise ValueError(f"Unknown WanGP model: {model_type}")
+        if not _supports_reference_video(metadata):
+            raise ValueError(f"Model {model_type} does not support reference_to_video")
+        raw_image_paths = request.get("reference_image_paths", [])
+        raw_video_paths = request.get("reference_video_paths", [])
+        if not isinstance(raw_image_paths, list) or not isinstance(
+            raw_video_paths, list
+        ):
+            raise ValueError("reference media paths must be arrays")
+        image_paths = [Path(str(path)).resolve() for path in raw_image_paths]
+        video_paths = [Path(str(path)).resolve() for path in raw_video_paths]
+        if any(
+            not path.is_file() or path.stat().st_size == 0
+            for path in image_paths + video_paths
+        ):
+            raise ValueError(
+                "reference_to_video requires readable reference media paths"
+            )
+        if not image_paths and not video_paths:
+            raise ValueError(
+                "reference_to_video requires at least one reference image or video"
+            )
+        if len(video_paths) > 2:
+            raise ValueError("reference_to_video supports at most two reference videos")
+        image_roles = _media_roles(metadata, "image")
+        if image_roles.get("single_reference") and len(image_paths) > 1:
+            raise ValueError(
+                "reference_to_video model accepts only one reference image"
+            )
+        if image_paths and not _reference_modes(metadata):
+            raise ValueError(
+                "reference_to_video model does not accept reference images"
+            )
+        if video_paths and not _media_roles(metadata, "video").get("control"):
+            raise ValueError(
+                "reference_to_video model does not accept reference videos"
+            )
+        if image_paths:
+            settings["image_refs"] = [str(path) for path in image_paths]
+            image_mode = _reference_modes(metadata)[0]
+            settings["video_prompt_type"] = image_mode
+        if video_paths:
+            video_modes = _video_reference_modes(metadata)
+            mode = next(
+                (
+                    value
+                    for value in video_modes
+                    if value == ("V+-U" if len(video_paths) == 2 else "V-U")
+                ),
+                None,
+            )
+            if mode is None:
+                raise ValueError(
+                    "reference_to_video video count has no advertised WanGP mode"
+                )
+            image_mode = settings.get("video_prompt_type")
+            settings["video_prompt_type"] = _combine_prompt_modes(
+                str(image_mode or ""), mode
+            )
+            settings["video_guide"] = str(video_paths[0])
+            if len(video_paths) == 2:
+                settings["video_guide2"] = str(video_paths[1])
+        if params.get("useReferenceVideoAudio"):
+            if not video_paths:
+                raise ValueError(
+                    "reference video audio requires at least one reference video"
+                )
+            audio_modes = _audio_video_modes(metadata)
+            if not audio_modes:
+                raise ValueError("reference video audio is not supported by this model")
+            settings["audio_prompt_type"] = audio_modes[0]
+
     if operation == "text_to_audio":
         style_prompt = str(params.get("prompt") or "")
         lyrics = str(params.get("lyrics") or "").strip()
@@ -300,9 +457,7 @@ def _settings(
         if params.get("speed") is not None and base_model_type == "index_tts25":
             settings["custom_settings"] = {"speech_speed": float(params["speed"])}
         if request.get("reference_audio_path"):
-            settings["audio_guide"] = str(
-                _input_path(request, "reference_audio_path")
-            )
+            settings["audio_guide"] = str(_input_path(request, "reference_audio_path"))
     return settings
 
 
@@ -370,6 +525,7 @@ def main() -> int:
             "image_to_image": "image",
             "text_to_video": "video",
             "image_to_video": "video",
+            "reference_to_video": "video",
             "text_to_audio": "audio",
             "tts_encoded": "audio",
         }.get(operation)

docs/wan2gp-contract.md

@@ -1,7 +1,7 @@
 # The Wan2GP MCP contract this package depends on
 
 Every item below was resolved by reading the Wan2GP source at commit
-`057f9ecab9ad57dfbec9768b2daf7a4426ce986c` (2026-09-06). **None of it was
+`362c3467a70e1136ceb52eec95907205a8f88543`. **None of it was
 confirmed against a live server.** There was no Wan2GP instance and no GPU on
 the machine where this package was written. Treat each shape as source-derived
 until someone runs `scripts/smoke.py` against a real server.
@@ -320,3 +320,19 @@ The nodes in this package call it once per poll and, when it raises, send
 - Which `model_type` ids a particular install exposes. `defaults/` is the
   registry, and a user can add finetunes under `finetunes/`
   (`wgp.py:3281`).
+
+## Reference video adapter
+
+The combined worker targets WanGP commit `362c3467a70e1136ceb52eec95907205a8f88543`.
+Reference support is discovered from `session.list_model_metadata()`. The
+adapter uses the pinned metadata fields `media_inputs.image.reference` (or its
+single or multiple reference roles), `media_inputs.video.control`, and the
+declared `setting_values` choices for `image_ref_choices`,
+`guide_custom_choices`, and `audio_prompt_type.sources`.
+
+Reference images are sent as ordered `image_refs`. One or two reference videos
+are sent as `video_guide` and `video_guide2`, selecting the advertised `V-U` or
+`V+-U` mode. WanGP's `GV` choice is generic control video and is excluded.
+Unsupported media kinds, counts, missing models, and missing task
+support fail before `run_task`. Audio is enabled only when a reference video
+is present and WanGP advertises a `K` audio source choice.

tests/fixtures/wangp_h3_fl2va_metadata.json

@@ -0,0 +1,33 @@
+{
+  "model_type": "minimax_h3_fl2va_pruned_pdd",
+  "name": "MiniMax H3 FL2VA",
+  "main_output": ["video"],
+  "media_inputs": {
+    "image": {
+      "reference": true,
+      "injected_frames": true
+    },
+    "video": {
+      "control": true
+    }
+  },
+  "capabilities": {
+    "text_to_video": true,
+    "image_to_video": true,
+    "reference_images": true,
+    "injected_frames": true,
+    "control_video": true
+  },
+  "setting_values": {
+    "video_prompt_type": {
+      "image_ref_choices": null,
+      "guide_custom_choices": {
+        "choices": [
+          ["Generate without using a Control Video", ""],
+          ["Use Control Video", "GV"],
+          ["Inject Frames", "KFI"]
+        ]
+      }
+    }
+  }
+}