Skip to content

Commit d225924

Browse files
feat: videoeditor-genai — typed image-gen clients (grok + imagen) and an image subcommand (#8)
New crate videoeditor-genai: typed request/response bindings for still-image generation, built for optional episode-asset generation from the CLI: - xai module: Grok Imagine images (OpenAI-compatible endpoint) with reference-image conditioning (data-URL strings, ≤7), n≤10, aspect_ratio enum probed from the API, b64_json responses (the asset CDN 403s plain downloads; url branch kept as UA-header fallback). - google module: Imagen 4 stills via the Gemini :predict endpoint — aspect-true, no reference input (requests with refs are rejected with a pointer at grok), safety-filter rejections surfaced as such. Veo video is the module's planned next resident. - CLI: `videoeditor image "prompt" -o out.png [--provider grok|imagen] [--ref img ...] [-n N] [--aspect 1:1] [--model id]`. - guide.md pipeline/env + CLAUDE.md layout updated. Provider behavior (ref limits, CDN workaround, safety-filter routing) ported from production use in the fallout-is-real-life engine. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 9a4674b commit d225924

10 files changed

Lines changed: 601 additions & 0 deletions

File tree

CLAUDE.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ vertical video out via headless Chrome + ffmpeg + ElevenLabs).
1313
single-shot `--screenshot`, it hangs on macOS).
1414
- `crates/videoeditor-media` — all ffmpeg/ffprobe invocations + assembly.
1515
- `crates/videoeditor-voice` — ElevenLabs TTS/STT (`ELEVENLABS_API_KEY`).
16+
- `crates/videoeditor-genai` — typed image-generation clients: xAI Grok
17+
Imagine (`XAI_API_KEY`, reference images) + Google Imagen (`AI_STUDIO`);
18+
Veo/Grok video is the planned next tenant.
1619
- `examples/hello-bench` — smallest end-to-end episode; keep it rendering.
1720

1821
## Commands

Cargo.lock

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ tungstenite = "0.24"
2121
ureq = { version = "2", features = ["json"] }
2222

2323
videoeditor-chrome = { version = "0.1.1", path = "crates/videoeditor-chrome" }
24+
videoeditor-genai = { version = "0.1.1", path = "crates/videoeditor-genai" }
2425
videoeditor-media = { version = "0.1.1", path = "crates/videoeditor-media" }
2526
videoeditor-timeline = { version = "0.1.1", path = "crates/videoeditor-timeline" }
2627
videoeditor-voice = { version = "0.1.1", path = "crates/videoeditor-voice" }
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[package]
2+
name = "videoeditor-genai"
3+
description = "Typed generative-asset clients for videoeditor: xAI Grok Imagine and Google Imagen stills (reference-image conditioning included)"
4+
keywords = ["image-generation", "grok", "imagen", "genai", "video"]
5+
categories = ["multimedia::images", "api-bindings"]
6+
version.workspace = true
7+
edition.workspace = true
8+
rust-version.workspace = true
9+
license.workspace = true
10+
repository.workspace = true
11+
authors.workspace = true
12+
13+
[dependencies]
14+
anyhow.workspace = true
15+
base64.workspace = true
16+
serde.workspace = true
17+
serde_json.workspace = true
18+
ureq.workspace = true
Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
//! Google Imagen stills via the Gemini API `:predict` endpoint.
2+
//!
3+
//! Safe-stills provider: honors a real aspect/size param, but takes NO
4+
//! reference images, and its safety filter rejects named real people and
5+
//! branded characters — route those requests to [`crate::xai`]. An empty
6+
//! `predictions` array almost always means the filter fired, so the error
7+
//! says so instead of "no images".
8+
//!
9+
//! Veo (video) rides the same API family and is this module's planned
10+
//! second resident.
11+
12+
use crate::{AspectRatio, GeneratedImage, ImageRequest};
13+
use anyhow::{Context, Result, bail};
14+
use serde::{Deserialize, Serialize};
15+
16+
pub const DEFAULT_MODEL: &str = "imagen-4.0-generate-001";
17+
pub const MAX_IMAGES_PER_REQUEST: u8 = 4;
18+
19+
/// The subset of [`AspectRatio`] Imagen accepts.
20+
pub const SUPPORTED_ASPECTS: [AspectRatio; 5] = [
21+
AspectRatio::R1x1,
22+
AspectRatio::R3x4,
23+
AspectRatio::R4x3,
24+
AspectRatio::R9x16,
25+
AspectRatio::R16x9,
26+
];
27+
28+
pub fn api_key() -> Result<String> {
29+
std::env::var("AI_STUDIO")
30+
.or_else(|_| std::env::var("GEMINI_API_KEY"))
31+
.context("set AI_STUDIO or GEMINI_API_KEY (a Google AI Studio key)")
32+
}
33+
34+
#[derive(Serialize)]
35+
struct PredictPayload<'a> {
36+
instances: [Instance<'a>; 1],
37+
parameters: Parameters<'a>,
38+
}
39+
40+
#[derive(Serialize)]
41+
struct Instance<'a> {
42+
prompt: &'a str,
43+
}
44+
45+
#[derive(Serialize)]
46+
#[serde(rename_all = "camelCase")]
47+
struct Parameters<'a> {
48+
sample_count: u8,
49+
#[serde(skip_serializing_if = "Option::is_none")]
50+
aspect_ratio: Option<&'a str>,
51+
}
52+
53+
#[derive(Deserialize)]
54+
#[serde(rename_all = "camelCase")]
55+
struct PredictResponse {
56+
#[serde(default)]
57+
predictions: Vec<Prediction>,
58+
}
59+
60+
#[derive(Deserialize)]
61+
#[serde(rename_all = "camelCase")]
62+
struct Prediction {
63+
bytes_base64_encoded: Option<String>,
64+
}
65+
66+
/// Generate `req.n` stills. Rejects reference images (Imagen has no such
67+
/// input — use grok) and aspect ratios outside [`SUPPORTED_ASPECTS`].
68+
pub fn generate(req: &ImageRequest) -> Result<Vec<GeneratedImage>> {
69+
if req.n == 0 || req.n > MAX_IMAGES_PER_REQUEST {
70+
bail!(
71+
"imagen generates 1..={MAX_IMAGES_PER_REQUEST} images per request, got {}",
72+
req.n
73+
);
74+
}
75+
if !req.reference_images.is_empty() {
76+
bail!(
77+
"imagen takes no reference images — use the grok provider for reference-conditioned stills"
78+
);
79+
}
80+
if req.aspect != AspectRatio::Auto && !SUPPORTED_ASPECTS.contains(&req.aspect) {
81+
bail!(
82+
"imagen supports aspect ratios 1:1, 3:4, 4:3, 9:16, 16:9 (got {})",
83+
req.aspect
84+
);
85+
}
86+
87+
let model = req.model.as_deref().unwrap_or(DEFAULT_MODEL);
88+
let payload = PredictPayload {
89+
instances: [Instance {
90+
prompt: &req.prompt,
91+
}],
92+
parameters: Parameters {
93+
sample_count: req.n,
94+
aspect_ratio: match req.aspect {
95+
AspectRatio::Auto => None,
96+
a => Some(a.as_str()),
97+
},
98+
},
99+
};
100+
101+
let url = format!("https://generativelanguage.googleapis.com/v1beta/models/{model}:predict");
102+
let resp = ureq::post(&url)
103+
.set("x-goog-api-key", &api_key()?)
104+
.timeout(std::time::Duration::from_secs(300))
105+
.send_json(serde_json::to_value(&payload)?);
106+
let resp = match resp {
107+
Ok(r) => r,
108+
Err(ureq::Error::Status(code, r)) => {
109+
bail!("imagen {code}: {}", r.into_string().unwrap_or_default())
110+
}
111+
Err(e) => return Err(e.into()),
112+
};
113+
let parsed: PredictResponse = resp.into_json().context("parsing imagen response")?;
114+
if parsed.predictions.is_empty() {
115+
bail!(
116+
"imagen returned no images — likely a safety-filter rejection \
117+
(named real people, branded characters, explicit content). \
118+
Rewrite the prompt or switch to the grok provider."
119+
);
120+
}
121+
122+
parsed
123+
.predictions
124+
.into_iter()
125+
.map(|p| {
126+
let b64 = p
127+
.bytes_base64_encoded
128+
.context("imagen prediction had no image bytes")?;
129+
Ok(GeneratedImage {
130+
bytes: crate::decode_b64(&b64)?,
131+
revised_prompt: None,
132+
})
133+
})
134+
.collect()
135+
}
136+
137+
#[cfg(test)]
138+
mod tests {
139+
use super::*;
140+
141+
#[test]
142+
fn payload_uses_camel_case_fields() {
143+
let payload = PredictPayload {
144+
instances: [Instance { prompt: "a bun" }],
145+
parameters: Parameters {
146+
sample_count: 2,
147+
aspect_ratio: Some(AspectRatio::R9x16.as_str()),
148+
},
149+
};
150+
let v = serde_json::to_value(&payload).unwrap();
151+
assert_eq!(v["parameters"]["sampleCount"], 2);
152+
assert_eq!(v["parameters"]["aspectRatio"], "9:16");
153+
assert_eq!(v["instances"][0]["prompt"], "a bun");
154+
}
155+
156+
#[test]
157+
fn refs_are_rejected() {
158+
let req = ImageRequest {
159+
prompt: "x".into(),
160+
model: None,
161+
n: 1,
162+
aspect: AspectRatio::Auto,
163+
reference_images: vec!["a.png".into()],
164+
};
165+
assert!(generate(&req).unwrap_err().to_string().contains("grok"));
166+
}
167+
}

0 commit comments

Comments
 (0)