Add agentic assistant panel to the image editor

Bring the timeline/storyboard editor's agent-chat pattern to the image
(sketch) editor: a toggleable right-side assistant panel whose LLM can
drive the canvas through a set of ui_sketch_* frontend tools.

- sketchAgentBridge: serializable handler interface + snapshot types, with
  get/set/has, mirroring timelineAgentBridge.
- useSketchAgentBridge: builds the handler from the editor's per-instance
  stores (editor, session bindings, canvas refs) plus the direct-gen job
  runner, and registers it while the editor is the focused surface.
- ui_sketch_* tools (lib/tools/builtin/sketch.ts): get_state, add/remove/
  duplicate/select layer, set_layer_props, reorder, merge_down,
  flatten_visible, generate (text-to-image / image-to-image), set_color,
  set_tool, resize_canvas, selection, and get_layer_image (vision). Wired
  into the frontend-tools IPC manifest.
- SketchAgentPanel: reuses ChatView + GlobalChatStore, passing the open
  document id as workflow_id so the server forwards the ui_sketch_* tools.
- Mounted as a toggleable column in SketchEditor with an Assistant toggle
  in the prompt bar; assistantPanelOpen flag added to the ui slice.
- Extend SketchCanvasRefStore with getLayerData / fillLayerWithColor so the
  bridge can read and fill layers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WyF6AfWZPdihy1TLKLi9ca

Author: claude

web/src/components/sketch/SketchAgentPanel.tsx

@@ -0,0 +1,171 @@
+/** @jsxImportSource @emotion/react */
+import { memo, useCallback, useEffect, useMemo } from "react";
+import { css } from "@emotion/react";
+import { useTheme } from "@mui/material/styles";
+import type { Theme } from "@mui/material/styles";
+import { useShallow } from "zustand/react/shallow";
+
+import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
+
+import { FlexColumn, Text, SPACING, getSpacingPx } from "../ui_primitives";
+import ChatView from "../chat/containers/ChatView";
+import ChatPanelHeader from "../chat/containers/ChatPanelHeader";
+import useGlobalChatStore from "../../stores/GlobalChatStore";
+import { useSketchSessionStore } from "../../stores/sketch/SketchSessionStore";
+
+const styles = (_theme: Theme) =>
+  css({
+    "&": {
+      height: "100%",
+      minHeight: 0,
+      display: "flex",
+      flexDirection: "column"
+    },
+    // ChatView is tuned for the full-page global chat (large left/bottom
+    // padding, centered max-width). Tighten it for this narrow side panel.
+    "& .chat-view": {
+      padding: `0 ${getSpacingPx(SPACING.xs)} ${getSpacingPx(
+        SPACING.xs
+      )} ${getSpacingPx(SPACING.xs)}`
+    },
+    "& .chat-input-section": {
+      width: "100%",
+      maxWidth: "100%"
+    },
+    "& .chat-thread-container": {
+      maxWidth: "100%",
+      paddingBottom: getSpacingPx(SPACING.md)
+    }
+  });
+
+/**
+ * Chat surface for the image / sketch editor. Reuses {@link ChatView} wired to
+ * the shared {@link useGlobalChatStore}, so the assistant can call the
+ * `ui_sketch_*` frontend tools the editor registers on the sketch agent
+ * bridge — adding layers, generating imagery, and reshaping the canvas like a
+ * real editor.
+ */
+const SketchAgentPanel = () => {
+  const theme = useTheme();
+  const cssStyles = useMemo(() => styles(theme), [theme]);
+
+  // Bind the open document as the chat's `workflow_id`. The server only
+  // forwards client `ui_*` tools to the model when a turn carries a
+  // workflow_id (unified-websocket-runner gates on it), so without this the
+  // assistant never sees the editor's ui_sketch_* tools. The id is editor
+  // context, not a routing signal — we don't set `workflow_target`, so the turn
+  // stays a normal chat turn and the document is never run as a workflow.
+  const documentId = useSketchSessionStore((s) => s.documentId);
+
+  const { status, statusMessage, progress } = useGlobalChatStore(
+    useShallow((state) => ({
+      status: state.status,
+      statusMessage: state.statusMessage,
+      progress: state.progress
+    }))
+  );
+
+  const { selectedModel, setSelectedModel } = useGlobalChatStore(
+    useShallow((state) => ({
+      selectedModel: state.selectedModel,
+      setSelectedModel: state.setSelectedModel
+    }))
+  );
+
+  const { sendMessage, stopGeneration, connect, createNewThread, switchThread } =
+    useGlobalChatStore(
+      useShallow((state) => ({
+        sendMessage: state.sendMessage,
+        stopGeneration: state.stopGeneration,
+        connect: state.connect,
+        createNewThread: state.createNewThread,
+        switchThread: state.switchThread
+      }))
+    );
+
+  // Subscribe to the cache + current thread so the panel re-renders as the
+  // active conversation streams in; the messages themselves are read synchronously.
+  const { currentThreadId, messageCache, getCurrentMessagesSync } =
+    useGlobalChatStore(
+      useShallow((state) => ({
+        currentThreadId: state.currentThreadId,
+        messageCache: state.messageCache,
+        getCurrentMessagesSync: state.getCurrentMessagesSync
+      }))
+    );
+  const messages = useMemo(
+    () => getCurrentMessagesSync(),
+    [getCurrentMessagesSync, currentThreadId, messageCache]
+  );
+
+  // Establish the chat connection (and send the frontend-tool manifest, which
+  // now includes the editor's ui_sketch_* tools) when the panel mounts.
+  useEffect(() => {
+    connect().catch((err) => {
+      console.error("Failed to connect image editor chat:", err);
+    });
+  }, [connect]);
+
+  const chatStatus = useMemo(
+    () => (status === "stopping" ? "loading" : status),
+    [status]
+  );
+
+  const handleNewChat = useCallback(async () => {
+    try {
+      const id = await createNewThread();
+      switchThread(id);
+    } catch (err) {
+      console.error("Failed to start new image editor chat:", err);
+    }
+  }, [createNewThread, switchThread]);
+
+  const welcomePlaceholder = useMemo(
+    () => (
+      <FlexColumn
+        align="center"
+        justify="center"
+        fullHeight
+        padding={3}
+        sx={{ textAlign: "center" }}
+      >
+        <AutoAwesomeIcon sx={{ fontSize: 40, mb: 1.5, opacity: 0.5 }} />
+        <Text size="normal" weight={600} sx={{ mb: 1 }}>
+          Editor Assistant
+        </Text>
+        <Text size="small" color="secondary" sx={{ maxWidth: 280 }}>
+          Ask me to edit the image — e.g. &quot;generate a mountain landscape on
+          a new layer&quot;, &quot;add a blank layer filled with black&quot;, or
+          &quot;set the background layer to 50% opacity&quot;.
+        </Text>
+      </FlexColumn>
+    ),
+    []
+  );
+
+  return (
+    <div css={cssStyles}>
+      <ChatPanelHeader onNewChat={handleNewChat} />
+      <div style={{ flex: 1, minHeight: 0 }}>
+        <ChatView
+          status={chatStatus}
+          messages={messages}
+          workflowId={documentId}
+          sendMessage={sendMessage}
+          progress={progress.current}
+          total={progress.total}
+          progressMessage={statusMessage}
+          model={selectedModel}
+          onModelChange={setSelectedModel}
+          onStop={stopGeneration}
+          onNewChat={handleNewChat}
+          requireToolSupport
+          hideModePicker
+          noMessagesPlaceholder={welcomePlaceholder}
+        />
+      </div>
+    </div>
+  );
+};
+
+export default memo(SketchAgentPanel);

web/src/components/sketch/SketchEditor.tsx

@@ -69,6 +69,8 @@ import {
   SketchCanvasPane
 } from "./editor-shell";
 import { ConnectedGeneratedLayerSection } from "./Inspector";
+import SketchAgentPanel from "./SketchAgentPanel";
+import { useSketchAgentBridge } from "../../hooks/sketch/useSketchAgentBridge";
 import { useSketchCanvasRefStore } from "../../stores/sketch/SketchCanvasRefStore";
 import { useSketchSessionStore } from "../../stores/sketch/SketchSessionStore";
 import { useSketchWorkflowFreshnessCheck } from "../../hooks/sketch/useSketchWorkflowFreshnessCheck";
@@ -182,6 +184,12 @@ export interface SketchEditorProps {
   /** Document-level actions rendered at the trailing edge of the top mode bar
    * (e.g. Save/Done when embedded in an asset tab). */
   headerActions?: React.ReactNode;
+  /**
+   * Whether this editor is the focused/visible surface. Drives whether this
+   * instance registers the agent bridge so the `ui_sketch_*` tools target the
+   * focused document. Defaults to `true`.
+   */
+  active?: boolean;
 }
 
 const SketchEditor = forwardRef<SketchEditorHandle, SketchEditorProps>(
@@ -194,7 +202,8 @@ const SketchEditor = forwardRef<SketchEditorHandle, SketchEditorProps>(
       onExportImage,
       onExportMask,
       suspendKeyboardShortcuts,
-      headerActions
+      headerActions,
+      active = true
     },
     ref
   ) {
@@ -205,6 +214,11 @@ const SketchEditor = forwardRef<SketchEditorHandle, SketchEditorProps>(
     // every child so the column's reserved width also collapses, letting
     // the canvas grow into the freed space.
     const panelsHidden = useSketchStore((s) => s.panelsHidden);
+    const assistantPanelOpen = useSketchStore((s) => s.assistantPanelOpen);
+
+    // Register the agent bridge for this instance while it is the focused
+    // surface so the `ui_sketch_*` tools drive this document.
+    useSketchAgentBridge(active);
 
     // ─── Session layer (all transient editor-session state) ─────────────
     const session = useEditorSession({
@@ -246,6 +260,10 @@ const SketchEditor = forwardRef<SketchEditorHandle, SketchEditorProps>(
         getMaskDataUrl: () => canvasRef.current?.getMaskDataUrl() ?? null,
         setLayerData: (layerId, data) =>
           canvasRef.current?.setLayerData(layerId, data),
+        getLayerData: (layerId) =>
+          canvasRef.current?.getLayerData(layerId) ?? null,
+        fillLayerWithColor: (layerId, color) =>
+          canvasRef.current?.fillLayerWithColor(layerId, color),
         clearActiveLayer: () => session.canvasActions.handleClearLayer()
       });
       return () => {
@@ -531,6 +549,28 @@ const SketchEditor = forwardRef<SketchEditorHandle, SketchEditorProps>(
               <ConnectedGeneratedLayerSection />
             </FlexColumn>
           )}
+
+          {/* AI assistant chat column — a toggleable right-most panel that
+          drives the editor through the ui_sketch_* agent tools. Gated on the
+          same panelsHidden chrome toggle so Tab collapses it too. */}
+          {!panelsHidden && assistantPanelOpen && (
+            <FlexColumn
+              className="sketch-editor__assistant-panel"
+              sx={{
+                width: SKETCH_SIZE.assistantPanelWidth,
+                minWidth: SKETCH_SIZE.assistantPanelWidth,
+                maxWidth: SKETCH_SIZE.assistantPanelWidth,
+                minHeight: 0,
+                flexShrink: 0,
+                backgroundColor: theme.vars.palette.background.paper,
+                borderLeft: `1px solid ${theme.vars.palette.divider}`,
+                overflow: "hidden"
+              }}
+              gap={0}
+            >
+              <SketchAgentPanel />
+            </FlexColumn>
+          )}
         </FlexRow>
 
         {/* Full-width status bar — standalone editor only (gates internally). */}

web/src/components/sketch/StandaloneSketchEditor.tsx

@@ -68,10 +68,8 @@ interface StandaloneSketchEditorProps {
   active?: boolean;
 }
 
-const StandaloneSketchEditorBody: React.FC<
-  Omit<StandaloneSketchEditorProps, "active">
-> = memo(
-  function StandaloneSketchEditorBody({ documentId, headerActions }) {
+const StandaloneSketchEditorBody: React.FC<StandaloneSketchEditorProps> = memo(
+  function StandaloneSketchEditorBody({ documentId, headerActions, active }) {
     const theme = useTheme();
     const styles = useMemo(() => containerStyles(theme), [theme]);
     const editorRef = useRef<SketchEditorHandle | null>(null);
@@ -213,6 +211,7 @@ const StandaloneSketchEditorBody: React.FC<
         <SketchEditor
           ref={editorRef}
           documentId={documentId}
+          active={active}
           initialDocument={seed.document}
           initialEditorState={initialEditorState ?? undefined}
           headerActions={
@@ -250,7 +249,7 @@ const StandaloneSketchEditor: React.FC<StandaloneSketchEditorProps> = ({
   ...bodyProps
 }) => (
   <SketchProvider active={active}>
-    <StandaloneSketchEditorBody {...bodyProps} />
+    <StandaloneSketchEditorBody active={active} {...bodyProps} />
   </SketchProvider>
 );
 

web/src/components/sketch/editor-shell/ConnectedModePromptBar.tsx

@@ -83,6 +83,8 @@ const ConnectedModePromptBarInner: React.FC<ConnectedModePromptBarProps> = ({
   const theme = useTheme();
 
   const panelsHidden = useSketchStore((s) => s.panelsHidden);
+  const assistantPanelOpen = useSketchStore((s) => s.assistantPanelOpen);
+  const toggleAssistantPanel = useSketchStore((s) => s.toggleAssistantPanel);
   const docW = useSketchStore((s) => s.document.canvas.width);
   const docH = useSketchStore((s) => s.document.canvas.height);
 
@@ -357,6 +359,20 @@ const ConnectedModePromptBarInner: React.FC<ConnectedModePromptBarProps> = ({
           Generate
         </EditorButton>
 
+        {/* Assistant toggle — opens the right-side AI chat panel that drives
+            the editor through the ui_sketch_* agent tools. */}
+        <EditorButton
+          variant={assistantPanelOpen ? "contained" : "outlined"}
+          size="small"
+          onClick={toggleAssistantPanel}
+          startIcon={<AutoAwesomeIcon fontSize="small" />}
+          aria-pressed={assistantPanelOpen}
+          data-testid="sketch-assistant-toggle"
+          sx={{ flexShrink: 0, height: 34 }}
+        >
+          Assistant
+        </EditorButton>
+
         {trailingActions}
       </FlexRow>