feat: add altText mode to ai utilities (#662)

Author: jeremydw

.changeset/rude-lobsters-grab.md

@@ -0,0 +1,5 @@
+---
+'@blinkk/root-cms': patch
+---
+
+feat: add ai alt text generation

packages/root-cms/core/ai.ts

@@ -5,7 +5,11 @@ import {vertexAI} from '@genkit-ai/vertexai';
 import {Timestamp} from 'firebase-admin/firestore';
 import {Genkit, genkit, MessageData} from 'genkit';
 import {logger} from 'genkit/logging';
-import {ChatPrompt, SendPromptOptions} from '../shared/ai/prompts.js';
+import {
+  ChatPrompt,
+  AiResponse,
+  SendPromptOptions,
+} from '../shared/ai/prompts.js';
 import {RootCMSClient} from './client.js';
 import {CMSPluginOptions} from './plugin.js';
 
@@ -64,7 +68,10 @@ export class Chat {
     options: SendPromptOptions
   ): Promise<MessageData[]> {
     const messages = this.history;
-    if (messages.length === 0) {
+    const hasSystemPrompt = messages.some(
+      (msg) => msg.role === 'system' && msg.content.length > 0
+    );
+    if (!hasSystemPrompt) {
       messages.push({
         role: 'system',
         content: [{text: await this.buildSystemPrompt(options)}],
@@ -89,10 +96,14 @@ export class Chat {
   }
 
   /** Builds the request sent to the AI based on the `ChatMode`. */
-  private async buildChatRequest(
+  private async buildGenerateRequest(
     prompt: ChatPrompt | ChatPrompt[],
     options: SendPromptOptions
-  ) {
+  ): Promise<{
+    messages: MessageData[];
+    model: string;
+    prompt: ChatPrompt | ChatPrompt[];
+  }> {
     if (options.mode === 'edit') {
       return {
         messages: await this.buildMessages(options),
@@ -111,8 +122,8 @@ export class Chat {
   async sendPrompt(
     prompt: ChatPrompt | ChatPrompt[],
     options: SendPromptOptions = {}
-  ): Promise<string> {
-    const chatRequest = await this.buildChatRequest(prompt, options);
+  ): Promise<AiResponse> {
+    const chatRequest = await this.buildGenerateRequest(prompt, options);
     // TODO: Use streaming responses per https://genkit.dev/docs/models/#streaming
     // to improve UI performance.
     const res = await this.ai.generate({
@@ -125,7 +136,11 @@ export class Chat {
       history: this.history,
       modifiedAt: Timestamp.now(),
     });
-    return res.text;
+    // Using the `output` property provides both data and text responses.
+    if (options.mode === 'edit') {
+      return res.output as AiResponse & {editData?: any};
+    }
+    return {message: res.text, data: null};
   }
 
   dbDoc() {
@@ -147,15 +162,28 @@ export class Chat {
     // Edit mode prompts.
     if (options.mode === 'edit') {
       const rootDir = process.cwd();
-      const rootCmsDefs = fs.readFileSync(
-        path.resolve(rootDir, 'root-cms.d.ts'),
-        {
-          encoding: 'utf8',
-        }
-      );
-      const text = (await import('../shared/ai/prompts/edit.txt')).default;
-      text.replace('{{ROOT_CMS_DEFS}}', rootCmsDefs);
-      return text;
+      // The `root-cms.d.ts` file may not be bundled with the server code,
+      // so check whether it exists first before attempting to add it to the prompt.
+      const rootCmsDefsPath = path.resolve(rootDir, 'root-cms.d.ts');
+      const rootCmsDefs = fs.existsSync(rootCmsDefsPath)
+        ? fs.readFileSync(rootCmsDefsPath, {
+            encoding: 'utf8',
+          })
+        : null;
+      const text = [(await import('../shared/ai/prompts/edit.txt')).default];
+      if (rootCmsDefs) {
+        text.push(
+          'Here is the `root-cms.d.ts` file for this project:',
+          '```',
+          rootCmsDefs,
+          '```'
+        );
+      }
+      return text.join('\n');
+    }
+
+    if (options.mode === 'altText') {
+      return (await import('../shared/ai/prompts/altText.txt')).default;
     }
 
     // Chat mode (default) prompts.
@@ -188,7 +216,7 @@ export class ChatClient {
     this.user = user;
   }
 
-  async createChat(): Promise<Chat> {
+  private async createChat(): Promise<Chat> {
     const chatId = crypto.randomUUID();
     // Save chat to db so that user has a chat history and can enable "sharing"
     // with others. Store the model used with the metadata.
@@ -204,7 +232,11 @@ export class ChatClient {
     return chat;
   }
 
-  async getChat(chatId: string): Promise<Chat> {
+  async getOrCreateChat(chatId?: string): Promise<Chat> {
+    return chatId ? this.getChat(chatId) : this.createChat();
+  }
+
+  private async getChat(chatId: string): Promise<Chat> {
     // Fetch chat from db to preserve the conversation's history.
     const docRef = this.dbCollection().doc(chatId);
     const chatDoc = await docRef.get();

packages/root-cms/core/api.ts

@@ -1,7 +1,11 @@
 import {Server, Request, Response} from '@blinkk/root';
 import {multipartMiddleware} from '@blinkk/root/middleware';
-import {ChatPrompt, SendPromptOptions} from '../shared/ai/prompts.js';
-import {Chat, ChatClient, RootAiModel} from './ai.js';
+import {
+  ChatPrompt,
+  AiResponse,
+  SendPromptOptions,
+} from '../shared/ai/prompts.js';
+import {ChatClient, RootAiModel} from './ai.js';
 import {RootCMSClient} from './client.js';
 import {runCronJobs} from './cron.js';
 import {arrayToCsv, csvToArray} from './csv.js';
@@ -15,6 +19,13 @@ export interface ChatApiRequest {
   options?: SendPromptOptions;
 }
 
+export interface ChatApiResponse {
+  success: boolean;
+  chatId: string;
+  response: AiResponse;
+  error?: string;
+}
+
 export interface ApiOptions {
   getRenderer: (req: Request) => Promise<AppModule>;
 }
@@ -258,19 +269,16 @@ export function api(server: Server, options: ApiOptions) {
     try {
       const cmsClient = new RootCMSClient(req.rootConfig!);
       const chatClient = new ChatClient(cmsClient, req.user.email);
-      let chat: Chat;
-      if (reqBody.chatId) {
-        chat = await chatClient.getChat(reqBody.chatId);
-      } else {
-        chat = await chatClient.createChat();
-      }
-      const chatResponse = await chat.sendPrompt(prompt, {
-        mode: reqBody.options?.mode || 'chat',
-        editData: reqBody.options?.editData,
-      });
-      res
-        .status(200)
-        .json({success: true, chatId: chat.id, response: chatResponse});
+      const chat = await chatClient.getOrCreateChat(reqBody.chatId);
+      const apiResponse: ChatApiResponse = {
+        success: true,
+        chatId: chat.id,
+        response: await chat.sendPrompt(prompt, {
+          mode: reqBody.options?.mode || 'chat',
+          editData: reqBody.options?.editData,
+        }),
+      };
+      res.status(200).json(apiResponse);
     } catch (err) {
       console.error(err.stack || err);
       res.status(500).json({success: false, error: 'UNKNOWN'});

packages/root-cms/core/app.tsx

@@ -229,11 +229,7 @@ export async function renderSignIn(
     firebaseConfig: options.cmsConfig.firebaseConfig,
   };
   const mainHtml = renderToString(
-    <SignIn
-      title="Sign in"
-      ctx={ctx}
-      favicon={options.cmsConfig.favicon}
-    />
+    <SignIn title="Sign in" ctx={ctx} favicon={options.cmsConfig.favicon} />
   );
   let html = `<!doctype html>\n${mainHtml}`;
   const nonce = generateNonce();

packages/root-cms/shared/ai/prompts.ts

@@ -9,14 +9,12 @@ export type ChatPrompt = Part | Part[];
  * - chat: Used in the "Chat with Root AI" sidebar tool.
  * - edit: Used in the "Edit with AI" modal dialog for editing JSON files.
  */
-export type ChatMode = 'chat' | 'edit';
+export type ChatMode = 'chat' | 'edit' | 'altText';
 
-/** The parsed response from the AI */
-export interface ParsedChatResponse {
-  /** The textual response from the AI. For chat requests, this is the full response. For other types of requests, this contains just the AI's "message" response, whereas any other data will be included in the `data` property. */
+export interface AiResponse {
   message: string;
-  /** JSON data returned by the AI. */
-  data: Record<string, any>;
+  data: Record<string, any> | null;
+  error?: string;
 }
 
 /**
@@ -29,29 +27,3 @@ export interface SendPromptOptions {
   /** Data sent for edit mode requests. */
   editData?: Record<string, any>;
 }
-
-/**
- * Parses the response from the AI request. The response structure differs
- * depending on the prompt and the mode. For example, chat mode receives a text
- * response. Edit mode receives a response containing JSON, which must be
- * parsed out from the text response.
- */
-export function parseResponse(
-  response: string,
-  mode?: ChatMode
-): ParsedChatResponse {
-  // TODO(jeremydw): Research whether we can require the AI or genkit to respond in structured
-  // JSON format, so that we don't have to parse the response ourselves.
-  // Use https://genkit.dev/docs/models/#structured-output to enforce structured output
-  // instead of parsing the response.
-  if (mode === 'edit') {
-    const jsonMatch = response.match(/^```json\n?(.*?)\n?```$/s);
-    const responseAsJson = JSON.parse(jsonMatch ? jsonMatch[1] : response);
-    return {
-      message: responseAsJson.message,
-      data: responseAsJson.data,
-    };
-  }
-  // Chat messages don't contain any data, just the chat response.
-  return {message: response, data: {}};
-}

packages/root-cms/shared/ai/prompts/altText.txt

@@ -0,0 +1,5 @@
+Create a descriptive and concise alt text for the attached image.
+
+- The alt text should be a brief but comprehensive description of the image, including key subjects, the setting, and any relevant details or actions.
+- The alt text should not exceed 125 characters.
+- Only provide one generation, and include only that data in the response. No surrounding text, clarifications, etc. Just the alt text.

packages/root-cms/shared/ai/prompts/edit.txt

@@ -10,7 +10,7 @@ Input Requirements & Handling:
 
 - The input you receive will be broken into three parts. (1) `root-cms-d.ts` (the TypeScript interface), (2) the existing JSON file you are editing (or an empty JSON file when generating new content), and (3) the user input (screenshots, chat messages, instructions, etc.). You must ensure that the resulting JSON file is compatible with the structure defined by the TypeScript interface and contains all of the marketing content provided by the user input.
 
-- When you provide your response, if the user instructs you to only edit one part of the original JSON, make sure to leave the rest of the JSON file untouched. Your response, however, should always contain the _entire_ JSON file combining the original contents with your new edits (think of it like you're applying a PATCH operation and the response is the entire file).
+- When you provide your response, if the user instructs you to only edit one part of the original JSON, make sure to leave the rest of the JSON file untouched. You must return the entire JSON document after applying changes. Never omit, rename, reorder or invent top-level keys. Copy all unmodified fields exactly as they appear in the input.
 
 Root CMS JSON File Structure:
 
@@ -111,16 +111,10 @@ When adding new items to arrays that use `oneOf` fields, try your best to figure
 
 Additionally, when adding new items to existing JSON data, you must carefully check the structure of those items as defined by `root-cms.d.ts`. Avoid inventing new fields for content types, use only those defined by the `root-cms.d.ts`. For example, if you are adding a button to a module, carefully check the button structure defined in `root-cms.d.ts` and only populate fields that are defined there.
 
-5. Here is the `root-cms.d.ts` file for this project:
-
-```
-{{ROOT_CMS_DEFS}}
-```
-
 Finally, when you provide your response, it MUST be structured in the following way for handling by the chat client.
 
 {
-  "data": <The JSON generated by you via the instructions above>,
+  "data": <The JSON data generated by you via the instructions above>,
   "message": <A brief message that describes what you did.>
 }
 

packages/root-cms/signin/styles/global.css

@@ -90,6 +90,7 @@ body.menu\:open {
 .bootstrap-error {
   font-size: 14px;
   font-weight: 400;
+  line-height: 20px;
   position: absolute;
   bottom: 100px;
   left: 0;
@@ -98,6 +99,7 @@ body.menu\:open {
   animation: 0.5s forwards bootstrap-loading-error;
   animation-delay: 1s;
   opacity: 0;
+  padding: 0 24px;
 }
 
 @keyframes bootstrap-loading-error {

packages/root-cms/ui/components/AiEditModal/AiEditModal.css

@@ -5,48 +5,62 @@
   gap: 12px;
 }
 
-.AiEditModal__JsonInput,
-.AiEditModal__JsonInput .mantine-JsonInput-wrapper,
-.AiEditModal__JsonInput .mantine-JsonInput-input {
-  height: 100%;
-}
-
 .AiEditModal__JsonInput .mantine-JsonInput-input {
   font-size: 0.875rem;
   line-height: 1.5;
 }
 
 .AiEditModal__SplitPanel {
   display: grid;
-  grid-template-columns: 50% 50%;
+  grid-template-columns: 1fr 1fr;
   gap: 24px;
+  height: 680px;
+  max-height: calc(100vh - 280px);
+}
+
+.AiEditModal__SplitPanel__JsonPanel {
+  display: flex;
+  flex-direction: column;
+  height: 100%;
 }
 
 .AiEditModal__JsonEditor {
   height: 100%;
-  overflow: auto;
+  width: 100%;
+}
+
+.AiEditModal__JsonInput,
+.AiEditModal__JsonInput .mantine-JsonInput-wrapper,
+.AiEditModal__JsonInput .mantine-JsonInput-input {
+  height: 100%;
 }
 
 .AiEditModal__SplitPanel__ChatPanel {
   height: 100%;
   overflow: auto;
   display: flex;
   flex-direction: column;
-  height: calc(max(100vh - 300px, 400px));
 }
 
-.AiEditModal__Tabs,
-.AiEditModal__Tabs .mantine-Tabs-root, 
+.AiEditModal__Tabs {
+  display: flex;
+  flex: 1;
+  flex-direction: column;
+}
+
 .AiEditModal__Tabs .mantine-Tabs-body {
+  display: flex;
+  flex: 1 1 0;
   height: 100%;
+  min-height: 0;
 }
 
 .AiEditModal__JsonDiffViewer {
-  border: 1px solid rgb(206, 212, 218);
   border-radius: 4px;
+  border: 1px solid rgb(206, 212, 218);
   height: 100%;
-  max-height: calc(max(100vh - 300px, 400px));
   overflow: auto;
+  width: 100%;
 }
 
 .AiEditModal__JsonDiffViewer .JsDiff__container {