fix(gh-test-evidence): add batch limiting and update repo URL

- Update comfyanonymous/ComfyUI to Comfy-Org/ComfyUI
- Add batch limit (50 PRs) and time limit (8 min) to avoid CI timeout
- Skip PRs that don't need updates (already analyzed + correct comment state)
- Return early when limits are reached

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: snomiao

app/tasks/gh-test-evidence/gh-test-evidence.ts

@@ -10,14 +10,25 @@ import { OpenAI } from "openai";
 import { pageFlow } from "sflow";
 import z from "zod";
 
-const REPOS = ["https://github.com/Comfy-Org/desktop", "https://github.com/comfyanonymous/ComfyUI"];
+const REPOS = ["https://github.com/Comfy-Org/desktop", "https://github.com/Comfy-Org/ComfyUI"];
 
 const BOT_COMMENT_MARKER = "<!-- COMFY_PR_BOT_TEST_EVIDENCE -->";
 
+// Batch limit to avoid CI timeout (10 min limit, ~3s per PR = ~200 max, use 50 for safety)
+const MAX_PRS_PER_RUN = 50;
+// Time limit in ms (8 minutes to leave buffer before 10 min timeout)
+const TIME_LIMIT_MS = 8 * 60 * 1000;
+
 const TestEvidenceSchema = z.object({
-  isTestExplanationIncluded: z.boolean().describe("true if PR body includes test plan or test explanation"),
-  isTestScreenshotIncluded: z.boolean().describe("true if PR body includes test screenshots or images"),
-  isTestVideoIncluded: z.boolean().describe("true if PR body includes test videos or YouTube links"),
+  isTestExplanationIncluded: z
+    .boolean()
+    .describe("true if PR body includes test plan or test explanation"),
+  isTestScreenshotIncluded: z
+    .boolean()
+    .describe("true if PR body includes test screenshots or images"),
+  isTestVideoIncluded: z
+    .boolean()
+    .describe("true if PR body includes test videos or YouTube links"),
 });
 
 type TestEvidence = z.infer<typeof TestEvidenceSchema>;
@@ -61,6 +72,8 @@ if (import.meta.main) {
 
 export default async function runGhTestEvidenceTask() {
   console.log("Starting test evidence check task...");
+  const startTime = Date.now();
+  let processedCount = 0;
 
   for (const repoUrl of REPOS) {
     console.log(`Processing repo: ${repoUrl}`);
@@ -81,22 +94,64 @@ export default async function runGhTestEvidenceTask() {
     console.log(`Found ${prs.length} open PRs in ${repoUrl}`);
 
     for (const pr of prs) {
+      // Check time limit
+      const elapsed = Date.now() - startTime;
+      if (elapsed >= TIME_LIMIT_MS) {
+        console.log(
+          `Time limit reached (${Math.round(elapsed / 1000)}s), stopping. Processed ${processedCount} PRs.`,
+        );
+        return;
+      }
+
+      // Check batch limit
+      if (processedCount >= MAX_PRS_PER_RUN) {
+        console.log(`Batch limit reached (${MAX_PRS_PER_RUN} PRs), stopping.`);
+        return;
+      }
+
       try {
-        await processPR(pr, repoUrl);
+        const wasProcessed = await processPR(pr, repoUrl);
+        if (wasProcessed) processedCount++;
       } catch (error) {
         console.error(`Error processing PR ${pr.html_url}:`, error);
       }
     }
   }
 
-  console.log("Test evidence check task completed");
+  console.log(`Test evidence check task completed. Processed ${processedCount} PRs.`);
 }
 
-async function processPR(pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][0], repoUrl: string) {
+/** Process a PR and return true if any work was done (API calls made) */
+async function processPR(
+  pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][0],
+  repoUrl: string,
+): Promise<boolean> {
   // Skip drafts
   if (pr.draft) {
     console.log(`Skipping draft PR: ${pr.html_url}`);
-    return;
+    return false;
+  }
+
+  // Check existing task state
+  const existingTask = await GithubTestEvidenceTask.findOne({ prUrl: pr.html_url });
+
+  // Check if PR was updated since last analysis
+  const prUpdatedAt = new Date(pr.updated_at);
+  const needsAnalysis =
+    !existingTask?.evidenceAnalyzedAt || prUpdatedAt > existingTask.evidenceAnalyzedAt;
+
+  // If no analysis needed and we have comment state tracked, skip entirely
+  if (!needsAnalysis && existingTask?.evidence) {
+    const hasMissingEvidence =
+      !existingTask.evidence.isTestExplanationIncluded ||
+      (!existingTask.evidence.isTestScreenshotIncluded &&
+        !existingTask.evidence.isTestVideoIncluded);
+    const hasComment = !!existingTask.commentId;
+
+    // If state is consistent (has warning + has comment, or no warning + no comment), skip
+    if ((hasMissingEvidence && hasComment) || (!hasMissingEvidence && !hasComment)) {
+      return false;
+    }
   }
 
   // Save basic PR info
@@ -105,13 +160,10 @@ async function processPR(pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][
     prNumber: pr.number,
     prTitle: pr.title,
     prBody: pr.body,
-    prUpdatedAt: new Date(pr.updated_at),
+    prUpdatedAt,
     repoUrl,
   });
 
-  // Check if we need to re-analyze (PR was updated after last analysis)
-  const needsAnalysis = !task.evidenceAnalyzedAt || new Date(pr.updated_at) > task.evidenceAnalyzedAt;
-
   if (needsAnalysis) {
     console.log(`Analyzing PR: ${pr.html_url}`);
     const evidence = await analyzeTestEvidence(pr);
@@ -182,9 +234,13 @@ async function processPR(pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][
       });
     }
   }
+
+  return true;
 }
 
-async function analyzeTestEvidence(pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][0]): Promise<TestEvidence> {
+async function analyzeTestEvidence(
+  pr: Awaited<ReturnType<typeof ghc.pulls.list>>["data"][0],
+): Promise<TestEvidence> {
   const openai = new OpenAI({
     apiKey: process.env.OPENAI_API_KEY || DIE("OPENAI_API_KEY not found"),
   });
@@ -218,7 +274,11 @@ Be lenient - if there's any indication of testing explanation or visual evidence
             isTestScreenshotIncluded: { type: "boolean" },
             isTestVideoIncluded: { type: "boolean" },
           },
-          required: ["isTestExplanationIncluded", "isTestScreenshotIncluded", "isTestVideoIncluded"],
+          required: [
+            "isTestExplanationIncluded",
+            "isTestScreenshotIncluded",
+            "isTestVideoIncluded",
+          ],
           additionalProperties: false,
         },
       },