Handle rate limits and scope tool permissions in headless calls

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: ronshapiro

rsi_orchestrator/headless_claude.test.ts

@@ -0,0 +1,58 @@
+import {asCliError, HeadlessClaudeError, primaryModel} from "./headless_claude";
+
+describe("primaryModel", () => {
+  test("returns undefined when there's no modelUsage", () => {
+    expect(primaryModel(undefined)).toBeUndefined();
+  });
+
+  test("returns undefined for an empty modelUsage", () => {
+    expect(primaryModel({})).toBeUndefined();
+  });
+
+  test("returns the only model when there's just one", () => {
+    expect(primaryModel({"claude-sonnet-5": {costUSD: 0.01}})).toBe("claude-sonnet-5");
+  });
+
+  test("picks the highest-cost model when several models were used in one session", () => {
+    // A cheap sub-step on one model alongside the model that did the real generation work — the
+    // real work should win, not whichever key happened to be inserted first.
+    expect(primaryModel({
+      "claude-haiku-4-5-20251001": {costUSD: 0.001142},
+      "claude-sonnet-5": {costUSD: 0.14135159999999997},
+    })).toBe("claude-sonnet-5");
+  });
+
+  test("treats a missing costUSD as zero", () => {
+    expect(primaryModel({
+      "claude-haiku-4-5-20251001": {},
+      "claude-sonnet-5": {costUSD: 0.01},
+    })).toBe("claude-sonnet-5");
+  });
+});
+
+describe("asCliError", () => {
+  test("returns undefined when the error has no stdout", () => {
+    expect(asCliError(new Error("boom"))).toBeUndefined();
+  });
+
+  test("returns undefined when stdout isn't valid JSON", () => {
+    expect(asCliError({stdout: "not json"})).toBeUndefined();
+  });
+
+  test("recovers a HeadlessClaudeError from a rate-limited response on stdout", () => {
+    const stdout = JSON.stringify({
+      is_error: true,
+      result: "You've hit your session limit · resets 6:10pm",
+      api_error_status: 429,
+    });
+    const recovered = asCliError({stdout});
+    expect(recovered).toBeInstanceOf(HeadlessClaudeError);
+    expect(recovered!.isRateLimited).toBe(true);
+    expect(recovered!.message).toBe("You've hit your session limit · resets 6:10pm");
+  });
+
+  test("a non-429 CLI error is not treated as rate-limited", () => {
+    const stdout = JSON.stringify({is_error: true, result: "boom", api_error_status: 500});
+    expect(asCliError({stdout})!.isRateLimited).toBe(false);
+  });
+});

rsi_orchestrator/headless_claude.ts

@@ -6,25 +6,75 @@ const execFileAsync = promisify(execFile);
 export interface HeadlessClaudeOptions {
   cwd?: string;
   timeoutMs?: number;
+  // Defaults to a safe read-only set — this task family only needs to read repo files, not run
+  // arbitrary commands or write anything. Without an explicit allow-list, headless calls have no
+  // way to approve tool use, so anything beyond the default-allowed tools gets silently denied
+  // and Claude wastes turns retrying workarounds instead of just reading the file.
+  allowedTools?: string[];
 }
 
+const DEFAULT_ALLOWED_TOOLS = ["Read", "Grep", "Glob"];
+
 export interface HeadlessClaudeResult {
   text: string;
-  // The canonical model ID that actually ran (e.g. "claude-sonnet-5"), read back from
-  // `--output-format json`'s modelUsage — undefined if the CLI didn't report one. Recording this
-  // per generated artifact is what Phase 4's model-routing learning reads later.
+  // The canonical model ID that actually did the substantive work (highest-cost entry in
+  // modelUsage — a session can involve more than one model, e.g. a cheap model for a small
+  // sub-step alongside the model that did the real generation). Recording this per generated
+  // artifact is what Phase 4's model-routing learning reads later.
   model: string | undefined;
   costUsd: number | undefined;
 }
 
 export type HeadlessClaudeRunner =
   (prompt: string, options?: HeadlessClaudeOptions) => Promise<HeadlessClaudeResult>;
 
+/** Thrown when the CLI itself reports an error (as opposed to a malformed-response parse error).
+ * `isRateLimited` distinguishes a usage/session-limit hit (429) — expected under subscription
+ * billing, and the caller should stop the run rather than keep retrying every remaining
+ * candidate against the same wall — from a genuine unexpected failure. */
+export class HeadlessClaudeError extends Error {
+  public readonly isRateLimited: boolean;
+
+  constructor(
+    message: string,
+    public readonly apiErrorStatus: number | undefined,
+  ) {
+    super(message);
+    this.name = "HeadlessClaudeError";
+    this.isRateLimited = apiErrorStatus === 429;
+  }
+}
+
 interface ClaudeCliJsonOutput {
   result: string;
   is_error: boolean; // eslint-disable-line camelcase
   total_cost_usd?: number; // eslint-disable-line camelcase
-  modelUsage?: Record<string, unknown>;
+  api_error_status?: number; // eslint-disable-line camelcase
+  modelUsage?: Record<string, {costUSD?: number}>;
+}
+
+export function primaryModel(modelUsage: ClaudeCliJsonOutput["modelUsage"]): string | undefined {
+  if (!modelUsage) return undefined;
+  const entries = Object.entries(modelUsage);
+  if (entries.length === 0) return undefined;
+  return entries.reduce((a, b) => ((b[1].costUSD ?? 0) > (a[1].costUSD ?? 0) ? b : a))[0];
+}
+
+/**
+ * The CLI often still writes valid JSON to stdout even when the process exits non-zero (e.g. a
+ * rate limit) — execFile treats that as a rejected promise carrying an error whose `.stdout`
+ * holds that JSON. Recover the structured error from it rather than surfacing a raw exec
+ * failure with no usable information.
+ */
+export function asCliError(error: unknown): HeadlessClaudeError | undefined {
+  const stdout = (error as {stdout?: string} | undefined)?.stdout;
+  if (!stdout) return undefined;
+  try {
+    const parsed = JSON.parse(stdout) as ClaudeCliJsonOutput;
+    return new HeadlessClaudeError(parsed.result, parsed.api_error_status);
+  } catch {
+    return undefined;
+  }
 }
 
 /**
@@ -37,22 +87,31 @@ interface ClaudeCliJsonOutput {
  * tests instead of depending on this one directly.
  */
 export const runHeadlessClaude: HeadlessClaudeRunner = async (prompt, options = {}) => {
-  const {stdout} = await execFileAsync(
-    "claude",
-    ["-p", prompt, "--output-format", "json"],
-    {
-      cwd: options.cwd,
-      timeout: options.timeoutMs,
-      maxBuffer: 1024 * 1024 * 32,
-    },
-  );
+  let stdout: string;
+  try {
+    ({stdout} = await execFileAsync(
+      "claude",
+      [
+        "-p", prompt,
+        "--output-format", "json",
+        "--allowedTools", (options.allowedTools ?? DEFAULT_ALLOWED_TOOLS).join(","),
+      ],
+      {
+        cwd: options.cwd,
+        timeout: options.timeoutMs,
+        maxBuffer: 1024 * 1024 * 32,
+      },
+    ));
+  } catch (e) {
+    throw asCliError(e) ?? e;
+  }
   const parsed = JSON.parse(stdout) as ClaudeCliJsonOutput;
   if (parsed.is_error) {
-    throw new Error(`Headless Claude call failed: ${parsed.result}`);
+    throw new HeadlessClaudeError(parsed.result, parsed.api_error_status);
   }
   return {
     text: parsed.result,
-    model: parsed.modelUsage ? Object.keys(parsed.modelUsage)[0] : undefined,
+    model: primaryModel(parsed.modelUsage),
     costUsd: parsed.total_cost_usd,
   };
 };

rsi_orchestrator/rashi_tosafot_translation.test.ts

@@ -1,4 +1,5 @@
 import {Edit} from "../precomputed/ai_edits";
+import {HeadlessClaudeError} from "./headless_claude";
 import {
   CritiqueVerdict,
   GeneratedEdit,
@@ -124,6 +125,32 @@ describe("translateRashiTosafotComments", () => {
     expect(recordGeneration).not.toHaveBeenCalled();
   });
 
+  test("stops the whole run when generate hits a rate limit, without writing", async () => {
+    const writeEdit = jest.fn();
+    const generate = jest.fn()
+      .mockRejectedValueOnce(new HeadlessClaudeError("You've hit your session limit", 429));
+    await translateRashiTosafotComments(fakeGenerationDeps({
+      listCandidates: () => [candidate({ref: "a"}), candidate({ref: "b"})],
+      generate,
+      writeEdit,
+    }));
+    expect(generate).toHaveBeenCalledTimes(1); // never reached candidate "b"
+    expect(writeEdit).not.toHaveBeenCalled();
+  });
+
+  test("skips a candidate on a non-rate-limit error and continues to the next", async () => {
+    const written: string[] = [];
+    await translateRashiTosafotComments(fakeGenerationDeps({
+      listCandidates: () => [candidate({ref: "a"}), candidate({ref: "b"})],
+      generate: async (c) => {
+        if (c.ref === "a") throw new Error("transient CLI failure");
+        return generated();
+      },
+      writeEdit: (c) => written.push(c.ref),
+    }));
+    expect(written).toEqual(["b"]);
+  });
+
   test("processes multiple candidates independently", async () => {
     const written: string[] = [];
     await translateRashiTosafotComments(fakeGenerationDeps({

rsi_orchestrator/rashi_tosafot_translation.ts

@@ -7,7 +7,7 @@ import {Edit} from "../precomputed/ai_edits";
 import {readGenerationRecord, upsertGenerationRecord} from "../precomputed/rsi_state/generation_record";
 import {checkTextStaleness, DEFAULT_STALENESS_THRESHOLDS} from "../precomputed/rsi_state/staleness";
 import {toFlatArray} from "../sefariaTextType";
-import {runHeadlessClaude} from "./headless_claude";
+import {HeadlessClaudeError, runHeadlessClaude} from "./headless_claude";
 
 /**
  * Translates and punctuates Rashi/Tosafot comments — the first task type on the new agentic
@@ -73,6 +73,8 @@ export async function generateWithSelfCritique(
 export interface TranslationDeps {
   listCandidates: () => TranslationCandidate[];
   isFresh: (candidate: TranslationCandidate) => boolean;
+  // May reject — translateRashiTosafotComments stops the whole run on a HeadlessClaudeError with
+  // isRateLimited, and skips just this candidate on any other error.
   generate: (candidate: TranslationCandidate) => Promise<GeneratedEdit | undefined>;
   writeEdit: (candidate: TranslationCandidate, edit: Edit) => void;
   recordGeneration: (candidate: TranslationCandidate, generated: GeneratedEdit) => void;
@@ -81,9 +83,22 @@ export interface TranslationDeps {
 export async function translateRashiTosafotComments(deps: TranslationDeps): Promise<void> {
   for (const candidate of deps.listCandidates()) {
     if (deps.isFresh(candidate)) continue;
-    // Deliberately sequential — see the same rationale in triage_suggestions.ts.
-    // eslint-disable-next-line no-await-in-loop
-    const generated = await deps.generate(candidate);
+    let generated;
+    try {
+      // Deliberately sequential — see the same rationale in triage_suggestions.ts.
+      // eslint-disable-next-line no-await-in-loop
+      generated = await deps.generate(candidate);
+    } catch (e) {
+      if (e instanceof HeadlessClaudeError && e.isRateLimited) {
+        // Every remaining candidate would fail against the same wall — stop the run rather than
+        // burn through it logging the identical failure. Subscription usage limits are the
+        // expected budget signal under self-hosted billing; see RecursiveSelfImprovingAgentPlan.md.
+        console.error(`Stopping: hit the usage limit (${e.message})`);
+        return;
+      }
+      console.error(`Skipping ${candidate.ref}: ${e}`);
+      continue;
+    }
     if (!generated) continue;
     deps.writeEdit(candidate, generated.edit);
     deps.recordGeneration(candidate, generated);