refactor(api): extract InvestigationLease table, fix review issues

Migrates lease state from the InvestigationRun.leaseOwner/leaseExpiresAt
fields (nullable, representable-invalid) into a separate InvestigationLease
table where the row's *existence* represents "this investigation is PROCESSING
with a live lease holder". This eliminates the invalid states that the old
schema permitted (leaseOwner without leaseExpiresAt, PENDING/COMPLETE
investigations with orphan lease fields, etc.) and removes the now-redundant
InvestigationRun model entirely.

Key design changes:
- InvestigationLease row existence ↔ PROCESSING status structural invariant
- progressClaims moves to InvestigationLease (auto-cleaned on row delete)
- attemptCount, queuedAt, retryAfter promoted to Investigation directly
- Retry backoff now application-managed (not graphile-worker maxAttempts):
  TRANSIENT failures reclaim to PENDING + re-enqueue with exponential delay
  (10s × 2^(attempt-1)); NON_RETRYABLE failures mark FAILED immediately
- Per-investigation jobKey (investigate:${id}) deduplicate concurrent enqueues

Review fixes included in this commit:
- Bug: duplicate no-op assertion in interim-source-selection test replaced
  with an explicit InvestigationLease DB query (the original assertion was
  checking the same already-verified variable twice, losing coverage)
- Coverage: extract markInvestigationFailedInTx and releaseLeaseToRetryInTx
  from attempt-audit.ts so all three terminal/reclaim helpers can be unit
  tested with mock tx clients, not just persistCompletedInvestigation
- Coverage: add invariant-violation throw tests for all three tx helpers
  (lease deleted but status not PROCESSING → throw rolls back lease deletion)
- Coverage: add investigation-lease.test.ts pinning the retry backoff schedule
  (BASE_BACKOFF_MS, formula, concrete per-attempt values per spec §3.7)
- Fix: TypeScript narrowing bug in investigation-lifecycle.ts (investigation
  variable closure capture in $transaction callback)
- Clean: remove redundant seedInvestigationWithLeaseFields calls in lifecycle
  fuzz test (startedAt/heartbeatAt are not asserted and were already set by
  seedInvestigation)
- Docs: document in tryClaimLease that retryAfter is intentionally not gated
  there; investigateNow bypassing the backoff window is by design

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: hugbubby

SPEC.md

@@ -980,12 +980,16 @@ model Investigation {
   model                 InvestigationModel
   modelVersion          String?                // Provider-reported model revision/version when available
   checkedAt             DateTime?              // Null until completion
-  // Transient progress state during active PROCESSING. Contains
-  // { pending: ClaimPayload[], confirmed: ClaimPayload[] } while the
-  // orchestrator is running. Nullified on every lifecycle transition
-  // (COMPLETE, FAILED, lease release, stale-run recovery, requeue).
-  progressClaims        Json?
-  run                   InvestigationRun?
+  queuedAt              DateTime               @default(now())
+  // Monotonically increasing attempt counter. Incremented atomically when a
+  // worker claims the lease. Gives each retry a distinct attemptNumber for
+  // the InvestigationAttempt audit trail.
+  attemptCount          Int                    @default(0)
+  // INV-LEASE: The InvestigationLease row exists iff the investigation is
+  // PROCESSING and has an active lease holder. Structurally prevents
+  // leaseOwner/leaseExpiresAt without PROCESSING, and vice versa.
+  lease                 InvestigationLease?
+  openAiKeySource       InvestigationOpenAiKeySource?
   attempts              InvestigationAttempt[]
   updates               Investigation[]        @relation("InvestigationUpdateLineage")
   claims                Claim[]
@@ -1010,34 +1014,35 @@ model InvestigationInput {
   createdAt               DateTime           @default(now())
 }
 
-model InvestigationRun {
-  id              String                        @id @default(cuid())
-  investigationId String                        @unique
-  investigation   Investigation                 @relation(fields: [investigationId], references: [id], onDelete: Cascade)
-  leaseOwner      String?                       // Worker identity holding the run
-  leaseExpiresAt  DateTime?                     // When the worker's lease expires
-  recoverAfterAt  DateTime?                     // Grace period before another worker can recover
-  queuedAt        DateTime?
-  startedAt       DateTime?
-  heartbeatAt     DateTime?
-  openAiKeySource InvestigationOpenAiKeySource?
-  createdAt       DateTime                      @default(now())
-  updatedAt       DateTime                      @updatedAt
+// INV-LEASE: The existence of an InvestigationLease row means "this
+// investigation is PROCESSING and has an active lease holder". All fields
+// are NOT NULL — structurally prevents partial lease states. The row is
+// deleted on every terminal transition (COMPLETE, FAILED) and on lease
+// release (transient failure → PENDING), so progressClaims is automatically
+// cleaned up without needing sentinel values.
+model InvestigationLease {
+  investigationId String        @id
+  investigation   Investigation @relation(fields: [investigationId], references: [id], onDelete: Cascade)
+  leaseOwner      String
+  leaseExpiresAt  DateTime
+  startedAt       DateTime
+  heartbeatAt     DateTime
+  progressClaims  Json?         // { pending: ClaimPayload[], confirmed: ClaimPayload[] }
+  createdAt       DateTime      @default(now())
 
   @@index([leaseExpiresAt])
-  @@index([recoverAfterAt])
 }
 
 model InvestigationOpenAiKeySource {
-  runId      String           @id
-  run        InvestigationRun @relation(fields: [runId], references: [id], onDelete: Cascade)
-  ciphertext String                        // AES-256-GCM encrypted user API key
-  iv         String
-  authTag    String
-  keyId      String                        // Identifies the encryption key version
-  expiresAt  DateTime                      // Short-lived lease; worker must start before expiry
-  createdAt  DateTime         @default(now())
-  updatedAt  DateTime         @updatedAt
+  investigationId String        @id
+  investigation   Investigation @relation(fields: [investigationId], references: [id], onDelete: Cascade)
+  ciphertext      String                     // AES-256-GCM encrypted user API key
+  iv              String
+  authTag         String
+  keyId           String                     // Identifies the encryption key version
+  expiresAt       DateTime                   // Short-lived lease; worker must start before expiry
+  createdAt       DateTime      @default(now())
+  updatedAt       DateTime      @updatedAt
 
   @@index([expiresAt])
 }
@@ -1597,21 +1602,18 @@ WITH latest_versions AS (
 SELECT
   lv."postVersionId",
   i."id" AS "investigationId",
-  i."status" AS "investigationStatus",
-  r."id" AS "runId",
-  r."leaseOwner",
-  r."leaseExpiresAt",
-  r."recoverAfterAt"
+  i."status" AS "investigationStatus"
 FROM latest_versions lv
 JOIN "Post" p ON p."id" = lv."postId"
 JOIN "ContentBlob" cb ON cb."id" = lv."contentBlobId"
 LEFT JOIN "Investigation" i ON i."postVersionId" = lv."postVersionId"
-LEFT JOIN "InvestigationRun" r ON r."investigationId" = i."id"
+LEFT JOIN "InvestigationLease" il ON il."investigationId" = i."id"
 WHERE cb."wordCount" <= 10000
   AND (
     i."id" IS NULL                          -- no investigation yet
-    OR i."status" = 'PENDING'               -- pending (may need run)
-    OR (i."status" = 'PROCESSING' AND ...)  -- stuck processing (lease/recovery expired)
+    OR i."status" = 'PENDING'               -- pending, ready for enqueueing
+    OR (i."status" = 'PROCESSING'           -- stuck processing (lease expired or missing)
+        AND (il."investigationId" IS NULL OR il."leaseExpiresAt" <= NOW()))
   )
 ORDER BY p."uniqueViewScore" DESC
 LIMIT :budget;
@@ -1622,21 +1624,32 @@ which handles idempotent creation of the `Investigation` row and job enqueueing.
 
 ## 3.7 Job Queue
 
-Postgres-backed (graphile-worker or `FOR UPDATE SKIP LOCKED`). No Redis.
+Postgres-backed (graphile-worker). No Redis.
 Used by selector work and all `investigateNow` requests.
-User-key requests attach an encrypted short-lived lease for worker-side credential handoff.
+User-key requests attach an encrypted short-lived key source on the Investigation
+for worker-side credential handoff.
+
+Each graphile-worker job is enqueued with `maxAttempts: 1` and a per-investigation
+`jobKey` (`investigate:${investigationId}`). Retry control is managed by the
+application, not graphile-worker: transient failures reclaim the investigation to
+PENDING and explicitly re-enqueue with a backoff delay.
 
 ```
 Investigation selected (by selector or any investigateNow request)
   → Upsert investigation for postVersionId (idempotent: one investigation per content version)
   → If already exists: reuse existing investigation row and do not enqueue duplicate work
-  → Worker picks up job → UPDATE status = PROCESSING
+  → Worker picks up job → claim lease (PENDING → PROCESSING, atomically increment attemptCount)
   → Worker calls Investigator.investigate()
-  → On success: UPDATE status = COMPLETE
+  → On success: delete lease, UPDATE status = COMPLETE
   → On failure: classify and retry or fail permanently
 
+Retry model:
+  - Investigation.attemptCount tracks retries (incremented at lease claim).
+  - MAX_INVESTIGATION_ATTEMPTS = 4. When exhausted, the investigation is marked FAILED.
+  - Transient retries use exponential backoff: delay = 10s × 2^(attempt - 1).
+
 Failure classes:
-  TRANSIENT (retry up to 3x with exponential backoff):
+  TRANSIENT (reclaim to PENDING, re-enqueue with backoff, up to MAX_INVESTIGATION_ATTEMPTS):
     - Provider 5xx errors, rate limits (429), network timeouts
   NON_RETRYABLE (mark FAILED immediately):
     - Structured output fails Zod validation (likely prompt/schema issue, not transient)
@@ -1645,8 +1658,9 @@ Failure classes:
   PARTIAL (mark FAILED, log partial output for debugging):
     - Provider returns truncated or incomplete tool-call trace
 
-If a user-key lease is missing/expired when the worker starts, the run fails and
-requires an explicit user re-request.
+If a user-key source is missing/expired when the worker starts, the investigation
+fails and requires an explicit user re-request. Key sources are consumed (deleted)
+on every terminal transition (COMPLETE, FAILED).
 
 `FAILED` is terminal for a given `postVersionId` in v1. Re-running that exact content
 version requires an explicit operator/admin action (e.g., reset status or delete/recreate row),
@@ -1985,11 +1999,11 @@ openerrata/
 │       │   │   │   ├── services/
 │       │   │   │   │   ├── orchestrator.ts            # Main investigation orchestrator
 │       │   │   │   │   ├── orchestrator-errors.ts     # Error classification
-│       │   │   │   │   ├── run-lease.ts               # Atomic lease claim/release + heartbeat
+│       │   │   │   │   ├── investigation-lease.ts      # Atomic lease claim/release + heartbeat
 │       │   │   │   │   ├── prompt-context.ts          # Post metadata extraction for prompts
 │       │   │   │   │   ├── attempt-audit.ts           # Audit record persistence
 │       │   │   │   │   ├── markdown-resolution.ts     # Trust-policy-based markdown resolution
-│       │   │   │   │   ├── investigation-lifecycle.ts # Status transitions + progressClaims
+│       │   │   │   │   ├── investigation-lifecycle.ts # Status transitions + lease recovery
 │       │   │   │   │   ├── selector.ts                # Investigation selection cron
 │       │   │   │   │   ├── queue.ts                   # graphile-worker integration
 │       │   │   │   │   ├── queue-lifecycle.ts

src/typescript/api/AGENTS.md

@@ -54,9 +54,10 @@ tRPC handler → `postRouter` or `publicRouter` → Prisma → PostgreSQL.
 3. Orchestrator (`src/lib/services/orchestrator.ts`) claims a run lease, calls
    the investigator, stores claims + sources, marks COMPLETE or FAILED.
    Delegates to focused sub-modules:
-   - `services/run-lease.ts` — Atomic lease claim/release and heartbeat renewal.
-     Runs keep `PROCESSING` status throughout; lease expiry is the worker
-     exclusivity mechanism (not a status reset to `PENDING`).
+   - `services/investigation-lease.ts` — Atomic lease claim/release and heartbeat
+     renewal via the `InvestigationLease` table. The row's existence represents
+     a PROCESSING investigation with a lease holder; lease expiry is the worker
+     exclusivity mechanism.
    - `services/prompt-context.ts` — Extracts post metadata (platform, URL,
      author, image occurrences, video flag) from the Prisma query result.
    - `services/attempt-audit.ts` — Persists `InvestigationAttempt` records and
@@ -87,11 +88,13 @@ tRPC handler → `postRouter` or `publicRouter` → Prisma → PostgreSQL.
 
 ### Failure Classification (spec §3.7)
 
-- **TRANSIENT**: Provider 5xx, 429, network timeouts → graphile-worker retries
-  with backoff. Investigation stays `PROCESSING`; the run lease is released so
-  the next worker attempt can claim it. Status is **not** reset to `PENDING`
-  (doing so would allow the selector/investigateNow to re-enqueue a duplicate
-  graphile-worker job via jobKey replacement).
+- **TRANSIENT**: Provider 5xx, 429, network timeouts → delete lease, reclaim
+  investigation to `PENDING`, and explicitly re-enqueue with exponential backoff
+  (`10s × 2^(attempt-1)`). `Investigation.attemptCount` tracks retries; after
+  `MAX_INVESTIGATION_ATTEMPTS` (4) the investigation is marked FAILED.
+  The per-investigation `jobKey` (`investigate:${investigationId}`) ensures
+  concurrent enqueue calls from the re-enqueue, selector, or investigateNow
+  all resolve to exactly one graphile-worker job.
 - **NON_RETRYABLE**: Zod validation failure, content-policy refusal, auth errors
   → mark FAILED immediately, don't rethrow.
 - **FAILED is terminal** for a given `postVersionId`. No automatic

src/typescript/api/prisma/migrations/0023_extract_investigation_lease/migration.sql

@@ -0,0 +1,108 @@
+-- Extract InvestigationLease table from InvestigationRun.
+--
+-- Instead of merging lease fields into Investigation (with CHECK constraints),
+-- this migration creates a separate InvestigationLease table where all fields
+-- are NOT NULL. The row's existence represents "this investigation is PROCESSING
+-- and has a lease holder" — the representable-valid principle is enforced by
+-- the schema itself:
+--   - Can't have leaseOwner without leaseExpiresAt (both are NOT NULL)
+--   - Can't have lease fields on a non-PROCESSING investigation (no row)
+--   - progressClaims lives on the lease row — automatically cleaned up on delete
+
+-- ── Phase 1: Create InvestigationLease table ────────────────────────────────
+
+CREATE TABLE "InvestigationLease" (
+  "investigationId" TEXT NOT NULL,
+  "leaseOwner" TEXT NOT NULL,
+  "leaseExpiresAt" TIMESTAMP(3) NOT NULL,
+  "startedAt" TIMESTAMP(3) NOT NULL,
+  "heartbeatAt" TIMESTAMP(3) NOT NULL,
+  "progressClaims" JSONB,
+  "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  CONSTRAINT "InvestigationLease_pkey" PRIMARY KEY ("investigationId"),
+  CONSTRAINT "InvestigationLease_investigationId_fkey"
+    FOREIGN KEY ("investigationId") REFERENCES "Investigation"("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE INDEX "InvestigationLease_leaseExpiresAt_idx" ON "InvestigationLease"("leaseExpiresAt");
+
+-- ── Phase 2: Backfill from InvestigationRun + Investigation.progressClaims ──
+
+INSERT INTO "InvestigationLease" ("investigationId", "leaseOwner", "leaseExpiresAt", "startedAt", "heartbeatAt", "progressClaims")
+SELECT r."investigationId", r."leaseOwner", r."leaseExpiresAt", r."startedAt", r."heartbeatAt", i."progressClaims"
+FROM "InvestigationRun" r
+JOIN "Investigation" i ON i."id" = r."investigationId"
+WHERE r."leaseOwner" IS NOT NULL
+  AND r."leaseExpiresAt" IS NOT NULL
+  AND i."status" = 'PROCESSING';
+
+-- ── Phase 3: Add queuedAt to Investigation ──────────────────────────────────
+
+ALTER TABLE "Investigation" ADD COLUMN "queuedAt" TIMESTAMP(3);
+UPDATE "Investigation" i SET "queuedAt" = r."queuedAt" FROM "InvestigationRun" r WHERE r."investigationId" = i."id";
+UPDATE "Investigation" SET "queuedAt" = "createdAt" WHERE "queuedAt" IS NULL;
+ALTER TABLE "Investigation" ALTER COLUMN "queuedAt" SET NOT NULL;
+ALTER TABLE "Investigation" ALTER COLUMN "queuedAt" SET DEFAULT CURRENT_TIMESTAMP;
+
+-- ── Phase 3b: Add attemptCount to Investigation ─────────────────────────────
+
+ALTER TABLE "Investigation" ADD COLUMN "attemptCount" INTEGER NOT NULL DEFAULT 0;
+
+-- ── Phase 3c: Add retryAfter to Investigation ───────────────────────────────
+
+ALTER TABLE "Investigation" ADD COLUMN "retryAfter" TIMESTAMP(3);
+
+-- ── Phase 4: Re-point InvestigationOpenAiKeySource FK ───────────────────────
+
+ALTER TABLE "InvestigationOpenAiKeySource"
+  ADD COLUMN "investigationId" TEXT;
+
+UPDATE "InvestigationOpenAiKeySource" ks
+SET "investigationId" = r."investigationId"
+FROM "InvestigationRun" r
+WHERE r."id" = ks."runId";
+
+ALTER TABLE "InvestigationOpenAiKeySource"
+  ALTER COLUMN "investigationId" SET NOT NULL;
+
+ALTER TABLE "InvestigationOpenAiKeySource"
+  ADD CONSTRAINT "InvestigationOpenAiKeySource_investigationId_fkey"
+  FOREIGN KEY ("investigationId") REFERENCES "Investigation"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Drop old FK and PK
+ALTER TABLE "InvestigationOpenAiKeySource"
+  DROP CONSTRAINT "InvestigationOpenAiKeySource_runId_fkey";
+ALTER TABLE "InvestigationOpenAiKeySource"
+  DROP CONSTRAINT "InvestigationOpenAiKeySource_pkey";
+ALTER TABLE "InvestigationOpenAiKeySource"
+  ADD PRIMARY KEY ("investigationId");
+ALTER TABLE "InvestigationOpenAiKeySource"
+  DROP COLUMN "runId";
+
+-- ── Phase 5: Fix zombies ────────────────────────────────────────────────────
+
+-- Delete expired leases
+DELETE FROM "InvestigationLease" WHERE "leaseExpiresAt" <= NOW();
+
+-- PROCESSING without lease row → PENDING
+UPDATE "Investigation" SET "status" = 'PENDING'
+WHERE "status" = 'PROCESSING'
+  AND "id" NOT IN (SELECT "investigationId" FROM "InvestigationLease");
+
+-- ── Phase 6: Drop progressClaims from Investigation ─────────────────────────
+
+ALTER TABLE "Investigation" DROP COLUMN IF EXISTS "progressClaims";
+
+-- ── Phase 7: Drop InvestigationRun (explicit, no CASCADE) ───────────────────
+
+-- Explicitly drop all known dependencies first so unexpected ones fail loud
+ALTER TABLE "InvestigationRun"
+  DROP CONSTRAINT IF EXISTS "InvestigationRun_investigationId_fkey";
+DROP INDEX IF EXISTS "InvestigationRun_investigationId_key";
+DROP INDEX IF EXISTS "InvestigationRun_leaseExpiresAt_idx";
+DROP INDEX IF EXISTS "InvestigationRun_recoverAfterAt_idx";
+ALTER TABLE "InvestigationRun"
+  DROP CONSTRAINT IF EXISTS "InvestigationRun_lease_pair_consistency_check";
+
+-- Now drop the table — will fail if any unexpected FK or dependency remains
+DROP TABLE "InvestigationRun";