Promote staging to main

.github/PULL_REQUEST_TEMPLATE/promotion.md

@@ -4,6 +4,13 @@
 > **THE BUTTON CREATES A NEW COMMIT AND BREAKS THE RELEASE MODEL.**
 > **COMMENT `/promote` TO COMPLETE THE PROMOTION.**
 
+<!-- promotion-target: REPLACE_WITH_STAGING_SHA -->
+<!-- Replace the value above with the staging commit this release was
+     validated against (git rev-parse origin/staging). Bare /promote promotes
+     exactly that sha — commits that land on staging afterwards do not ride
+     along — and /promote <sha> overrides it. An unfilled placeholder fails
+     the promotion checks; deleting the whole line promotes the staging tip. -->
+
 ## Summary
 
 <!-- One sentence: what this release ships. -->
@@ -14,4 +21,4 @@
 
 ## Validation
 
-<!-- Staging Deploy is green on the tip. Note anything soak-tested on staging.oddish.app. -->
+<!-- Staging Deploy is green on the pinned target. Note anything soak-tested on staging.oddish.app. -->

.github/scripts/promote/verify_promotion_target.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Shared promotion preconditions for Promotion Preflight and /promote.
+#
+# Resolves the target — TARGET_SHA if set, else the sha pinned in PR_BODY's
+# `promotion-target` marker, else the staging tip — and verifies the
+# fast-forward invariants and the staging deploy, then emits `sha=<target>`
+# to GITHUB_OUTPUT. Read-only: the caller decides whether to push.
+set -euo pipefail
+
+git fetch origin main staging
+
+raw="${TARGET_SHA:-}"
+if [ -z "$raw" ]; then
+  body="$(printf '%s' "${PR_BODY:-}" | tr -d '\r')"
+  if printf '%s' "$body" | grep -q '<!--[[:space:]]*promotion-target:'; then
+    # A pin that is present but not a sha must fail, never fall through to
+    # the tip — an unfilled template placeholder is not consent to ship more.
+    raw="$(printf '%s' "$body" \
+        | grep -m1 -oE '<!--[[:space:]]*promotion-target:[[:space:]]*[0-9a-fA-F]{7,40}[[:space:]]*-->' \
+        | grep -oE '[0-9a-fA-F]{7,40}' | head -n1)" \
+      || { echo "::error::the promotion-target pin in the PR body is not a commit sha; fix the pin or use an explicit target sha"; exit 1; }
+    echo "promoting the sha pinned in the PR body: $raw"
+  else
+    echo "::notice::no promotion-target pin in the PR body — promoting the staging tip"
+  fi
+fi
+target="${raw:-$(git rev-parse origin/staging)}"
+target="$(git rev-parse --verify --quiet "${target}^{commit}")" \
+  || { echo "::error::'$raw' does not resolve to a commit"; exit 1; }
+
+git merge-base --is-ancestor "$target" origin/staging \
+  || { echo "::error::$target is not on staging"; exit 1; }
+git merge-base --is-ancestor origin/main "$target" \
+  || { echo "::error::main is not an ancestor of $target — fast-forward impossible; run Sync Preflight for the repair steps"; exit 1; }
+
+if gh workflow view "Staging Deploy" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+  gh run list --repo "$GITHUB_REPOSITORY" --workflow "Staging Deploy" \
+      --branch staging --commit "$target" --json conclusion -q '.[0].conclusion' \
+    | grep -qx success \
+    || {
+         echo "::error::Staging Deploy is not green on $target"
+         echo "::notice::A queued deploy is superseded when a newer commit lands (GitHub keeps one pending run per concurrency group), so a commit that staging moved past may never have deployed. A dispatched deploy always runs the tip of staging, not an older commit, so promote the staging tip instead of this sha."
+         exit 1
+       }
+else
+  echo "::warning::Staging Deploy workflow not found — skipping deploy-green precondition (bootstrap)"
+fi
+
+echo "sha=$target" >> "$GITHUB_OUTPUT"
+echo "checks passed for $target"

.github/workflows/promote-comment.yml

@@ -1,5 +1,8 @@
 name: Promote on comment
-# Comment `/promote` on the staging -> main pull request.
+# Comment `/promote` on the staging -> main pull request to promote the sha
+# pinned in its body's `promotion-target` marker (the staging tip when no pin
+# exists), or `/promote <sha>` to override the pin
+# (the same target rule as the Promotion Preflight `target_sha` input).
 #
 # For an organization member with write access, the job always runs the
 # promotion checks and reports them back on the pull request; anyone else gets
@@ -76,6 +79,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
           PR: ${{ github.event.issue.number }}
+          COMMENT: ${{ github.event.comment.body }}
         run: |
           set -euo pipefail
           read -r base head state < <(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" \
@@ -86,18 +90,13 @@ jobs:
           cross=$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json isCrossRepository -q .isCrossRepository)
           [ "$cross" = "false" ] || { echo "::error::#$PR comes from a fork; promotion uses the repository's own staging branch"; exit 1; }
 
-          git fetch origin main staging
-          target=$(git rev-parse origin/staging)
-          git merge-base --is-ancestor origin/main "$target" \
-            || { echo "::error::main is not an ancestor of $target — run Sync Preflight for the repair steps"; exit 1; }
-
-          gh run list --repo "$GITHUB_REPOSITORY" --workflow "Staging Deploy" \
-              --branch staging --commit "$target" --json conclusion -q '.[0].conclusion' \
-            | grep -qx success \
-            || { echo "::error::Staging Deploy is not green on $target"; exit 1; }
-
-          echo "sha=$target" >> "$GITHUB_OUTPUT"
-          echo "checks passed for $target"
+          # The word after `/promote` on the command line is the target;
+          # bare `/promote` promotes the sha pinned in the pull request body
+          # (the script falls back to the staging tip when there is no pin).
+          TARGET_SHA=$(printf '%s' "$COMMENT" | head -n1 | tr -d '\r' | awk '{print $2}')
+          PR_BODY=$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json body -q .body)
+          export TARGET_SHA PR_BODY
+          .github/scripts/promote/verify_promotion_target.sh
 
       - name: Fast-forward main
         id: push
@@ -136,7 +135,12 @@ jobs:
           if [ "$CHECKS" != "success" ]; then
             body="Promotion checks failed. See the run for the reason: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
           elif [ "$PUSH_OUTCOME" = "success" ]; then
-            body="Promoted. main now points at \`${SHA}\`, the same commit as staging."
+            left=$(git rev-list --count "${SHA}..origin/staging")
+            if [ "$left" -eq 0 ]; then
+              body="Promoted. main now points at \`${SHA}\`, the same commit as staging."
+            else
+              body="Promoted. main now points at \`${SHA}\`; staging still carries ${left} unpromoted commit(s)."
+            fi
           elif [ "$PUSH_OUTCOME" = "failure" ]; then
             body="Promotion checks passed but the push failed. See the run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
           else

.github/workflows/promote.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       target_sha:
-        description: "Staging commit to promote (default: staging tip)"
+        description: "Staging commit to promote (default: the promotion PR's pinned target, else the staging tip)"
         required: false
 
 jobs:
@@ -24,32 +24,25 @@ jobs:
           fetch-depth: 0
 
       - name: Verify promotion preconditions
+        id: checks
         shell: bash
+        env:
+          TARGET_SHA: ${{ inputs.target_sha }}
         run: |
           set -euo pipefail
-          git fetch origin main staging
-          target="${{ inputs.target_sha }}"
-          target="${target:-$(git rev-parse origin/staging)}"
-          target="$(git rev-parse --verify "${target}^{commit}")" \
-            || { echo "::error::target_sha does not resolve to a commit"; exit 1; }
-          git merge-base --is-ancestor "$target" origin/staging \
-            || { echo "::error::$target is not on staging"; exit 1; }
-          git merge-base --is-ancestor origin/main "$target" \
-            || { echo "::error::main is not an ancestor of $target — fast-forward impossible; see the recovery runbook"; exit 1; }
           pr=$(gh pr list --repo "$GITHUB_REPOSITORY" --base main --head staging --state open --json number -q '.[0].number')
           [ -n "$pr" ] || { echo "::error::no open staging->main promotion PR"; exit 1; }
-          if gh workflow view "Staging Deploy" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
-            gh run list --repo "$GITHUB_REPOSITORY" --workflow "Staging Deploy" \
-                --branch staging --commit "$target" --json conclusion -q '.[0].conclusion' \
-              | grep -qx success \
-              || {
-                   echo "::error::Staging Deploy is not green on $target"
-                   echo "::notice::A queued deploy is superseded when a newer commit lands (GitHub keeps one pending run per concurrency group), so a commit that staging moved past may never have deployed. A dispatched deploy always runs the tip of staging, not an older commit, so promote the staging tip instead of this sha."
-                   exit 1
-                 }
-          else
-            echo "::warning::Staging Deploy workflow not found — skipping deploy-green precondition (bootstrap)"
-          fi
+          PR_BODY=$(gh pr view "$pr" --repo "$GITHUB_REPOSITORY" --json body -q .body)
+          export PR_BODY
+          .github/scripts/promote/verify_promotion_target.sh
+
+      - name: Print the push command
+        shell: bash
+        env:
+          TARGET: ${{ steps.checks.outputs.sha }}
+        run: |
+          set -euo pipefail
+          target="$TARGET"
           {
             echo "## Promotion preflight PASSED"
             echo

CHANGELOG.md

@@ -8,6 +8,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [2026-08-07]
 
+### Changed
+
+- The verdict now says `accept` or `reject` instead of `is_good: true/false`. Stored payloads keep `is_good` too, so old rows, the dashboard queries, and the Slack alert still work. The badge shows "Accepted" or "Rejected".
+- The verdict judge used to bury its hard rules inside exceptions, and it accepted a task whose own audit had found a `must_fix` leak — on tests the untouched base model already passed (0.96 against a 0.25 threshold). The prompt (`verdict_prompt.txt`) is rewritten as two steps: first look for evidence that rejects the task by itself (a leak, weak tests, a failed baseline), and only then weigh the trials' opinions, which need agreement.
+- The task overview panel used to list only the current experiment's trials, but the verdict is computed over every trial of the task — so the panel could show a verdict whose deciding trial it refused to list. It now shows every trial of the version. Trials from other experiments carry a dashed "elsewhere" chip and open in a new tab. Long subtypes also stopped pushing the "View trial" button out of its row.
+- The verdict badge used to hide its rerun button once a verdict existed, and the button that did exist re-classified every trial from scratch. Tasks with a verdict now show "Rerun verdict" (`qa/backfill` with `force: false`), which keeps the stored trial analyses and redoes only the verdict. The full re-classify stays on `qa/retry`.
+- Submitting new trials used to delete the task's verdict immediately, and the task had no verdict until QA finished the new trials. The old verdict now stays until the new QA run replaces it.
+
 ### Removed
 
 - The cc_chat dashboard chat feature is gone end to end: the `/chat-sessions`

CLAUDE.md

@@ -26,7 +26,11 @@ deploy, and the fast-forward condition, then prints the push command) and
 executes that push themselves; never merge, squash, or push to `main` directly.
 An organization member with `write`, `maintain`, or `admin` access can instead
 comment `/promote` on the promotion pull request; the workflow runs the same
-checks and, when the promote token is set, does the push.
+checks and, when the promote token is set, does the push. Bare `/promote`
+promotes the sha pinned in the pull request body (the template's
+`promotion-target` marker), so commits that reach `staging` after the
+promotion pull request was written do not ride along; `/promote <sha>`
+overrides the pin, and a body without one promotes the staging tip.
 
 **Never complete a promotion pull request with the merge button.** The button
 squashes, which puts a new commit on `main` and breaks the fast-forward
@@ -71,7 +75,9 @@ copy gets a different commit id, so the branches stay diverged.
 Not every change has to be releasable to merge. Land unfinished work behind a
 flag that is off by default (as `ODDISH_GKE_ENABLED` and
 `ODDISH_PRE_TRIAL_ENABLED` do), or promote only part of `staging` by giving
-the promotion workflow the commit to stop at.
+the promotion workflow the commit to stop at (the `target_sha` input on
+Promotion Preflight, `/promote <sha>` on the promotion pull request, or the
+`promotion-target` pin in its body).
 
 ## Useful pointers
 

backend/README.md

@@ -451,6 +451,7 @@ or globally via the env default. Under `enforce`, an over-cap submission gets
 HTTP **402** (`"Your organization is over its monthly budget …"`); under
 `shadow` it emits `metric=quota.would_block reason=org_over_budget`. Admins see
 month-to-date org usage on `GET /quotas`; any member can read the org budget
-snapshot + adaptive daily goal on `GET /quotas/org`. Advisory-lock order is
-org → payer → row locks (ENFORCE-only on admission; the org lock is always
-taken first, even when no org cap is configured).
+snapshot + adaptive daily goal on `GET /quotas/org`. Admission takes no
+locks; concurrent submissions can briefly overshoot a cap and the
+enforcement sweep cancels the overage. Only the sweep takes the quota
+advisory locks (org → payer, non-blocking).

backend/api/routers/tasks.py

@@ -388,9 +388,7 @@ async def create_task_sweep(
                 request_hash=request_hash,
             )
         except TimeoutError as exc:
-            # Quota advisory-lock waits (and other DB wait timeouts) surface as
-            # bare TimeoutError from asyncpg. Map to 503 so the CLI retries with
-            # a legible message instead of an opaque "Internal Server Error".
+            # asyncpg raises bare TimeoutError on DB wait timeouts.
             logger.error(
                 "create_task_sweep timed out for task_id=%s org_id=%s",
                 submission.task_id,

frontend/e2e/experiment-network-shape.spec.ts

@@ -1,6 +1,13 @@
-import { expect, test, type Page } from "@playwright/test";
+import { expect, test } from "@playwright/test";
 import { clerk, setupClerkTestingToken } from "@clerk/testing/playwright";
 
+import {
+  countSince,
+  holdCountedResponses,
+  recordRequests,
+  STRICT_MODE_ABORT_MS,
+} from "./network-log";
+
 /**
  * Each test here checks the number and kind of network requests the
  * experiment page makes. Each assertion encodes a bug that was measured
@@ -22,6 +29,11 @@ import { clerk, setupClerkTestingToken } from "@clerk/testing/playwright";
  * credentials, and it skips when they are missing. The experiment needs
  * at least one non-probe trial; set E2E_EXPERIMENT_ID to choose one,
  * otherwise the first experiment on the dashboard is used.
+ *
+ * Counting is race-hardened by the rules in network-log.ts: only finished
+ * requests issued after recording started count, requests are attributed
+ * to their issue time, and every counted endpoint's response is held
+ * briefly so StrictMode's aborted duplicate can never finish first.
  */
 
 const CLERK_EMAIL = process.env.E2E_CLERK_EMAIL;
@@ -45,30 +57,6 @@ const TASK_FILES_STREAM_RE = /\/api\/tasks\/[^/]+\/files\?[^#]*\bstream=1\b/;
 const ANY_FILES_STREAM_RE = /\/files\?[^#]*\bstream=1\b/;
 const TRIAL_FILES_STREAM_RE = /\/api\/trials\/[^/]+\/files\?[^#]*\bstream=1\b/;
 
-type LoggedRequest = { url: string; method: string };
-
-function recordRequests(page: Page): LoggedRequest[] {
-  const log: LoggedRequest[] = [];
-  // Only finished requests are counted. In development, React StrictMode
-  // runs every effect twice and the first run's fetch gets aborted. Those
-  // aborted requests are not real duplicates, so they must not count.
-  page.on("requestfinished", (request) =>
-    log.push({ url: request.url(), method: request.method() }),
-  );
-  return log;
-}
-
-function countSince(
-  log: LoggedRequest[],
-  from: number,
-  re: RegExp,
-  method = "GET",
-): number {
-  return log
-    .slice(from)
-    .filter((r) => r.method === method && re.test(r.url)).length;
-}
-
 test.describe("experiment page network shape", () => {
   test.skip(
     !hasClerkEnv,
@@ -84,6 +72,14 @@ test.describe("experiment page network shape", () => {
     await page.goto("/");
     await clerk.signIn({ page, emailAddress: CLERK_EMAIL! });
 
+    // The trial-detail endpoint is not in this list: its analysis-status
+    // override route below fulfills directly (no fallback), so it applies
+    // the hold itself.
+    await holdCountedResponses(page, [
+      TASK_SHELLS_RE,
+      TASK_FILES_RE,
+      TRIAL_FILES_STREAM_RE,
+    ]);
     const log = recordRequests(page);
 
     // This rewrites the fetched trial's analysis_status to "running" so
@@ -95,6 +91,9 @@ test.describe("experiment page network shape", () => {
     await page.route("**/api/trials/*", async (route) => {
       if (route.request().method() !== "GET" || analysisStatusOverride === null)
         return route.fallback();
+      // Same hold as holdCountedResponses, applied here because this
+      // handler fulfills instead of falling back.
+      await new Promise((resolve) => setTimeout(resolve, STRICT_MODE_ABORT_MS));
       const response = await route.fetch();
       const body = (await response.json()) as Record<string, unknown>;
       await route.fulfill({
@@ -154,7 +153,7 @@ test.describe("experiment page network shape", () => {
     // Phase 2 — open a trial. Exactly one detail fetch happens, shared by
     // the drawer and the analysis card. The visible task pane fetches its
     // plain tree listing, and nothing downloads file contents.
-    const openMark = log.length;
+    const openMark = Date.now();
     await trialCell.click();
     await expect(page.getByRole("tab", { name: "Summary" })).toBeVisible({
       timeout: 15_000,
@@ -178,7 +177,7 @@ test.describe("experiment page network shape", () => {
 
     // Phase 3 — the analysis reads as active, so the trial must be
     // refetched on an interval.
-    const pollMark = log.length;
+    const pollMark = Date.now();
     await expect
       .poll(() => countSince(log, pollMark, TRIAL_DETAIL_RE), {
         timeout: 8_000,
@@ -189,21 +188,21 @@ test.describe("experiment page network shape", () => {
     // it, and after that no detail request may appear for a full refetch
     // interval.
     analysisStatusOverride = "success";
-    const terminalMark = log.length;
+    const terminalMark = Date.now();
     await expect
       .poll(() => countSince(log, terminalMark, TRIAL_DETAIL_RE), {
         timeout: 8_000,
       })
       .toBeGreaterThanOrEqual(1);
-    const quietMark = log.length;
+    const quietMark = Date.now();
     await page.waitForTimeout(6_500);
     expect(countSince(log, quietMark, TRIAL_DETAIL_RE)).toBe(0);
 
     // Phase 5 — the file-view listing fires only once the Files tab is
     // actually shown. The panel sends this listing with stream=1, and the
     // trial-files endpoint ignores that parameter and answers with a
     // plain listing.
-    const filesMark = log.length;
+    const filesMark = Date.now();
     await page.getByRole("tab", { name: "Files" }).click();
     await expect
       .poll(() => countSince(log, filesMark, TRIAL_FILES_STREAM_RE), {