Render RewardKit reward design and outputs

.github/scripts/preview/compute_deployment_plan.sh

@@ -71,6 +71,17 @@ if [ "$run_migrations" = "true" ]; then
   deploy_backend=true
 fi
 
+# The same password rotation happens on every prepare-preview-database run,
+# and that job also runs on frontend-only pushes whenever a preview backend
+# is live (BACKEND_BASE set) to keep the branch smoke-tested. A live backend
+# left holding the pre-rotation password 500s every DB-backed request until
+# something redeploys it -- so a run that will prepare the branch against an
+# existing backend must redeploy that backend too, mirroring the migrations
+# rule above.
+if [ -n "$BACKEND_BASE" ] && [ "$deploy_backend" != "true" ]; then
+  deploy_backend=true
+fi
+
 any_change=false
 if [ "$deploy_backend" = "true" ] ||
   [ "$run_migrations" = "true" ] ||

AGENTS.md

@@ -1053,6 +1053,31 @@ counts show no row. Persisted `_verifier` CTRF counts are the sole source.
 Historical trials without that summary show no count; opening a trial must not
 list or read its artifacts to reconstruct one.
 
+RewardKit output follows the same pattern: the drawer renders the reward
+breakdown from the embedded `_rewards` / `_reward_details` summaries in
+`trial.result` (written by `harbor_artifacts.py`), falling back to the scoped
+trial files API for `verifier/reward.json` / `verifier/reward-details.json`
+when the embedded copies are missing (imports) or truncated. That fallback is
+a tab resource: opening a trial on Summary must not list artifacts, and the
+files request starts only after the Rewards tab is selected or directly
+addressed with `?tab=rewards`. Imported rows expose that lazy discovery tab
+even when they predate embedded summaries; an empty lookup hides it again.
+The task page's Reward design section
+(`reward-design-card.tsx`) reconstructs the reward program client-side from
+the task's `tests/` TOMLs plus a best-effort static scan of Python criteria,
+enriched with criteria observed in a completed trial's breakdown — it executes
+nothing.
+
+Every reward surface is addressable, extending the existing URL contract:
+the trial drawer's Rewards tab is `?tab=rewards`, and while that tab is
+active `?file=` addresses a criterion as `dimension/criterion` (the same
+per-tab `?file=` scoping the files/artifacts tabs use). The task pane's
+reward-design view is `?taskView=reward` beside `?taskFile=`/`?taskLines=`
+(task page and experiment page alike), and the task page's inline design
+section anchors as `#reward-design` / `#reward-design-<dimension>`. When
+adding a new reward surface, give it an address in one of these channels —
+component-local view state alone is not acceptable.
+
 On an experiment page, removing a task always calls the scoped
 `DELETE /experiments/{experiment_id}/tasks/{task_id}` proxy. It unlinks that
 experiment membership and its scoped trials without deleting the task, even

DOCS.md

@@ -705,3 +705,42 @@ dashboard shows those counts as a small passed/total line in the trial
 drawer's summary. Missing, malformed, or oversized CTRF reports are ignored
 and never change the settled `reward`; verifiers without a test report simply
 show no test line.
+
+## Reward Kit rewards (reward.json + reward-details.json)
+
+Tasks whose verifier uses [Harbor's RewardKit](https://www.harborframework.com/docs/rewardkit)
+(`harbor-rewardkit`) report **multiple named rewards** — one score per
+dimension (`tests/<dimension>/`) plus the aggregates defined in
+`tests/reward.toml` — in `/logs/verifier/reward.json`, and a per-criterion
+breakdown (scores, weights, judge reasoning, errors) in
+`/logs/verifier/reward-details.json`.
+
+Oddish captures both:
+
+- The trial's headline scalar (`trial.reward`) stays exactly as before: the
+  `"reward"` key of `reward.json`, or the sole value when only one is
+  reported. **Name your strict aggregate `reward`** in `reward.toml` so
+  pass@k, baseline gates, and the matrix keep working — other names are
+  treated as extra dimensions.
+- The full named-rewards map is persisted under the reserved
+  `trial.result._rewards` key.
+- A bounded summary of the breakdown (long judge reasoning truncated,
+  criterion lists capped) is persisted under `trial.result._reward_details`,
+  with the artifact path of the full document. The complete
+  `reward-details.json` stays with the trial artifacts.
+
+The dashboard renders these in the trial drawer as a reward breakdown —
+each dimension with its criteria, the judge's raw answer and reasoning, and
+the files the judge read — and renders `reward.json` /
+`reward-details.json` as reward trees in the Files and Artifacts tabs. On
+the task page, RewardKit tasks additionally get a **Reward design** section
+reconstructed from `tests/reward.toml`, the judge TOMLs, and a static scan
+of Python criteria files, including a what-if mode that simulates how
+criterion outcomes flow through each aggregation into the final rewards.
+
+Both keys follow the metrics.json rules: `_rewards` and `_reward_details`
+are Oddish-owned (task-authored keys of the same name are dropped), and a
+missing, malformed, or oversized details file is ignored — it never changes
+the settled `reward`. Imported trials rely on the artifact fallback: the
+drawer lazy-loads `verifier/reward.json` and `verifier/reward-details.json`
+through the scoped trial files API when the embedded copies are absent.

frontend/package.json

@@ -56,6 +56,7 @@
     "remark-breaks": "4.0.0",
     "remark-gfm": "4.0.1",
     "shiki": "4.0.2",
+    "smol-toml": "1.4.2",
     "swr": "2.4.1",
     "tailwind-merge": "3.5.0",
     "tailwindcss-animate": "1.0.7",

frontend/src/app/(app)/tasks/[task_id]/task-detail-client.tsx

@@ -72,6 +72,7 @@ import {
   Loader2,
   Star,
 } from "lucide-react";
+import { RewardDesignCard } from "@/components/reward-design-card";
 
 const TaskFilesPanel = dynamic(
   () =>
@@ -901,18 +902,28 @@ export function TaskDetailClient({
   // --- Drawer addressability ------------------------------------------
   // The drawer state lives in the URL so any view on this page can be
   // linked: ?trial=<id> opens that trial, ?drawer=task opens the task
-  // files drawer, and ?taskFile= / ?taskLines= address the task pane's
-  // file and line range (the trial pane's ?file= / ?lines= are handled
-  // inside TrialDetailPanel).
+  // files drawer, ?taskFile= / ?taskLines= address the task pane's file
+  // and line range, and ?taskView=reward addresses its reward-design view
+  // (the trial pane's ?file= / ?lines= are handled inside
+  // TrialDetailPanel).
   const [taskPaneFile, setTaskPaneFile] = useState<string | null>(null);
   const [taskPaneLines, setTaskPaneLines] = useState<LineRange | null>(null);
+  const [taskPaneView, setTaskPaneView] = useState<"reward" | null>(null);
   const taskPaneFileRef = useRef<string | null>(null);
   const handleTaskPaneFileChange = useCallback((path: string | null) => {
     // A different file makes the old line anchor meaningless — drop it.
     if (!sameFilePath(taskPaneFileRef.current, path)) setTaskPaneLines(null);
     taskPaneFileRef.current = path;
     setTaskPaneFile(path);
   }, []);
+  // Only the reward view is addressed; the overview is the unmarked
+  // default, so it maps to null and keeps URLs clean.
+  const handleTaskPaneViewChange = useCallback(
+    (view: "overview" | "reward" | null) => {
+      setTaskPaneView(view === "reward" ? "reward" : null);
+    },
+    []
+  );
 
   // Hydrate the drawer from the URL once the version's trials are known.
   const drawerHydratedRef = useRef(false);
@@ -948,6 +959,8 @@ export function TaskDetailClient({
       setTaskPaneFile(urlTaskFile);
       if (urlTaskLines) setTaskPaneLines(urlTaskLines);
     }
+    const urlTaskView = params.get("taskView");
+    if (urlTaskView === "reward") setTaskPaneView("reward");
 
     if (urlTrialId) {
       const trial = orderedTrials.find((t) => t.id === urlTrialId);
@@ -958,7 +971,10 @@ export function TaskDetailClient({
       }
       unresolvedTrialParamRef.current = true;
     }
-    if (params.get("drawer") === "task" || (!urlTrialId && urlTaskFile)) {
+    if (
+      params.get("drawer") === "task" ||
+      (!urlTrialId && (urlTaskFile || urlTaskView === "reward"))
+    ) {
       hydrationOpeningRef.current = true;
       handleOpenTaskFiles();
     }
@@ -1006,6 +1022,7 @@ export function TaskDetailClient({
       taskPaneFileRef.current = null;
       setTaskPaneFile(null);
       setTaskPaneLines(null);
+      setTaskPaneView(null);
     }
   }, [drawer]);
 
@@ -1063,6 +1080,7 @@ export function TaskDetailClient({
         next.delete("lines");
         next.delete("taskFile");
         next.delete("taskLines");
+        next.delete("taskView");
       }
     }
     if (drawer) {
@@ -1076,13 +1094,18 @@ export function TaskDetailClient({
       } else {
         next.delete("taskLines");
       }
+      if (taskPaneView === "reward") {
+        next.set("taskView", "reward");
+      } else {
+        next.delete("taskView");
+      }
     }
 
     if (next.toString() !== current.toString()) {
       const url = `${window.location.pathname}${next.toString() ? `?${next.toString()}` : ""}`;
       window.history.replaceState(window.history.state, "", url);
     }
-  }, [drawer, taskPaneFile, taskPaneLines]);
+  }, [drawer, taskPaneFile, taskPaneLines, taskPaneView]);
 
   const handleRerun = useCallback(() => {
     void mutate();
@@ -1375,6 +1398,15 @@ export function TaskDetailClient({
           />
         ) : null}
 
+        {/* RewardKit tasks get their reward program rendered as an
+            explorable design; other tasks render nothing here. */}
+        <RewardDesignCard
+          taskId={task.id}
+          taskVersion={selectedVersion?.version}
+          trials={trialsForVersion}
+          anchorId="reward-design"
+        />
+
         <div className="space-y-3">
           <div className="flex items-baseline justify-between">
             <h2 className="font-mono text-[12px] font-semibold tracking-[0.06em] text-[color:var(--paper-ink-2)] uppercase">
@@ -1434,6 +1466,8 @@ export function TaskDetailClient({
                 selectedLines={taskPaneLines}
                 onSelectLinesChange={setTaskPaneLines}
                 onSelectedFileChange={handleTaskPaneFileChange}
+                initialView={taskPaneView}
+                onViewChange={handleTaskPaneViewChange}
                 apiBaseUrl="/api"
                 contentOnly={true}
               />
@@ -1452,6 +1486,8 @@ export function TaskDetailClient({
                 selectedLines={taskPaneLines}
                 onSelectLinesChange={setTaskPaneLines}
                 onSelectedFileChange={handleTaskPaneFileChange}
+                initialView={taskPaneView}
+                onViewChange={handleTaskPaneViewChange}
                 onRetryComplete={handleRerun}
                 allowRetry={true}
                 onNavigateToFirstTrial={

frontend/src/components/experiment-detail-view.tsx

@@ -820,7 +820,8 @@ function ExperimentSummaryBar({
                   </span>
                 )}
               </>
-            ) : summary.ownedTokenTrialCount === 0 && summary.costTrialCount > 0 ? (
+            ) : summary.ownedTokenTrialCount === 0 &&
+              summary.costTrialCount > 0 ? (
               // Priced work exists and this experiment's own trials reported
               // nothing at all: an explicit zero ("nothing new was spent")
               // reads honestly where a dash would read as "unknown". With
@@ -956,6 +957,17 @@ export function ExperimentDetailView({
   const [taskPaneLines, setTaskPaneLines] = useState<LineRange | null>(() =>
     parseLineRange(searchParams.get("taskLines"))
   );
+  // ?taskView=reward addresses the task pane's reward-design view; the
+  // overview is the unmarked default.
+  const [taskPaneView, setTaskPaneView] = useState<"reward" | null>(() =>
+    searchParams.get("taskView") === "reward" ? "reward" : null
+  );
+  const handleTaskPaneViewChange = useCallback(
+    (view: "overview" | "reward" | null) => {
+      setTaskPaneView(view === "reward" ? "reward" : null);
+    },
+    []
+  );
   // Mirrors taskPaneFile so the change handler can compare without an
   // impure setState updater.
   const taskPaneFileRef = useRef<string | null>(taskPaneFile);
@@ -976,6 +988,7 @@ export function ExperimentDetailView({
       taskId !== lastDrawerTaskIdRef.current
     ) {
       handleTaskPaneFileChange(null);
+      setTaskPaneView(null);
     }
     lastDrawerTaskIdRef.current = taskId;
   }, [drawerState?.task.id, handleTaskPaneFileChange]);
@@ -1176,6 +1189,11 @@ export function ExperimentDetailView({
       } else {
         next.delete("taskLines");
       }
+      if (taskPaneView === "reward") {
+        next.set("taskView", "reward");
+      } else {
+        next.delete("taskView");
+      }
     } else if (pendingUrlTrialId == null) {
       // Same pending guard as above: a trial-only deep link keeps the drawer
       // closed until the trial resolves, and stripping the params here would
@@ -1187,14 +1205,21 @@ export function ExperimentDetailView({
       next.delete("lines");
       next.delete("taskFile");
       next.delete("taskLines");
+      next.delete("taskView");
     }
 
     if (next.toString() !== current.toString()) {
       const url = `${window.location.pathname}${next.toString() ? `?${next.toString()}` : ""}`;
       // Keep URL query in sync without triggering app-router navigation work.
       window.history.replaceState(window.history.state, "", url);
     }
-  }, [drawerState, pendingUrlTrialId, taskPaneFile, taskPaneLines]);
+  }, [
+    drawerState,
+    pendingUrlTrialId,
+    taskPaneFile,
+    taskPaneLines,
+    taskPaneView,
+  ]);
 
   useEffect(() => {
     if (hydratedFromUrl.current || tasksForExperiment.length === 0) return;
@@ -1360,9 +1385,7 @@ export function ExperimentDetailView({
   useEffect(() => {
     if (pendingUrlTrialId == null) return;
     for (const host of tasksForExperiment) {
-      const trial = (host.trials ?? []).find(
-        (t) => t.id === pendingUrlTrialId
-      );
+      const trial = (host.trials ?? []).find((t) => t.id === pendingUrlTrialId);
       if (trial) {
         openDeepLinkTrial(host, trial);
         return;
@@ -1493,7 +1516,8 @@ export function ExperimentDetailView({
       // the fields; the client fold's partial owned sum beats a hard $0.00.
       ownedCostUsd: costTotals.owned_cost_usd ?? base.ownedCostUsd,
       ownedTrialCount: costTotals.owned_trial_count ?? base.ownedTrialCount,
-      ownedHasEstimated: costTotals.owned_has_estimated ?? base.ownedHasEstimated,
+      ownedHasEstimated:
+        costTotals.owned_has_estimated ?? base.ownedHasEstimated,
       ownedHasNative: costTotals.owned_has_native ?? base.ownedHasNative,
       ownedTokenCount: costTotals.owned_token_count ?? base.ownedTokenCount,
       ownedTokenTrialCount:
@@ -1746,6 +1770,8 @@ export function ExperimentDetailView({
               selectedLines={taskPaneLines}
               onSelectLinesChange={setTaskPaneLines}
               onSelectedFileChange={handleTaskPaneFileChange}
+              initialView={taskPaneView}
+              onViewChange={handleTaskPaneViewChange}
               apiBaseUrl={apiBaseUrl}
               cancelExperimentId={experimentId}
               showAnalysis={showAnalysis}
@@ -1789,6 +1815,8 @@ export function ExperimentDetailView({
               selectedLines={taskPaneLines}
               onSelectLinesChange={setTaskPaneLines}
               onSelectedFileChange={handleTaskPaneFileChange}
+              initialView={taskPaneView}
+              onViewChange={handleTaskPaneViewChange}
               apiBaseUrl={apiBaseUrl}
               contentOnly={true}
             />