Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 1 addition & 10 deletions frontend/src/components/experiment-detail-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import {
import {
buildExperimentAgentSummaries,
getExperimentAgentKey,
getModelScopedAgentsFromSummaries,
isBaselineAgentName,
type ExperimentAgentSummary,
} from "@/lib/experiment-agent-grouping";
Expand Down Expand Up @@ -140,16 +141,6 @@ interface ExperimentDetailViewProps {

const AGENT_SUMMARY_STORAGE_PREFIX = "oddish:experiment-agent-summaries:";

function getModelScopedAgentsFromSummaries(
summaries: ExperimentAgentSummary[]
): Set<string> {
return new Set(
summaries
.filter((summary) => summary.isModelScoped)
.map((summary) => summary.agent)
);
}

type ExperimentSummary = {
rewardSuccess: number;
rewardSum: number;
Expand Down
40 changes: 19 additions & 21 deletions frontend/src/components/experiment-trials-table.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1772,29 +1772,27 @@ export function ExperimentTrialsTable({
return (
<TooltipProvider>
<div className="space-y-4">
{/* Pass/k Graph - only shows when there are multiple trials per task-agent */}
{/* Graphs row: pass/k curve (needs multiple trials per task-agent)
and leaderboard. A card with nothing to show renders null and
contributes no grid cell, so the row reflows around it. */}
{showPassAtK ? (
<div className="grid items-stretch gap-4 xl:grid-cols-2">
<div className="h-full min-w-0">
<PassAtKGraph
tasks={tasks}
agentSummaries={sortedAgentSummaries}
hiddenAgents={hiddenAgents}
onToggleAgent={toggleAgent}
hoverAgent={hoverAgent}
onHoverAgent={setHoverAgent}
/>
</div>
<div className="h-full min-w-0">
<PassAtOneLeaderboard
tasks={tasks}
agentSummaries={sortedAgentSummaries}
hiddenAgents={hiddenAgents}
onToggleAgent={toggleAgent}
hoverAgent={hoverAgent}
onHoverAgent={setHoverAgent}
/>
</div>
<PassAtKGraph
tasks={tasks}
agentSummaries={sortedAgentSummaries}
hiddenAgents={hiddenAgents}
onToggleAgent={toggleAgent}
hoverAgent={hoverAgent}
onHoverAgent={setHoverAgent}
/>
<PassAtOneLeaderboard
tasks={tasks}
agentSummaries={sortedAgentSummaries}
hiddenAgents={hiddenAgents}
onToggleAgent={toggleAgent}
hoverAgent={hoverAgent}
onHoverAgent={setHoverAgent}
/>
</div>
) : null}

Expand Down
25 changes: 4 additions & 21 deletions frontend/src/components/pass-at-k-graph.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"use client";

import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react";
import { memo, useCallback, useMemo } from "react";
import {
CartesianGrid,
Line,
Expand All @@ -14,6 +14,7 @@ import type { TooltipContentProps } from "recharts";
import type { Task, Trial } from "@/lib/types";
import { calculatePassAtKCurve, type AgentPassAtKStats } from "@/lib/pass-at-k";
import { getExperimentAgentKey } from "@/lib/experiment-agent-grouping";
import { useElementSize } from "@/lib/use-element-size";
import type { AgentSummary } from "./experiment-trials-table";
import { AgentLegend } from "@/components/agent-legend";

Expand Down Expand Up @@ -103,31 +104,13 @@ export const PassAtKGraph = memo(function PassAtKGraph({
hoverAgent,
onHoverAgent,
}: PassAtKGraphProps) {
const chartContainerRef = useRef<HTMLDivElement>(null);
const [chartSize, setChartSize] = useState({ width: 0, height: 0 });
const { ref: chartContainerRef, size: chartSize } =
useElementSize<HTMLDivElement>();
const visibleAgentSummaries = useMemo(
() => agentSummaries.filter((summary) => !hiddenAgents.has(summary.key)),
[agentSummaries, hiddenAgents],
);

useEffect(() => {
const element = chartContainerRef.current;
if (!element) return;

const updateSize = () => {
const rect = element.getBoundingClientRect();
setChartSize({
width: Math.max(0, Math.floor(rect.width)),
height: Math.max(0, Math.floor(rect.height)),
});
};

updateSize();
const observer = new ResizeObserver(updateSize);
observer.observe(element);
return () => observer.disconnect();
}, []);

const { data, maxK, hasMultipleAttempts, agentColorByKey, agentLabelByKey } =
useMemo(() => {
const { agentStats, maxN } = buildAgentStats(tasks, agentSummaries);
Expand Down
13 changes: 3 additions & 10 deletions frontend/src/components/pass-at-one-leaderboard.tsx
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"use client";

import { memo, useMemo } from "react";
import type { Task, Trial } from "@/lib/types";
import type { Task } from "@/lib/types";
import { getExperimentAgentKey } from "@/lib/experiment-agent-grouping";
import { passAtOneFraction } from "@/lib/pass-at-k";
import type { AgentSummary } from "./experiment-trials-table";
import { AGENT_COLORS } from "./pass-at-k-graph";
import { QueueKeyIcon } from "./queue-key-icon";
Expand All @@ -25,14 +26,6 @@ type LeaderboardRow = {
mean: number;
};

function getPassAtOneValue(trials: Trial[]): number | null {
// Skipped trials are counted as non-passes in the denominator (like a harness
// error): a skipped trial did not pass, so it lowers pass@1 accordingly.
if (trials.length === 0) return null;
const passing = trials.filter((trial) => trial.reward === 1).length;
return passing / trials.length;
}

function calculateRows(
tasks: Task[],
agentSummaries: AgentSummary[],
Expand All @@ -51,7 +44,7 @@ function calculateRows(
(trial) =>
getExperimentAgentKey(trial, modelScopedAgents) === summary.key,
);
const value = getPassAtOneValue(trials);
const value = passAtOneFraction(trials);
if (value !== null) {
taskValues.push(value);
}
Expand Down
12 changes: 12 additions & 0 deletions frontend/src/lib/experiment-agent-grouping.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,18 @@ function getModelScopedAgents(tasks: Task[]): Set<string> {
);
}

// Recover the model-scoped agent set from already-built summaries, for
// consumers that hold summaries but not the tasks they were derived from.
export function getModelScopedAgentsFromSummaries(
summaries: readonly ExperimentAgentSummary[]
): Set<string> {
return new Set(
summaries
.filter((summary) => summary.isModelScoped)
.map((summary) => summary.agent)
);
}

export function getExperimentAgentKey(
trial: Pick<Trial, "agent" | "model" | "is_probe">,
modelScopedAgents: ReadonlySet<string>
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/lib/pass-at-k.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,19 @@ function calculatePassAtK(n: number, c: number, k: number): number {
return 1 - product;
}

/**
* Fraction of trials that passed (reward === 1) — the pass@1 point estimate
* for one task. Skipped and errored trials count in the denominator as
* non-passes. Null when there is nothing to score. Shared by the leaderboard
* and the Pareto frontier card so their scores can never drift apart.
*/
export function passAtOneFraction(
trials: readonly { reward: number | null }[],
): number | null {
if (trials.length === 0) return null;
return trials.filter((trial) => trial.reward === 1).length / trials.length;
}

interface PassAtKDataPoint {
k: number;
[agent: string]: number; // pass@k value for each agent
Expand Down
31 changes: 31 additions & 0 deletions frontend/src/lib/use-element-size.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { useEffect, useRef, useState } from "react";

/**
* Track an element's rendered size (floored, never negative). The chart
* cards render their fixed-size ResponsiveContainer only once this is
* non-zero, so recharts never lays out against an unmeasured box.
*/
export function useElementSize<T extends HTMLElement>() {
const ref = useRef<T>(null);
const [size, setSize] = useState({ width: 0, height: 0 });

useEffect(() => {
const element = ref.current;
if (!element) return;

const updateSize = () => {
const rect = element.getBoundingClientRect();
setSize({
width: Math.max(0, Math.floor(rect.width)),
height: Math.max(0, Math.floor(rect.height)),
});
};

updateSize();
const observer = new ResizeObserver(updateSize);
observer.observe(element);
return () => observer.disconnect();
}, []);

return { ref, size };
}