feat(summary): auto-enqueue a trajectory summary for every finished trial

A trajectory summary only ever existed where something asked for one: the
post-trial classifier's component map, which runs on `run_analysis` tasks, or a
dashboard/share page read. Trials on tasks with QA off carried no summary until
a human opened them.

`_run_post_trial_hooks` now enqueues one per finished trial, behind
`ODDISH_AUTO_TRAJECTORY_SUMMARY` (off by default). Eligibility is real agent
trials only -- terminal, not cancelled, not superseded, not a probe, not a
nop/oracle baseline, and with a fetchable trajectory. Baselines and probes are
excluded because their trajectories are near-empty or harness noise and each
would still pay for a full LLM call.

The enqueue itself is delegated to `get_or_enqueue_summary_job` through a seam,
because that function owns the payload and its `schema_version` idempotency
key. A second enqueue site with its own key would not find the first one's job,
so a page view after a trial finished would pay for the same summary twice.
Eligibility deliberately does not skip a trial that already has a mirrored
summary: the mirror can be at an older schema, and only the seam knows the
current one.

Runs in the hook's own transaction, after the stage transition, so a hook that
rolls back leaves no paid job behind and the QA job enqueued above it finds the
summary cached rather than building its own.

Author: stateofkate

AGENTS.md

@@ -207,7 +207,14 @@ enqueues one trial-scoped
 summary schema. Its endpoint returns explicit queued/running/retrying state and
 the client polls until the summary is stored. Terminal failures are returned,
 not re-enqueued by anonymous refreshes; a schema bump creates the next valid
-idempotency key. Public capability jobs, cache entries, summary warmup, and
+idempotency key. `ODDISH_AUTO_TRAJECTORY_SUMMARY` (off by default) adds a second
+trigger for that same job: every finished non-baseline, non-probe trial enqueues
+one from `_run_post_trial_hooks`, so trials on `run_analysis=False` tasks stop
+depending on a human opening them. It must go through
+`get_or_enqueue_summary_job` -- reached from `oddish/` via the enqueuer seam in
+`workers.queue.trajectory_summary_job`, since only `backend/` knows
+`SCHEMA_VERSION`. A second enqueue site owning its own idempotency key would not
+find this job, and the next page view would pay for the same summary again. Public capability jobs, cache entries, summary warmup, and
 cohort queries are keyed by the published experiment as well as task version;
 they must never include trials from another experiment on the same version. Any
 completed, fetchable trajectory is enough to queue analysis; cohort size is

backend/tests/test_trajectory_summary_auto_enqueue_seam.py

@@ -0,0 +1,43 @@
+"""Importing the worker's provider module installs the auto-enqueue seam.
+
+The oddish side no-ops silently when nothing is registered, so a missing
+registration would not fail anything -- trials would just quietly stop getting
+summaries. That is what this pins.
+
+It also pins the delegation: the enqueuer must go through
+``get_or_enqueue_summary_job`` rather than build its own ``worker_jobs`` row.
+That function owns the ``schema_version`` idempotency key, and a second writer
+with a different key would not find the first one's job, so a page view after a
+trial finished would pay for the same summary twice.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from oddish.workers.queue import trajectory_summary_job as tsj
+
+
+def test_importing_the_provider_registers_the_enqueuer():
+    import worker.trajectory_summary_provider as mod
+
+    assert tsj._enqueuer is mod.enqueue_trajectory_summary
+
+
+@pytest.mark.asyncio
+async def test_the_enqueuer_delegates_to_get_or_enqueue_summary_job(monkeypatch):
+    import api.services.summarize_trajectory as svc
+    import worker.trajectory_summary_provider as mod
+
+    seen: list[tuple] = []
+    sentinel = object()
+
+    async def _fake(session, trial):
+        seen.append((session, trial))
+        return sentinel
+
+    monkeypatch.setattr(svc, "get_or_enqueue_summary_job", _fake)
+
+    session, trial = object(), object()
+    assert await mod.enqueue_trajectory_summary(session, trial) is sentinel
+    assert seen == [(session, trial)]

backend/worker/trajectory_summary_provider.py

@@ -21,7 +21,11 @@
 from oddish.workers.queue.analysis_handler import (
     register_trajectory_summary_provider as register_analysis_provider,
 )
+from oddish.workers.queue.trajectory_summary_job import (
+    register_trajectory_summary_enqueuer,
+)
 
+from api.services import summarize_trajectory
 from api.services.summarize_trajectory import get_or_generate_summary
 
 
@@ -44,6 +48,17 @@ async def provide_trajectory_summary(
         )
 
 
-# Importing this module (from backend.worker.functions) installs the hook.
+async def enqueue_trajectory_summary(session, trial):
+    """TrajectorySummaryEnqueuer impl for the post-trial auto-enqueue.
+
+    A pass-through on purpose: ``get_or_enqueue_summary_job`` stays the only
+    writer of the job's payload and its ``schema_version`` idempotency key, so a
+    later page view finds this job instead of paying for the summary again.
+    """
+    return await summarize_trajectory.get_or_enqueue_summary_job(session, trial)
+
+
+# Importing this module (from backend.worker.functions) installs the hooks.
 register_analysis_provider(provide_trajectory_summary)
 register_job_provider(provide_trajectory_summary)
+register_trajectory_summary_enqueuer(enqueue_trajectory_summary)

oddish/src/oddish/config.py

@@ -1341,6 +1341,13 @@ def analyzer_snapshot(self) -> str:
     # ODDISH_GATE_LLM_ON_BASELINES; default off leaves every path unchanged.
     gate_llm_on_baselines: bool = False
 
+    # When enabled, every finished agent trial enqueues its own trajectory-summary
+    # job from the post-trial hook, instead of a summary existing only where
+    # post-trial classification built one (run_analysis tasks) or somebody read a
+    # trajectory. Env-driven via ODDISH_AUTO_TRAJECTORY_SUMMARY. Off by default:
+    # it is one LLM call per trial, and standalone oddish registers no enqueuer.
+    auto_trajectory_summary: bool = False
+
     # DEPRECATED (default OFF; see workers.queue.concurrency_controller). The
     # self-tuning advisory controller predates database-backed admin overrides,
     # which are now the supported way to change a per-model limit at runtime:
@@ -1576,6 +1583,15 @@ def normalize_model_overrides(self) -> "Settings":
                 "on",
             }
 
+        auto_summary_raw = os.getenv("ODDISH_AUTO_TRAJECTORY_SUMMARY")
+        if auto_summary_raw is not None:
+            self.auto_trajectory_summary = auto_summary_raw.strip().lower() in {
+                "1",
+                "true",
+                "yes",
+                "on",
+            }
+
         raw_buckets = os.getenv("ODDISH_PROVIDER_RATE_LIMITS")
         if raw_buckets:
             try:

oddish/src/oddish/workers/queue/trajectory_summary_job.py

@@ -0,0 +1,88 @@
+"""Auto-enqueue a trajectory-summary job for every finished agent trial.
+
+The durable job already exists (``ANALYZER`` + ``payload.mode ==
+"trajectory_summary"``, handled in ``workers.jobs.handlers``). What was missing
+is anything that creates one on its own: the only enqueue site is a public page
+read, so a trial on a ``run_analysis=False`` task carried no summary until a
+human opened its trajectory.
+
+Only the *policy* lives here -- the flag and which trials are worth summarizing.
+The enqueue itself is delegated to the seam below, which the hosted backend fills
+with ``get_or_enqueue_summary_job``. That function owns the payload shape and the
+``schema_version`` idempotency key, and it must stay the single writer of them: a
+second enqueue site with its own key would not find the first one's job, so a
+page view after a trial finished would pay for the same summary twice. Standalone
+oddish leaves the seam empty and this whole module is a no-op, which is correct --
+there is no summary generator to reach.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Awaitable, Callable
+
+from oddish.config import is_nop_oracle_agent, settings
+from oddish.core.cost_basis import CANCELLED_HARBOR_STAGE
+from oddish.core.helpers import _has_fetchable_trajectory
+from oddish.db import TrialStatus
+
+# Takes (session, trial) and returns the one durable job for that trial and
+# schema, enqueueing it if absent. Runs in the caller's transaction.
+TrajectorySummaryEnqueuer = Callable[[Any, Any], Awaitable[Any]]
+
+_enqueuer: TrajectorySummaryEnqueuer | None = None
+
+
+def register_trajectory_summary_enqueuer(fn: TrajectorySummaryEnqueuer) -> None:
+    """Install the hosted enqueue implementation."""
+    global _enqueuer
+    _enqueuer = fn
+
+
+def trial_wants_trajectory_summary(trial) -> bool:
+    """Whether *trial* is one a human would ever read a summary of.
+
+    Baselines and probes are excluded because their trajectories are near-empty
+    or sanctioned-harness noise, and each would still pay for a full LLM call.
+
+    Deliberately does *not* skip a trial that already has a
+    ``trajectory_summary``: a mirrored summary can be at an older
+    ``schema_version``, and only the enqueuer knows the current one. Short-
+    circuiting here would silently skip exactly the trials a schema bump means
+    to re-summarize.
+    """
+    if trial.status not in (TrialStatus.SUCCESS, TrialStatus.FAILED):
+        return False
+    if (trial.harbor_stage or "") == CANCELLED_HARBOR_STAGE:
+        return False
+    if trial.superseded_by_trial_id is not None:
+        return False
+    if trial.is_probe:
+        return False
+    if is_nop_oracle_agent(trial.agent):
+        return False
+    return _has_fetchable_trajectory(trial)
+
+
+async def enqueue_trajectory_summary_job(session, trial):
+    """Ensure *trial* has a summary job, or return ``None`` if it wants none.
+
+    ``None`` when the flag is off, no enqueuer is registered, or the trial is
+    ineligible. Runs in the caller's transaction so the job commits with the
+    rest of the post-trial hook rather than outliving a rollback.
+    """
+    if not settings.auto_trajectory_summary:
+        return None
+    enqueuer = _enqueuer
+    if enqueuer is None:
+        return None
+    if not trial_wants_trajectory_summary(trial):
+        return None
+    return await enqueuer(session, trial)
+
+
+__all__ = [
+    "TrajectorySummaryEnqueuer",
+    "enqueue_trajectory_summary_job",
+    "register_trajectory_summary_enqueuer",
+    "trial_wants_trajectory_summary",
+]

oddish/src/oddish/workers/queue/trial_handler.py

@@ -958,6 +958,9 @@ async def _store_trial_results(
 
 async def _run_post_trial_hooks(trial_id: str) -> None:
     from oddish.queue import maybe_gate_llm_trials, maybe_start_qa_stage
+    from oddish.workers.queue.trajectory_summary_job import (
+        enqueue_trajectory_summary_job,
+    )
 
     async with get_session() as session:
         if (
@@ -982,6 +985,10 @@ async def _run_post_trial_hooks(trial_id: str) -> None:
             console.print(
                 f"[blue]Task {trial.task_id} transitioned to next stage[/blue]"
             )
+        # In this same transaction, and after the stage transition: a hook that
+        # rolls back must not leave a paid job behind, and the QA job enqueued
+        # above finds the summary already cached rather than building its own.
+        await enqueue_trajectory_summary_job(session, trial)
 
 
 async def _finish_trial_settlement(