cases.jsonl

{"id":"linear-slack.chat","agent":"linear-slack","kind":"chat","fixture":{"type":"slack.message.created","resource":{"channel":"C0TEST","ts":"100.1","text":"What's open on the board for the export work?","user":"U1"}},"inputs":{"SLACK_CHANNEL":"C0TEST"},"seeds":["linear/projects","linear/issues","linear/teams"],"expect":{"status":"succeeded","eventSource":"slack","sideEffectsAll":["harness.run"]},"rubric":"A grounded Slack answer about open Linear issues for the export work, citing real issues from the board. Read-only unless asked to create; must not fabricate issue refs."}
{"id":"linear.chat","agent":"linear","kind":"chat","fixture":{"type":"linear.AgentSessionEvent.prompted","resource":{"payload":{"agentSession":{"id":"session-1","issue":{"id":"issue-1"}},"agentActivity":{"body":"What's the current status of this issue?"}}}},"inputs":{},"seeds":[{"vfs":"/linear/issues/by-uuid/issue-1.json","file":"linear-issue-1.json"}],"expect":{"status":"succeeded","eventSource":"linear","sideEffectsAll":["llm.complete"],"logsAny":["linear event"]},"rubric":"A grounded conversational status reply about the issue. Read-only: must not claim to have edited or closed anything."}
{"id":"review.review","agent":"review","kind":"triage","fixture":{"type":"github.pull_request.opened","resource":{"pull_request":{"number":7,"html_url":"https://github.com/acme/widget/pull/7","user":{"login":"alice"},"head":{"sha":"abc123"},"state":"open","draft":false},"repository":{"name":"widget","owner":{"login":"acme"}}}},"inputs":{},"seeds":[],"expect":{"status":"succeeded","eventSource":"github","sideEffectsAll":["harness.run"]},"rubric":"A code review that runs the harness against the PR diff and surfaces real issues (e.g. the unpaginated export OOM)."}
{"id":"review.skip-label","agent":"review","kind":"triage","fixture":{"type":"github.pull_request.opened","resource":{"pull_request":{"number":8,"html_url":"https://github.com/acme/widget/pull/8","user":{"login":"alice"},"labels":[{"name":"no-agent-relay-review"}],"head":{"sha":"def456"},"state":"open","draft":false},"repository":{"name":"widget","owner":{"login":"acme"}}}},"inputs":{},"seeds":[],"expect":{"status":"succeeded","eventSource":"github","logsAny":["pr-reviewer skipped"]},"rubric":"A PR carrying the opt-out label must be skipped without running the review harness."}
{"id":"repo-hygiene.diagnose","agent":"repo-hygiene","kind":"triage","fixture":{"type":"github.pull_request.synchronize","resource":{"pull_request":{"number":7,"html_url":"https://github.com/acme/widget/pull/7","title":"Add CSV export endpoint","user":{"login":"alice"},"head":{"sha":"abc123","ref":"feat/x"},"base":{"ref":"main"}},"repository":{"name":"widget","full_name":"acme/widget","owner":{"login":"acme"}}}},"inputs":{"NOTION_DATABASE_ID":"db_test"},"seeds":[{"vfs":"/github/repos/acme/widget/pulls/7/meta.json","file":"github-pr-widget-7-meta.json"}],"expect":{"status":"succeeded","eventSource":"github","sideEffectsAll":["harness.run"],"logsAny":["repo-hygiene.notion-page.no-receipt","repo-hygiene.journal-failed"]},"rubric":"Diagnoses repo-hygiene issues in the PR and runs the harness; the Notion journal write is best-effort and logs when no receipt comes back."}
{"id":"hn-monitor.agentic-feeds","agent":"hn-monitor","kind":"scheduled","fixture":{"type":"cron.tick","name":"scan","cron":"0 9,17 * * *"},"inputs":{"SLACK_CHANNEL":"C123","TOPICS":"agents,ai,typescript,developer tools","LOOKBACK_HOURS":"24","MAX_STORIES":"8"},"seeds":[],"http":[{"match":"tags=front_page","file":"hn-front-page.json"},{"match":"tags=show_hn","file":"hn-show-hn.json"},{"match":"tags=story","file":"hn-new.json"}],"expect":{"status":"succeeded","eventSource":"cron","sideEffectsAll":["llm.complete"],"logsAny":["hn-monitor.feed-scan front_page=1 show_hn=1 new=2","hn-monitor.matched-agentic matched=3","hn-monitor.posted"]},"rubric":"Scans deterministic Front Page, Show HN, and New HN fixtures; keeps the three agent-infrastructure/coding-agent stories while rejecting high-engagement TypeScript, travel-agent, weather, and generic-AI noise; then posts a threaded digest."}
{"id":"hn-monitor.chat-detail","agent":"hn-monitor","kind":"chat","turns":[{"type":"cron.tick","name":"scan","cron":"0 9,17 * * *"},{"type":"slack.app_mention","channel":"C123","ts":"200.2","thread_ts":"200.1","text":"<@UBOT> tell me more about story 1 and what HN commenters think","user":"U1"}],"inputs":{"SLACK_CHANNEL":"C123","LOOKBACK_HOURS":"24","MAX_STORIES":"8"},"seeds":[],"http":[{"match":"tags=front_page","file":"hn-front-page.json"},{"match":"tags=show_hn","file":"hn-show-hn.json"},{"match":"tags=story","file":"hn-new.json"},{"match":"/api/v1/items/1001","file":"hn-item-1001.json"}],"expect":{"status":"succeeded","eventSource":"slack","sideEffectsAll":["llm.complete"],"logsAny":["hn-monitor.qa.hydrated","hn-monitor.qa.slack-replied"]},"rubric":"After a scheduled scan, answers an in-thread Slack follow-up about story 1 using the stored finding plus freshly hydrated HN metadata and top comments. It must frame comments as community reaction, not verified article facts, and include useful links."}
{"id":"hn-monitor.chat-exact-title-empty-memory","agent":"hn-monitor","kind":"chat","fixture":{"type":"slack.app_mention","resource":{"channel":"C123","ts":"300.2","thread_ts":"300.1","text":"<@UBOT> European Parliament MCP Server – Political Intelligence for AI Agents -> give me more info on this","user":"U1"}},"inputs":{"SLACK_CHANNEL":"C123"},"seeds":[],"http":[{"match":"restrictSearchableAttributes=title","file":"hn-european-title-search.json"},{"match":"/api/v1/items/4242","file":"hn-item-4242.json"}],"expect":{"status":"succeeded","eventSource":"slack","sideEffectsAll":["llm.complete"],"logsAll":["hn-monitor.qa.recalled","hn-monitor.qa.selected","hn-monitor.qa.hydrated","hn-monitor.qa.slack-replied"],"replyContains":["European Parliament MCP Server","https://example.com/eu-parliament-mcp","https://news.ycombinator.com/item?id=4242"]},"rubric":"With no prior memory or deterministic state, the exact European Parliament MCP title must resolve through strict live HN title lookup, hydrate current metadata and comments, and answer with the article and HN links. It must not ask for an ID, link, or date already recoverable from the supplied title, and must frame HN comments as community reaction."}
{"id":"spotify-releases.requires-token","agent":"spotify-releases","kind":"guard","fixture":{"type":"cron.tick","name":"check","cron":"0 10 * * *"},"inputs":{"SLACK_USER":"U123"},"seeds":[],"expect":{"status":"failed","errorIncludes":"SPOTIFY_TOKEN"},"rubric":"Missing the required SPOTIFY_TOKEN input must fail loudly rather than silently no-op."}
{"id":"vendor-monitor.requires-channel","agent":"vendor-monitor","kind":"guard","fixture":{"type":"cron.tick","name":"check","cron":"0 8 * * 1-5"},"inputs":{},"seeds":[],"expect":{"status":"failed","errorIncludes":"SLACK_CHANNEL"},"rubric":"Missing the required SLACK_CHANNEL input must fail loudly rather than silently no-op."}
{"id":"granola.note-classify","agent":"granola","kind":"triage","fixture":{"type":"granola.file.created","provider":"granola","paths":["/granola/notes/note-2.json"],"resource":{"path":"/granola/notes/note-2.json","kind":"file","id":"note-2","provider":"granola"}},"inputs":{},"seeds":[{"vfs":"/granola/notes/note-2.json","file":"granola-note-prospect.json"}],"expect":{"status":"succeeded","eventSource":"granola","sideEffectsAll":["llm.complete"],"logsAny":["granola-prospect.not-a-prospect"]},"rubric":"Reads the meeting note and classifies whether it describes a sales prospect before doing any Linear/PR work."}
{"id":"cloud-team-implementer.misroute","agent":"cloud-team-implementer","kind":"guard","fixture":{"type":"cron.tick","name":"tick","cron":"0 * * * *"},"inputs":{},"seeds":[],"expect":{"status":"succeeded","eventSource":"cron","logsAny":["cloud-team-implementer received a direct event; members are launched by the team dispatcher, not by subscriptions"]},"rubric":"A team member should never act on a direct event — it warns that it is launched by the dispatcher."}
{"id":"cloud-team-reviewer.misroute","agent":"cloud-team-reviewer","kind":"guard","fixture":{"type":"cron.tick","name":"tick","cron":"0 * * * *"},"inputs":{},"seeds":[],"expect":{"status":"succeeded","eventSource":"cron","logsAny":["cloud-team-reviewer received a direct event; members are launched by the team dispatcher, not by subscriptions"]},"rubric":"A team member should never act on a direct event — it warns that it is launched by the dispatcher."}
{"id":"inbox-buddy.chat","agent":"inbox-buddy","kind":"chat","fixture":{"type":"slack.app_mention","resource":{"channel":"C0TEST","ts":"100.1","text":"What's the latest on the Q3 export thread with Alice?","user":"U1"}},"inputs":{"SLACK_CHANNEL":"C0TEST"},"seeds":[{"vfs":"/google-mail/threads/T_alice_export.json","file":"gmail-thread-alice-export.json"},{"vfs":"/google-mail/threads/T_github_pr.json","file":"gmail-thread-github-pr.json"},{"vfs":"/google-mail/threads/T_bob_lunch.json","file":"gmail-thread-bob-lunch.json"},{"vfs":"/google-mail/threads/T_newsletter.json","file":"gmail-thread-newsletter.json"}],"expect":{"status":"succeeded","eventSource":"slack","sideEffectsAny":["llm.complete"],"logsAny":["inbox-buddy.context"],"replyContains":["Friday","finance"]},"rubric":"A grounded answer about the Q3 export thread with Alice, citing concrete details from that thread (final numbers by Friday, finance looped in for sign-off). Must not fabricate senders or facts."}
{"id":"inbox-buddy.chat-multiturn","agent":"inbox-buddy","kind":"chat","turns":[{"type":"slack.app_mention","channel":"C0TEST","ts":"100.1","text":"What's the latest on the Q3 export thread with Alice?","user":"U1"},{"type":"slack.app_mention","channel":"C0TEST","ts":"100.2","text":"Who did she loop in, and when did she say she'd send it?","user":"U1"}],"inputs":{"SLACK_CHANNEL":"C0TEST"},"seeds":[{"vfs":"/google-mail/threads/T_alice_export.json","file":"gmail-thread-alice-export.json"},{"vfs":"/google-mail/threads/T_github_pr.json","file":"gmail-thread-github-pr.json"},{"vfs":"/google-mail/threads/T_bob_lunch.json","file":"gmail-thread-bob-lunch.json"},{"vfs":"/google-mail/threads/T_newsletter.json","file":"gmail-thread-newsletter.json"}],"expect":{"status":"succeeded","eventSource":"slack","sideEffectsAny":["llm.complete"],"logsAny":["inbox-buddy.context"],"replyContains":["finance@acme.com","Friday"]},"rubric":"Two-turn conversation. The second answer must correctly state that Alice looped in finance (finance@acme.com) and said she would send the numbers by Friday — facts that appear only in the first thread/turn, so a correct, specific answer proves the agent retained context from turn 1. Must name Alice/finance explicitly and not fabricate."}