Merge pull request #36 from superagent-ai/daytona-webapp-modality

Extend Daytona-transport autobrin contender to support webapp modality (fixes #32)

Author: alanzabihi

src/contenders/autobrin.ts

@@ -109,7 +109,8 @@ export type AttemptRecord = {
 export function buildWebappPayload(input: {
   target: TargetHandle;
   controls: RunControls;
-  workspaceRoot: string;
+  /** Omit for the Daytona transport: the sandbox-side launcher defaults this to its own root. */
+  workspaceRoot?: string;
   contributors?: number;
 }): Record<string, unknown> {
   const webapp = webappTargetMetadata(input.target);
@@ -132,7 +133,7 @@ export function buildWebappPayload(input: {
       secret: webapp.secret,
       secretUploadingUrl: webapp.secretUploadingUrl,
     },
-    workspaceRoot: input.workspaceRoot,
+    ...(input.workspaceRoot !== undefined ? { workspaceRoot: input.workspaceRoot } : {}),
     model: input.controls.model,
     contributors: input.contributors ?? input.controls.contributors,
     ...buildGuardrails(input.controls),
@@ -301,6 +302,12 @@ export function buildReadAttemptsScript(attacksDir: string): string {
  * `runDaytonaEngagement`'s `afterEngagement` hook (before sandbox cleanup) -- by the time
  * `runDaytonaEngagement` itself resolves, the sandbox is already deleted.
  *
+ * The `<workspaceRoot>/workspace/attacks/<attempt>/*.json` layout is a modality-agnostic
+ * autobrin-flue convention (both `repo` and `webapp` modalities call the same `prepareWorkspace()`
+ * layout code), so this reads attempts back the same way regardless of `payload.modality` -- this
+ * mirrors the local-transport reader (`extractClaimFromWorkspace`), which is likewise never gated
+ * on modality.
+ *
  * Deliberately does NOT tolerate a script-level failure (non-zero exit, unparsable/non-array
  * output) by falling back to `[]`: the script's own per-file reads already tolerate an
  * individual attempt missing evaluate/report/disclosure.json (still mid-run, a legitimate `{}`),
@@ -312,8 +319,6 @@ export function buildReadAttemptsScript(attacksDir: string): string {
  * contract (cleanup still runs, the overall call rejects) instead of corrupting the claim.
  */
 export async function fetchAttemptsFromSandbox(sandbox: Sandbox, payload: EngagementPayload): Promise<AttemptRecord[]> {
-  if (payload.modality !== 'repo') return [];
-
   const attacksDir = `${engagementWorkspaceDir(payload)}/attacks`;
   const script = ['set -euo pipefail', "python3 - <<'PY'", buildReadAttemptsScript(attacksDir), 'PY'].join('\n');
   const response = await executeChecked(sandbox, script, '/', 60);
@@ -450,11 +455,14 @@ async function runViaLocalNpx(input: RunInput): Promise<NormalizedResult> {
 async function runViaDaytona(input: RunInput): Promise<NormalizedResult> {
   const { contenderId, config, contributors, task, target, controls, context } = input;
 
-  if (target.modality !== 'repo' || !target.repo) {
+  if (target.modality !== 'repo' && target.modality !== 'webapp') {
     throw new Error(
-      `autobrin contender "${contenderId}": transport "daytona" currently only supports modality "repo" (got "${target.modality}")`,
+      `autobrin contender "${contenderId}": transport "daytona" does not support modality "${target.modality}"`,
     );
   }
+  if (target.modality === 'repo' && !target.repo) {
+    throw new Error(`autobrin contender "${contenderId}": modality "repo" requires target.repo`);
+  }
 
   const started = Date.now();
   const engagementDir = path.join(
@@ -465,8 +473,14 @@ async function runViaDaytona(input: RunInput): Promise<NormalizedResult> {
   await mkdir(context.resultsDir, { recursive: true });
 
   // No workspaceRoot: the sandbox-side launcher defaults it to its own root (BENCHPRESS_ROOT),
-  // never this (local-only) engagementDir.
-  const payload = buildRepoPayload({ target, controls, contributors });
+  // never this (local-only) engagementDir. Webapp targets skip repo-modality-specific target
+  // materialization entirely: there is no target repo to clone into the sandbox, only a URL the
+  // sandbox reaches over the network (runDaytonaEngagement's own modality branch calls
+  // prepareWebappTarget instead of prepareRepoTarget -- see src/daytona/bootstrap.ts).
+  const payload =
+    target.modality === 'webapp'
+      ? buildWebappPayload({ target, controls, contributors })
+      : buildRepoPayload({ target, controls, contributors });
 
   const stdoutChunks: string[] = [];
   const stderrChunks: string[] = [];

src/daytona/payload.ts

@@ -18,9 +18,23 @@ export type RepoEngagementPayload = {
   resume?: boolean;
 };
 
+/** Mirrors autobrin-flue's `WebappTargetSchema` (`docs/modalities.md` on `staging`) field-for-field. */
+export type WebappTargetPayload = {
+  url: string;
+  repo?: string;
+  sha?: string;
+  username?: string;
+  password?: string;
+  role?: string;
+  outboundServiceUrl?: string;
+  proofUploadingUrl?: string;
+  secret?: string;
+  secretUploadingUrl?: string;
+};
+
 export type WebappEngagementPayload = {
   modality: 'webapp';
-  target: { url: string };
+  target: WebappTargetPayload;
   workspaceRoot: string;
   model?: string;
   thinking?: string;
@@ -54,17 +68,29 @@ export function buildRepoPayload(input: {
   };
 }
 
-export function buildWebappPayload(input: {
-  url: string;
-  model?: string;
-  thinking?: string;
-  contributors?: number;
-  guardrails?: EngagementGuardrails;
-  workspaceRoot?: string;
-}): WebappEngagementPayload {
+export function buildWebappPayload(
+  input: WebappTargetPayload & {
+    model?: string;
+    thinking?: string;
+    contributors?: number;
+    guardrails?: EngagementGuardrails;
+    workspaceRoot?: string;
+  },
+): WebappEngagementPayload {
   return {
     modality: 'webapp',
-    target: { url: input.url },
+    target: {
+      url: input.url,
+      repo: input.repo,
+      sha: input.sha,
+      username: input.username,
+      password: input.password,
+      role: input.role,
+      outboundServiceUrl: input.outboundServiceUrl,
+      proofUploadingUrl: input.proofUploadingUrl,
+      secret: input.secret,
+      secretUploadingUrl: input.secretUploadingUrl,
+    },
     workspaceRoot: input.workspaceRoot ?? BENCHPRESS_ROOT,
     model: input.model,
     thinking: input.thinking,
@@ -114,6 +140,15 @@ export function normalizeEngagementPayload(input: unknown): EngagementPayload {
     }
     return buildWebappPayload({
       url: target.url.trim(),
+      repo: typeof target.repo === 'string' ? target.repo : undefined,
+      sha: typeof target.sha === 'string' ? target.sha : undefined,
+      username: typeof target.username === 'string' ? target.username : undefined,
+      password: typeof target.password === 'string' ? target.password : undefined,
+      role: typeof target.role === 'string' ? target.role : undefined,
+      outboundServiceUrl: typeof target.outboundServiceUrl === 'string' ? target.outboundServiceUrl : undefined,
+      proofUploadingUrl: typeof target.proofUploadingUrl === 'string' ? target.proofUploadingUrl : undefined,
+      secret: typeof target.secret === 'string' ? target.secret : undefined,
+      secretUploadingUrl: typeof target.secretUploadingUrl === 'string' ? target.secretUploadingUrl : undefined,
       model: typeof input.model === 'string' ? input.model : undefined,
       thinking: typeof input.thinking === 'string' ? input.thinking : undefined,
       contributors: typeof input.contributors === 'number' ? input.contributors : undefined,

tests/autobrin-contender.test.ts

@@ -239,6 +239,9 @@ describe('fetchAttemptsFromSandbox (real python3 execution over a fake, real-she
   const repoPayload = (workspaceRoot: string): EngagementPayload =>
     ({ modality: 'repo', repo: 'owner/repo', workspaceRoot, targetPreparation: 'prepared', resume: false }) as EngagementPayload;
 
+  const webappPayload = (workspaceRoot: string): EngagementPayload =>
+    ({ modality: 'webapp', target: { url: 'http://127.0.0.1:8080' }, workspaceRoot, resume: false }) as EngagementPayload;
+
   it('returns one record per attempt directory, tolerating missing files', async () => {
     const { root, attacksDir } = makeSandboxRoot();
     const confirmedDir = path.join(attacksDir, 'attempt-confirmed');
@@ -273,13 +276,27 @@ describe('fetchAttemptsFromSandbox (real python3 execution over a fake, real-she
     expect(attempts).toEqual([]);
   });
 
-  it('returns an empty array for non-repo modalities without attempting a fetch', async () => {
-    const attempts = await fetchAttemptsFromSandbox(realShellSandbox(), {
-      modality: 'webapp',
-      target: { url: 'http://127.0.0.1:8080' },
-      workspaceRoot: '/home/daytona/benchpress',
-      resume: false,
-    } as EngagementPayload);
+  it('reads attempts back for webapp modality too (regression: previously skipped modalities other than repo)', async () => {
+    const { root, attacksDir } = makeSandboxRoot();
+    const confirmedDir = path.join(attacksDir, 'attempt-confirmed');
+    mkdirSync(confirmedDir, { recursive: true });
+    writeFileSync(path.join(confirmedDir, 'evaluate.json'), JSON.stringify({ verdict: 'confirmed' }));
+    writeFileSync(path.join(confirmedDir, 'report.json'), JSON.stringify({ location: 'http://127.0.0.1:8080/login' }));
+    writeFileSync(path.join(confirmedDir, 'disclosure.json'), JSON.stringify({ cve_id: 'CVE-2024-3234' }));
+
+    const attempts = await fetchAttemptsFromSandbox(realShellSandbox(), webappPayload(root));
+
+    expect(attempts).toHaveLength(1);
+    expect(attempts[0]?.evaluate).toEqual({ verdict: 'confirmed' });
+    expect(computeClaimFromAttempts(attempts).confirmedFindings).toEqual([
+      { location: 'http://127.0.0.1:8080/login', cve: 'CVE-2024-3234', summary: undefined, verdict: 'confirmed' },
+    ]);
+  });
+
+  it('returns an empty array when the webapp attacks directory does not exist at all', async () => {
+    const emptyRoot = mkdtempSync(path.join(tmpdir(), 'benchpress-sandbox-empty-webapp-'));
+    tmpDirs.push(emptyRoot);
+    const attempts = await fetchAttemptsFromSandbox(realShellSandbox(), webappPayload(emptyRoot));
     expect(attempts).toEqual([]);
   });
 

tests/autobrin-daytona-sequencing.test.ts

@@ -76,3 +76,89 @@ describe('autobrin daytona transport: local checkout must resolve before the san
     expect(checkoutMocks.ensureAutobrinCheckout).toHaveBeenCalledTimes(1);
   });
 });
+
+describe('autobrin daytona transport: webapp modality (superagent-ai/benchpress#32)', () => {
+  const baseTask = { id: 'CVE-2024-3234', benchmarkId: 'cve-bench' };
+  const webappTarget = {
+    benchmarkId: 'cve-bench',
+    taskId: 'CVE-2024-3234',
+    modality: 'webapp' as const,
+    metadata: {
+      webapp: {
+        url: 'http://127.0.0.1:54321',
+        username: 'tester',
+        password: 'hunter2',
+        proofUploadingUrl: 'http://127.0.0.1:54322/upload',
+      },
+    },
+  };
+  const baseControls = { model: 'kimi-azure/kimi-k2.6' };
+
+  const tmpDirs: string[] = [];
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    for (const dir of tmpDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+  });
+
+  function makeContext() {
+    const root = mkdtempSync(path.join(tmpdir(), 'benchpress-daytona-webapp-'));
+    tmpDirs.push(root);
+    return { runId: 'run1', resultsDir: path.join(root, 'results'), engagementsDir: path.join(root, 'engagements') };
+  }
+
+  it('builds a webapp payload (not buildRepoPayload) and never requires target.repo', async () => {
+    checkoutMocks.ensureAutobrinCheckout.mockReset().mockResolvedValue({ root: '/cache/x', ref: 'staging', commitSha: 'deadbeef' });
+    launcherMocks.runDaytonaEngagement.mockReset().mockResolvedValue({
+      sandboxId: 'sandbox-1',
+      engagement: { exitCode: 0, streamLogPath: 'x', resultPath: 'y', resultJson: {} },
+      computerUse: {},
+      keptSandbox: false,
+    });
+
+    const runner = createAutobrinRunner({ config: { id: 'x', type: 'autobrin', transport: 'daytona', image: 'test-image' } });
+    const result = await runner.run({ task: baseTask, target: webappTarget, controls: baseControls, context: makeContext() });
+
+    expect(result.exitCode).toBe(0);
+    expect(launcherMocks.runDaytonaEngagement).toHaveBeenCalledTimes(1);
+    const options = launcherMocks.runDaytonaEngagement.mock.calls[0]?.[0] as { payload: Record<string, unknown> };
+    expect(options.payload).toMatchObject({
+      modality: 'webapp',
+      target: {
+        url: 'http://127.0.0.1:54321',
+        username: 'tester',
+        password: 'hunter2',
+        proofUploadingUrl: 'http://127.0.0.1:54322/upload',
+      },
+    });
+    // Daytona transport omits workspaceRoot for webapp too -- the sandbox-side launcher defaults
+    // it to its own root, same as the existing repo-modality behavior.
+    expect('workspaceRoot' in options.payload).toBe(false);
+    expect('repo' in options.payload).toBe(false);
+  });
+
+  it('rejects an unsupported modality (e.g. "model") with a clear error, and never starts the sandbox', async () => {
+    checkoutMocks.ensureAutobrinCheckout.mockReset();
+    launcherMocks.runDaytonaEngagement.mockReset();
+    const runner = createAutobrinRunner({ config: { id: 'x', type: 'autobrin', transport: 'daytona', image: 'test-image' } });
+    const modelTarget = { benchmarkId: 'b', taskId: 't1', modality: 'model' as const };
+
+    await expect(
+      runner.run({ task: baseTask, target: modelTarget, controls: baseControls, context: makeContext() }),
+    ).rejects.toThrow(/does not support modality "model"/);
+    expect(launcherMocks.runDaytonaEngagement).not.toHaveBeenCalled();
+    expect(checkoutMocks.ensureAutobrinCheckout).not.toHaveBeenCalled();
+  });
+
+  it('still rejects modality "repo" missing target.repo, unchanged from before', async () => {
+    checkoutMocks.ensureAutobrinCheckout.mockReset();
+    launcherMocks.runDaytonaEngagement.mockReset();
+    const runner = createAutobrinRunner({ config: { id: 'x', type: 'autobrin', transport: 'daytona', image: 'test-image' } });
+    const repoTargetMissingRepo = { benchmarkId: 'b', taskId: 't1', modality: 'repo' as const };
+
+    await expect(
+      runner.run({ task: baseTask, target: repoTargetMissingRepo, controls: baseControls, context: makeContext() }),
+    ).rejects.toThrow(/modality "repo" requires target\.repo/);
+    expect(launcherMocks.runDaytonaEngagement).not.toHaveBeenCalled();
+  });
+});

tests/daytona.test.ts

@@ -94,6 +94,57 @@ describe('daytona payload', () => {
     });
   });
 
+  // Regression (superagent-ai/benchpress#32): the Daytona transport's webapp payload used to carry
+  // only target.url, silently dropping the rest of the canonical WebappTargetMetadata contract
+  // (username/password/proofUploadingUrl/etc.) that src/contenders/autobrin.ts's buildWebappPayload
+  // already sends -- neutering benchmarks like CVE-Bench, whose tasks always require credentials
+  // and a proof-upload URL (see src/benchmarks/cve-bench/metadata.ts).
+  it('carries the full webapp target contract through, not just url', () => {
+    const payload = buildWebappPayload({
+      url: 'http://127.0.0.1:8080',
+      repo: 'owner/repo',
+      sha: 'abc123',
+      username: 'attacker',
+      password: 'hunter2',
+      role: 'user',
+      outboundServiceUrl: 'http://target-internal:9000',
+      proofUploadingUrl: 'http://127.0.0.1:9091/upload',
+      secret: 'topsecret',
+      secretUploadingUrl: 'http://127.0.0.1:9091/secret',
+    });
+    expect(payload.target).toEqual({
+      url: 'http://127.0.0.1:8080',
+      repo: 'owner/repo',
+      sha: 'abc123',
+      username: 'attacker',
+      password: 'hunter2',
+      role: 'user',
+      outboundServiceUrl: 'http://target-internal:9000',
+      proofUploadingUrl: 'http://127.0.0.1:9091/upload',
+      secret: 'topsecret',
+      secretUploadingUrl: 'http://127.0.0.1:9091/secret',
+    });
+  });
+
+  it('normalizes webapp payloads without dropping credentials/proof-upload fields (regression: daytona transport silently stripped these)', () => {
+    const normalized = normalizeEngagementPayload({
+      modality: 'webapp',
+      target: {
+        url: 'http://127.0.0.1:8080',
+        username: 'attacker',
+        password: 'hunter2',
+        proofUploadingUrl: 'http://127.0.0.1:9091/upload',
+      },
+    });
+    if (normalized.modality !== 'webapp') throw new Error('expected a webapp payload');
+    expect(normalized.target).toEqual({
+      url: 'http://127.0.0.1:8080',
+      username: 'attacker',
+      password: 'hunter2',
+      proofUploadingUrl: 'http://127.0.0.1:9091/upload',
+    });
+  });
+
   it('normalizes payload JSON objects', () => {
     expect(
       normalizeEngagementPayload({