enhance scenario execution logging and summary reporting

Author: FreakDev

src/cli/parallel-display.test.ts

@@ -0,0 +1,76 @@
+import assert from "node:assert/strict";
+import { describe, it, beforeEach, afterEach } from "node:test";
+import type { ScenarioResult } from "../types/verdict.js";
+import { emptyTranscript } from "./concurrency.js";
+import { ParallelScenarioDisplay } from "./parallel-display.js";
+
+function stubResult(
+  overrides: Partial<ScenarioResult> & Pick<ScenarioResult, "scenario" | "status">,
+): ScenarioResult {
+  return {
+    filePath: "scenarios/example.md",
+    durationMs: 1000,
+    verdict: null,
+    transcript: emptyTranscript(),
+    ...overrides,
+  };
+}
+
+describe("ParallelScenarioDisplay", () => {
+  let stdoutLines: string[];
+  let originalIsTTY: boolean | undefined;
+  let originalWrite: typeof process.stdout.write;
+
+  beforeEach(() => {
+    stdoutLines = [];
+    originalIsTTY = process.stdout.isTTY;
+    originalWrite = process.stdout.write.bind(process.stdout);
+    Object.defineProperty(process.stdout, "isTTY", {
+      configurable: true,
+      value: false,
+    });
+    process.stdout.write = ((chunk: string | Uint8Array) => {
+      stdoutLines.push(String(chunk));
+      return true;
+    }) as typeof process.stdout.write;
+  });
+
+  afterEach(() => {
+    Object.defineProperty(process.stdout, "isTTY", {
+      configurable: true,
+      value: originalIsTTY,
+    });
+    process.stdout.write = originalWrite;
+  });
+
+  it("prints only the final status line when stdout is not a TTY", () => {
+    const display = new ParallelScenarioDisplay();
+    const result = stubResult({ scenario: "login-admin", status: "pass" });
+
+    display.start("login-admin");
+    display.finish("login-admin", result, ["[login-admin] hook log"], () => {});
+
+    assert.equal(stdoutLines.join(""), "[login-admin] hook log\n[login-admin] passed\n");
+  });
+
+  it("prints failure reason via callback after the final status line", () => {
+    const display = new ParallelScenarioDisplay();
+    const result = stubResult({
+      scenario: "checkout",
+      status: "fail",
+      verdict: {
+        status: "fail",
+        summary: "Cart empty",
+        checkpoints: [],
+      },
+    });
+    const failureLines: string[] = [];
+
+    display.finish("checkout", result, [], (r) => {
+      failureLines.push(r.verdict?.summary ?? "");
+    });
+
+    assert.match(stdoutLines.at(-1) ?? "", /checkout.*fail/);
+    assert.deepEqual(failureLines, ["Cart empty"]);
+  });
+});

src/cli/parallel-display.ts

@@ -0,0 +1,50 @@
+import chalk from "chalk";
+import ora, { type Ora } from "ora";
+import type { ScenarioResult } from "../types/verdict.js";
+
+export type ScenarioFailureLogger = (result: ScenarioResult) => void;
+
+/** In-place status lines for parallel scenario workers. */
+export class ParallelScenarioDisplay {
+  private readonly spinners = new Map<string, Ora>();
+  private readonly isTTY = Boolean(process.stdout.isTTY);
+
+  start(name: string): void {
+    if (!this.isTTY) return;
+    const spinner = ora({ text: `[${name}] running...` }).start();
+    this.spinners.set(name, spinner);
+  }
+
+  finish(
+    name: string,
+    result: ScenarioResult,
+    bufferedLines: string[],
+    logFailureReason: ScenarioFailureLogger,
+  ): void {
+    for (const line of bufferedLines) {
+      process.stdout.write(`${line}\n`);
+    }
+
+    if (result.status === "pass") {
+      const message = chalk.green(`[${name}] passed`);
+      const spinner = this.spinners.get(name);
+      if (spinner) {
+        spinner.succeed(message);
+        this.spinners.delete(name);
+      } else {
+        console.log(message);
+      }
+      return;
+    }
+
+    const message = chalk.red(`[${name}] ${result.status}`);
+    const spinner = this.spinners.get(name);
+    if (spinner) {
+      spinner.fail(message);
+      this.spinners.delete(name);
+    } else {
+      console.log(message);
+    }
+    logFailureReason(result);
+  }
+}

src/cli/run.ts

@@ -72,6 +72,7 @@ import {
   isScenarioRetryAllowed,
 } from "../healing/classify.js";
 import { spawnScenarioWorker, cleanupRunDirHeartbeats } from "./subprocess.js";
+import { ParallelScenarioDisplay } from "./parallel-display.js";
 import {
   buildReport,
   createRunId,
@@ -128,21 +129,27 @@ function logScenarioFailureReason(result: ScenarioResult): void {
   }
 }
 
-function logRunSummary(report: {
-  results: ScenarioResult[];
-  summary: {
-    total: number;
-    passed: number;
-    failed: number;
-    errors: number;
-    skipped: number;
-  };
-}): void {
+function logRunSummary(
+  report: {
+    results: ScenarioResult[];
+    summary: {
+      total: number;
+      passed: number;
+      failed: number;
+      errors: number;
+      skipped: number;
+    };
+  },
+  options?: { compact?: boolean },
+): void {
   if (report.results.length === 0) {
     return;
   }
   console.log("\nRun summary:");
   for (const result of report.results) {
+    if (options?.compact && result.status === "pass") {
+      continue;
+    }
     const duration = `${(result.durationMs / 1000).toFixed(1)}s`;
     if (result.status === "pass") {
       console.log(chalk.green(`  ✓ ${result.scenario} — pass (${duration})`));
@@ -679,25 +686,22 @@ export async function executeRun(
       workerHeartbeatIntervalMs: config.agent.workerHeartbeatIntervalMs,
     };
 
+    const display = new ParallelScenarioDisplay();
+
     const partial = await mapWithConcurrency(
       selectedSummaries,
       parallel,
       async (summary) => {
         const name = summary.frontmatter.name;
-        console.log(`[${name}] running...`);
-        const result = await spawnScenarioWorker({
+        display.start(name);
+        const { result, bufferedLines } = await spawnScenarioWorker({
           scenarioFilePath: summary.filePath,
           scenarioName: name,
           runDir,
           cwd,
           options: workerOptions,
         });
-        if (result.status === "pass") {
-          console.log(chalk.green(`[${name}] passed`));
-        } else {
-          console.log(chalk.red(`[${name}] ${result.status}`));
-          logScenarioFailureReason(result);
-        }
+        display.finish(name, result, bufferedLines, logScenarioFailureReason);
         return result;
       },
       {
@@ -789,7 +793,7 @@ export async function executeRun(
   const report = buildReport(runId, startedAt, results);
   writeReport(runDir, report, redactor);
 
-  logRunSummary(report);
+  logRunSummary(report, { compact: parallel !== undefined });
 
   const reportPath = finalizeRunReport(runDir, zipDestination);
   console.log(`\nReport: ${reportPath}`);

src/cli/subprocess.ts

@@ -146,9 +146,15 @@ function readWorkerResult(
   return JSON.parse(raw) as ScenarioResult;
 }
 
+export interface ScenarioWorkerOutcome {
+  result: ScenarioResult;
+  /** Prefixed worker log lines, buffered until the scenario completes. */
+  bufferedLines: string[];
+}
+
 export async function spawnScenarioWorker(
   request: ScenarioWorkerRequest,
-): Promise<ScenarioResult> {
+): Promise<ScenarioWorkerOutcome> {
   const { command, baseArgs: _baseArgs } = resolveCliInvocation();
   const args = buildWorkerArgs(request);
   const name = request.scenarioName;
@@ -203,15 +209,17 @@ export async function spawnScenarioWorker(
     heartbeatWatchdogs.set(child.pid!, watchTimer);
     heartbeatFilePaths.set(child.pid!, heartbeatFile);
 
+    const bufferedLines: string[] = [];
+
     child.stdout?.on("data", (chunk: Buffer) => {
       for (const line of chunk.toString().split("\n")) {
-        if (line.trim()) process.stdout.write(`[${name}] ${line}\n`);
+        if (line.trim()) bufferedLines.push(`[${name}] ${line}`);
       }
     });
 
     child.stderr?.on("data", (chunk: Buffer) => {
       for (const line of chunk.toString().split("\n")) {
-        if (line.trim()) process.stderr.write(`[${name}] ${line}\n`);
+        if (line.trim()) bufferedLines.push(`[${name}] ${line}`);
       }
     });
 
@@ -246,7 +254,7 @@ export async function spawnScenarioWorker(
       }
       try {
         const result = readWorkerResult(request.runDir, name);
-        resolve(result);
+        resolve({ result, bufferedLines });
       } catch (err) {
         reject(
           new Error(