fix(reliability): apply the relay-field policy to the packaged driver too (#4710)

Ring 2 has never passed. It was added on 2026-07-30 (0f68fe6, #4606),
after that day's last green release run, and every release since — five
nightlies plus v0.7.0-rc.33 — has failed on all three OS legs with
`streamShape golden mismatch (11); diverges from kernel`.

Nothing was wrong with the packaged backend. `ws-server.ts` already had
`stripRelayOnlyFields`: the WS relay backfills run identity onto frames
the kernel actor emits bare, so the driver cancels that enrichment before
recording and the two surfaces stay comparable. `protocol/messages.ts`
specifies this — job_id is "Stamped downstream by the relay ... not by the
kernel actor", and generation_complete is emitted by the actor as "a BARE
event" with job_id and index "stamped DOWNSTREAM by the relay".

`packaged.ts` is the same kind of relay and never got the rule. It records
raw frames, so Ring 2 diffed a relay stream against a bare oracle and
failed on exactly the fields the protocol says only the relay carries.

A rule only one of two relay surfaces applies is not a policy, it is a
discrepancy, so the sets move to `relay-fields.ts` and both drivers import
them. The packaged driver also drops `sdk_execution_target` — sent once per
socket at upgrade time by the production Fastify plugin, before any
workflow exists, so the oracle has nothing to compare it against.

Deliberately not changed: the journey manifest still declares only
["kernel", "ws-server"]. `PackagedDriver` has no `supports()`, so listing
"packaged" would make a bare `nodetool reliability run` fail whenever no
bundle is staged (compare.ts records an unstaged run as ok: false).

Refs run 30927453380

Author: georgi

reliability/harness/src/drivers/packaged.ts

@@ -26,6 +26,7 @@ import { packWebSocketMessage, unpackWebSocketMessage } from "@nodetool-ai/webso
 import type { Journey, JourneyInteraction } from "../core/journey.js";
 import { makeFrame, type RunFrame, type RunRecord } from "../core/record.js";
 import { AnchorWaiter } from "./anchors.js";
+import { isConnectionControlMessage, stripRelayOnlyFields } from "./relay-fields.js";
 import { findRepoRoot } from "./repo-root.js";
 import type { RunDriver } from "./types.js";
 
@@ -235,9 +236,16 @@ export class PackagedDriver implements RunDriver {
     const ws = new WebSocket(`ws://127.0.0.1:${server.port}/ws`);
 
     ws.on("message", (data: Buffer, isBinary: boolean) => {
-      const message = (
+      const raw = (
         isBinary ? unpackWebSocketMessage(data) : JSON.parse(data.toString("utf8"))
       ) as Record<string, unknown>;
+      // The packaged server is a relay, exactly like `ws-server.ts`'s surface:
+      // it boots the real `server.mjs`, whose Fastify plugin announces the
+      // connection and whose `UnifiedWebSocketRunner` backfills run identity
+      // onto frames the kernel oracle emits bare. Both are cancelled here so
+      // the packaged stream is comparable to the oracle frame-for-frame.
+      if (isConnectionControlMessage(raw)) return;
+      const message = stripRelayOnlyFields(raw);
       waiter.notify(message);
       if (typeof message["job_id"] === "string") {
         jobId = message["job_id"] as string;

reliability/harness/src/drivers/relay-fields.ts

@@ -0,0 +1,86 @@
+/**
+ * Relay-only field policy, shared by every driver whose surface speaks the
+ * WS wire protocol (`ws-server.ts`, `packaged.ts`).
+ *
+ * Both of those surfaces run the same relay — `UnifiedWebSocketRunner` — so
+ * both receive the same downstream enrichment the kernel oracle's raw stream
+ * never carries. `packages/protocol/src/messages.ts` specifies this: `job_id`
+ * is "Stamped downstream by the relay (the unified websocket runner and the
+ * browser runner), not by the kernel actor, so it is optional on the wire",
+ * and `generation_complete` is emitted by the actor as "a BARE event" with
+ * `job_id` and `index` "stamped DOWNSTREAM by the relay".
+ *
+ * Lives here rather than in each driver because a rule that only one relay
+ * surface applies is not a policy, it is a discrepancy — `packaged.ts` not
+ * having it is exactly what made the Ring 2 release gate unpassable.
+ */
+
+/**
+ * Message `type`s the relay backfills `job_id` onto uniformly (its "outbound
+ * spread", `unified-websocket-runner.ts`) that the kernel's own emissions for
+ * these types (`runner.ts`/`actor.ts`) never carry at all. Scoped per-type
+ * (not a blanket drop) because `job_update` DOES set both, and `edge_update`
+ * DOES set `job_id`, on the raw kernel stream too — dropping either there
+ * would hide a real regression instead of a harmless relay addition. Kept as
+ * a driver-local rule (not a `normalize.ts` field list) because
+ * `normalize.ts`'s generic per-key visitor has other callers — including its
+ * own unit tests — that legitimately expect `job_id`/`workflow_id` preserved
+ * on a `node_update`. The relay drivers are the only place that actually
+ * knows which of these are relay-only enrichment on their wire protocol.
+ */
+export const RELAY_ONLY_JOB_ID_TYPES: ReadonlySet<string> = new Set([
+  "node_update",
+  "generation_complete",
+  "output_update",
+  // `llm_call` (task D1, journey 8 — the first journey to run a real LLM
+  // node): the relay backfills `job_id`/`workflow_id` here the same way it
+  // does for `node_update`; the kernel's own `llm_call` emission
+  // (`BaseProvider`'s tracing helper) never carries either.
+  "llm_call"
+]);
+
+/** As {@link RELAY_ONLY_JOB_ID_TYPES}, for `workflow_id`. Includes
+ * `edge_update` — the kernel stamps `job_id` there but never `workflow_id`. */
+export const RELAY_ONLY_WORKFLOW_ID_TYPES: ReadonlySet<string> = new Set([
+  "node_update",
+  "generation_complete",
+  "output_update",
+  "edge_update",
+  "llm_call"
+]);
+
+/**
+ * Message `type`s that exist only as connection-scoped transport control and
+ * are never part of a run's stream. `sdk_execution_target` is sent once per
+ * socket, at upgrade time and BEFORE any workflow is submitted, by the
+ * production Fastify plugin when an SDK live-runner registry is wired
+ * (`packages/websocket/src/plugins/websocket.ts`). The kernel oracle has no
+ * connection to announce, so there is nothing to compare it against.
+ */
+export const CONNECTION_CONTROL_MESSAGE_TYPES: ReadonlySet<string> = new Set([
+  "sdk_execution_target"
+]);
+
+/** True when `message` is transport control rather than a run frame — callers
+ * drop these before recording (see {@link CONNECTION_CONTROL_MESSAGE_TYPES}). */
+export function isConnectionControlMessage(message: Record<string, unknown>): boolean {
+  return CONNECTION_CONTROL_MESSAGE_TYPES.has(String(message["type"]));
+}
+
+/** Strips fields the relay adds that the kernel driver's raw stream never has,
+ * so the two are comparable frame-for-frame (see
+ * {@link RELAY_ONLY_JOB_ID_TYPES}/{@link RELAY_ONLY_WORKFLOW_ID_TYPES}).
+ * `generation_complete.index` is the same kind of addition — an
+ * arrival-order stamp (`unified-websocket-runner.ts`'s
+ * `generationIndexByNode`) the kernel's own `generation_complete` emission
+ * never sets. */
+export function stripRelayOnlyFields(
+  message: Record<string, unknown>
+): Record<string, unknown> {
+  const type = String(message["type"]);
+  const rest = { ...message };
+  if (RELAY_ONLY_JOB_ID_TYPES.has(type)) delete rest["job_id"];
+  if (RELAY_ONLY_WORKFLOW_ID_TYPES.has(type)) delete rest["workflow_id"];
+  if (type === "generation_complete") delete rest["index"];
+  return rest;
+}

reliability/harness/src/drivers/ws-server.ts

@@ -35,6 +35,7 @@ import {
 import { AnchorWaiter } from "./anchors.js";
 import { registerFixtureNodes } from "./fixture-nodes.js";
 import { journeyPythonBridgeFactory } from "./python-bridge.js";
+import { stripRelayOnlyFields } from "./relay-fields.js";
 import type { RunDriver } from "./types.js";
 import { maybeFrontWithProxy, type WsProxyFrontEnd } from "../faults/ws-proxy.js";
 
@@ -45,53 +46,6 @@ const TERMINAL_JOB_STATUSES = new Set([
   "suspended"
 ]);
 
-/**
- * Message `type`s the ws-server relay backfills `job_id`/`workflow_id` onto
- * uniformly (its "outbound spread") that the kernel's own emissions for
- * these types (`runner.ts`/`actor.ts`) never carry at all. Scoped per-type
- * (not a blanket drop) because `job_update` DOES set both, and `edge_update`
- * DOES set `job_id`, on the raw kernel stream too — dropping either there
- * would hide a real regression instead of a harmless relay addition. Kept as
- * a driver-local rule (not a `normalize.ts` field list) because
- * `normalize.ts`'s generic per-key visitor has other callers — including its
- * own unit tests — that legitimately expect `job_id`/`workflow_id`
- * preserved on a `node_update`. This driver is the one place that actually
- * knows which of these are relay-only enrichment on ITS wire protocol.
- */
-const RELAY_ONLY_JOB_ID_TYPES = new Set([
-  "node_update",
-  "generation_complete",
-  "output_update",
-  // `llm_call` (task D1, journey 8 — the first journey to run a real LLM
-  // node): the relay backfills `job_id`/`workflow_id` here the same way it
-  // does for `node_update`; the kernel's own `llm_call` emission
-  // (`BaseProvider`'s tracing helper) never carries either.
-  "llm_call"
-]);
-const RELAY_ONLY_WORKFLOW_ID_TYPES = new Set([
-  "node_update",
-  "generation_complete",
-  "output_update",
-  "edge_update",
-  "llm_call"
-]);
-
-/** Strips fields this driver's relay adds that the kernel driver's raw
- * stream never has, so the two are comparable frame-for-frame (see
- * {@link RELAY_ONLY_JOB_ID_TYPES}/{@link RELAY_ONLY_WORKFLOW_ID_TYPES}).
- * `generation_complete.index` is the same kind of addition — an
- * arrival-order stamp (`unified-websocket-runner.ts`'s
- * `generationIndexByNode`) the kernel's own `generation_complete` emission
- * never sets. */
-function stripRelayOnlyFields(message: Record<string, unknown>): Record<string, unknown> {
-  const type = String(message["type"]);
-  const rest = { ...message };
-  if (RELAY_ONLY_JOB_ID_TYPES.has(type)) delete rest["job_id"];
-  if (RELAY_ONLY_WORKFLOW_ID_TYPES.has(type)) delete rest["workflow_id"];
-  if (type === "generation_complete") delete rest["index"];
-  return rest;
-}
-
 /** How long to let the server's post-disconnect cleanup settle before
  * recording the slot counters as final. Cleanup is asynchronous (the runner
  * finishes streaming, then frees the slot), so a snapshot taken the instant

reliability/harness/tests/relay-fields.test.ts

@@ -0,0 +1,169 @@
+import { describe, expect, it } from "vitest";
+import {
+  isConnectionControlMessage,
+  stripRelayOnlyFields
+} from "../src/drivers/relay-fields.js";
+
+describe("stripRelayOnlyFields", () => {
+  it("drops job_id on the types the relay backfills it onto", () => {
+    for (const type of ["node_update", "generation_complete", "output_update", "llm_call"]) {
+      expect(stripRelayOnlyFields({ type, job_id: "j-1" })).toEqual({ type });
+    }
+  });
+
+  it("keeps job_id on job_update and edge_update — the kernel stamps those itself", () => {
+    // Dropping these would hide a real regression (the kernel silently
+    // ceasing to stamp run identity) instead of a harmless relay addition.
+    expect(stripRelayOnlyFields({ type: "job_update", job_id: "j-1" })).toEqual({
+      type: "job_update",
+      job_id: "j-1"
+    });
+    expect(stripRelayOnlyFields({ type: "edge_update", job_id: "j-1" })).toEqual({
+      type: "edge_update",
+      job_id: "j-1"
+    });
+  });
+
+  it("drops workflow_id on relay-enriched types, including edge_update", () => {
+    for (const type of [
+      "node_update",
+      "generation_complete",
+      "output_update",
+      "edge_update",
+      "llm_call"
+    ]) {
+      expect(stripRelayOnlyFields({ type, workflow_id: "w-1" })).toEqual({ type });
+    }
+  });
+
+  it("keeps workflow_id on job_update", () => {
+    expect(stripRelayOnlyFields({ type: "job_update", workflow_id: "w-1" })).toEqual({
+      type: "job_update",
+      workflow_id: "w-1"
+    });
+  });
+
+  it("drops generation_complete.index — an arrival-order stamp the kernel never sets", () => {
+    expect(stripRelayOnlyFields({ type: "generation_complete", index: 0 })).toEqual({
+      type: "generation_complete"
+    });
+  });
+
+  it("does not mutate its argument", () => {
+    const message = { type: "node_update", job_id: "j-1" };
+    stripRelayOnlyFields(message);
+    expect(message).toEqual({ type: "node_update", job_id: "j-1" });
+  });
+});
+
+describe("isConnectionControlMessage", () => {
+  it("recognizes sdk_execution_target", () => {
+    expect(isConnectionControlMessage({ type: "sdk_execution_target", runner_id: "r-1" })).toBe(
+      true
+    );
+  });
+
+  it("leaves run frames alone", () => {
+    for (const type of ["job_update", "node_update", "edge_update", "output_update"]) {
+      expect(isConnectionControlMessage({ type })).toBe(false);
+    }
+  });
+});
+
+/**
+ * Regression guard for the Ring 2 release gate (run 30927453380, tag
+ * v0.7.0-rc.33): the packaged surface is a relay just like ws-server, but
+ * `packaged.ts` never applied the relay-field policy, so every packaged
+ * release run diffed relay-stamped frames against the bare kernel oracle and
+ * failed. These are the exact frames from that failure's log.
+ */
+describe("the frames that failed Ring 2 on v0.7.0-rc.33", () => {
+  it("reduces each one to the kernel's bare shape", () => {
+    expect(
+      stripRelayOnlyFields({
+        type: "edge_update",
+        counter: 1,
+        edge_id: "e-0",
+        job_id: "j-0",
+        status: "active",
+        workflow_id: "w-0"
+      })
+    ).toEqual({ type: "edge_update", counter: 1, edge_id: "e-0", job_id: "j-0", status: "active" });
+
+    expect(
+      stripRelayOnlyFields({
+        type: "node_update",
+        error: null,
+        job_id: "j-0",
+        node_id: "n-1",
+        node_name: "out1",
+        node_type: "nodetool.output.Output",
+        properties: { name: "result" },
+        provider_cost: null,
+        result: null,
+        status: "running"
+      })
+    ).toEqual({
+      type: "node_update",
+      error: null,
+      node_id: "n-1",
+      node_name: "out1",
+      node_type: "nodetool.output.Output",
+      properties: { name: "result" },
+      provider_cost: null,
+      result: null,
+      status: "running"
+    });
+
+    expect(
+      stripRelayOnlyFields({
+        type: "output_update",
+        disposition: "append",
+        job_id: "j-0",
+        metadata: {},
+        node_id: "n-1",
+        node_name: "out1",
+        output_name: "result",
+        output_type: "any",
+        value: "HELLO RELIABILITY",
+        workflow_id: "w-0"
+      })
+    ).toEqual({
+      type: "output_update",
+      disposition: "append",
+      metadata: {},
+      node_id: "n-1",
+      node_name: "out1",
+      output_name: "result",
+      output_type: "any",
+      value: "HELLO RELIABILITY"
+    });
+
+    expect(
+      stripRelayOnlyFields({
+        type: "generation_complete",
+        index: 0,
+        job_id: "j-0",
+        node_id: "n-0",
+        node_name: "upper1",
+        node_type: "nodetool.text.ToUppercase",
+        outputs: { output: "HELLO RELIABILITY" },
+        properties: { text: "hello reliability" }
+      })
+    ).toEqual({
+      type: "generation_complete",
+      node_id: "n-0",
+      node_name: "upper1",
+      node_type: "nodetool.text.ToUppercase",
+      outputs: { output: "HELLO RELIABILITY" },
+      properties: { text: "hello reliability" }
+    });
+
+    expect(
+      isConnectionControlMessage({
+        type: "sdk_execution_target",
+        runner_id: "e81fd54b-6544-4de9-a37c-18b6d843d440"
+      })
+    ).toBe(true);
+  });
+});