fix(stream): preserve multi-byte UTF-8 split across chunk boundaries (#2091)

## Summary

`BraintrustStream` corrupted multi-byte UTF-8 characters when an SSE
chunk boundary landed mid-character (e.g. Cloudflare `workerd` flushing
a small first chunk), emitting `U+FFFD` instead of the decoded
character.

Root cause: `btStreamParser` called `decoder.decode(chunk)` without `{
stream: true }`, so the `TextDecoder` flushed its internal buffer on
every chunk and dropped any trailing partial UTF-8 sequence.

## Fix

Pass `{ stream: true }` when decoding each `Uint8Array` chunk so partial
sequences are held until the next chunk arrives, and flush the decoder
once in the transform's `flush` handler to handle a truly-truncated
tail. No behavior change for `string` or `BraintrustStreamChunk` inputs,
and single-chunk UTF-8 input is byte-identical to before.

## Tests

Added a regression test in `js/src/functions/stream.test.ts` that splits
an SSE `text_delta` event inside the 3-byte sequence for `U+201C` and
asserts a clean round-trip with no `U+FFFD`. Fails on `main`, passes
here.

---------

Co-authored-by: Luca Forstner <luca.forstner@gmail.com>

Author: stretpjc

.changeset/bright-falcons-carry.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+fix(stream): Preserve multi-byte UTF-8 split across chunk boundaries

js/src/functions/stream.test.ts

@@ -55,6 +55,38 @@ test("final value passthrough", async () => {
   }
 });
 
+test("preserves multi-byte UTF-8 characters split across chunk boundaries", async () => {
+  // Regression test: btStreamParser previously decoded each Uint8Array chunk
+  // without `{ stream: true }`, so a partial UTF-8 sequence at the end of a
+  // chunk was flushed as U+FFFD instead of being held until the next chunk.
+  const encoder = new TextEncoder();
+  const payload = "“Hello, world.”"; // leading “ is U+201C -> bytes E2 80 9C
+  const sseBytes = encoder.encode(
+    `event: text_delta\ndata: ${JSON.stringify(payload)}\n\n`,
+  );
+
+  // Split inside the first multi-byte character: after E2 80, before 9C.
+  const quoteStart = sseBytes.indexOf(0xe2);
+  const splitAt = quoteStart + 2;
+
+  const inputStream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(sseBytes.subarray(0, splitAt));
+      controller.enqueue(sseBytes.subarray(splitAt));
+      controller.close();
+    },
+  });
+
+  const stream = new BraintrustStream(inputStream);
+  let out = "";
+  for await (const chunk of stream) {
+    if (chunk.type === "text_delta") out += chunk.data;
+  }
+
+  expect(out).toBe(payload);
+  expect(out).not.toContain("\uFFFD");
+});
+
 test("final value passthrough with abort", async () => {
   const inputStream = new ReadableStream({
     start(controller) {},

js/src/functions/stream.ts

@@ -283,14 +283,18 @@ function btStreamParser() {
     },
     async transform(chunk, controller) {
       if (chunk instanceof Uint8Array) {
-        parser.feed(decoder.decode(chunk));
+        parser.feed(decoder.decode(chunk, { stream: true }));
       } else if (typeof chunk === "string") {
         parser.feed(chunk);
       } else {
         controller.enqueue(chunk);
       }
     },
     async flush(controller) {
+      const tail = decoder.decode();
+      if (tail) {
+        parser.feed(tail);
+      }
       controller.terminate();
     },
   });