fix byte duplication when a chunk splits a character before a redaction

Once a binary chunk ends mid-character the streaming decoder holds its tail
bytes, but the raw chunk (partial bytes included) was still passed through.
If a later chunk then got rewritten (redaction or holdback), re-encoding the
decoded text emitted those held bytes a second time. Now the response switches
to emitting re-encoded text from the first mid-character split onward.

Also logs a debug line when a compressed end() chunk fails to decode instead
of silently skipping the scan.

Author: theoephraim

packages/varlock/src/runtime/patch-server-response.ts

@@ -63,6 +63,10 @@ type ScanState = {
   /** streaming decoder, so a multi-byte character split across chunks doesn't decode to garbage
    * (created lazily - responses that never hit the binary path don't need one) */
   decoder: TextDecoder | undefined,
+  /** set once a binary chunk ends mid-character: the decoder is holding its tail bytes, so raw
+   * chunks no longer line up with the decoded text and all further output must be re-encoded
+   * from it (otherwise the held bytes would go out twice if a later chunk gets rewritten) */
+  reEncode: boolean,
   zlibChunks: Array<Buffer>,
   /** streaming decoder for decompressed deltas, which may end inside a multi-byte character */
   decompressedDecoder: TextDecoder | undefined,
@@ -76,6 +80,7 @@ function getScanState(res: any): ScanState {
     pending: '',
     carry: '',
     decoder: undefined,
+    reEncode: false,
     zlibChunks: [],
     decompressedDecoder: undefined,
     decompressedLength: 0,
@@ -100,6 +105,25 @@ function decodeDecompressedDelta(state: ScanState, decompressed: Buffer, isFinal
   return state.decompressedDecoder.decode(delta, { stream: !isFinal });
 }
 
+/**
+ * Number of bytes at the end of `chunk` that are the start of an incomplete UTF-8 character
+ * (0 when the chunk ends on a character boundary). These are the bytes a streaming
+ * TextDecoder holds back until the rest of the character arrives.
+ */
+function incompleteTrailingUtf8Bytes(chunk: Uint8Array): number {
+  /* eslint-disable no-bitwise */
+  for (let i = 1; i <= 3 && i <= chunk.length; i++) {
+    const byte = chunk[chunk.length - i];
+    if ((byte & 0b11000000) === 0b10000000) continue; // continuation byte - keep looking for the lead
+    if ((byte & 0b11100000) === 0b11000000) return i < 2 ? i : 0; // 2-byte lead
+    if ((byte & 0b11110000) === 0b11100000) return i < 3 ? i : 0; // 3-byte lead
+    if ((byte & 0b11111000) === 0b11110000) return i < 4 ? i : 0; // 4-byte lead
+    return 0; // ascii or invalid lead - the decoder holds nothing
+  }
+  return 0;
+  /* eslint-enable no-bitwise */
+}
+
 function clearPendingFlush(state: ScanState) {
   if (state.flushTimer !== undefined) {
     clearTimeout(state.flushTimer);
@@ -230,6 +254,7 @@ export function patchGlobalServerResponse(opts?: {
     } else if (!compressionType) {
       chunkType = 'encoded';
       chunkStr = decodeChunk(state, rawChunk);
+      if (!state.reEncode && incompleteTrailingUtf8Bytes(rawChunk)) state.reEncode = true;
     } else {
       const decompress = getDecompressor(String(compressionType).toLowerCase());
       if (decompress) {
@@ -287,6 +312,11 @@ export function patchGlobalServerResponse(opts?: {
             return true;
           }
           args[0] = chunkType === 'encoded' ? new TextEncoder().encode(emit) : emit;
+        } else if (chunkType === 'encoded' && state.reEncode && emit) {
+          // the decoder is holding tail bytes of a split character, so the raw chunk no
+          // longer lines up with the decoded text - emit re-encoded text instead of the
+          // raw bytes to keep the outgoing byte stream consistent
+          args[0] = new TextEncoder().encode(emit);
         }
       }
     }
@@ -319,7 +349,8 @@ export function patchGlobalServerResponse(opts?: {
         try {
           decompressed = decompress(Buffer.concat(state.zlibChunks));
         } catch (err) {
-          // stream didn't decode, nothing more we can do at this point
+          // stream didn't decode, nothing more we can do at this point (fails open)
+          debug(`⚠️ leak scan skipped - compressed response did not decode at end() (${compressionType})`);
         }
       }
       if (decompressed !== undefined) {
@@ -367,6 +398,10 @@ export function patchGlobalServerResponse(opts?: {
         if (!this.headersSent && this.getHeader('content-length') !== undefined) {
           this.setHeader('content-length', Buffer.byteLength(emit));
         }
+      } else if (isBinaryChunk && state.reEncode && emit) {
+        // raw bytes stopped lining up with the decoded text earlier in the response
+        // (a chunk ended mid-character), so the final chunk is re-encoded too
+        args[0] = new TextEncoder().encode(emit);
       }
     }
 

packages/varlock/src/runtime/test/patch-server-response-redact.test.ts

@@ -73,6 +73,13 @@ beforeAll(async () => {
       // ends mid-lookalike but never completes the secret - must arrive intact
       res.write(`<html>${SECRET.slice(0, 12)}`);
       res.end('-not-a-secret</html>');
+    } else if (req.url === '/multibyte-split-then-redact') {
+      // first chunk ends mid-character (its lead byte must not go out raw), and the
+      // chunk completing it contains a secret, so the rest gets re-encoded on redaction
+      const buf = Buffer.from(`<html>é leak: ${SECRET}</html>`);
+      const mid = buf.indexOf(0xc3) + 1; // one byte into the 2-byte é
+      res.write(buf.subarray(0, mid));
+      res.end(buf.subarray(mid));
     } else if (req.url === '/gzip-leak') {
       res.setHeader('content-encoding', 'gzip');
       try {
@@ -147,6 +154,15 @@ describe('patchGlobalServerResponse with redactInsteadOfThrow', () => {
     expect(resp.headers.get('transfer-encoding')).toBe('chunked');
   });
 
+  it('does not duplicate bytes when a chunk splits a character before a redaction', async () => {
+    const body = await (await fetch(`${baseUrl}/multibyte-split-then-redact`)).text();
+    expect(body).not.toContain(SECRET);
+    expect(body).toContain('▒');
+    // a duplicated lead byte would decode as a replacement char before the é
+    expect(body.startsWith('<html>é leak: ')).toBe(true);
+    expect(body).not.toContain('�');
+  });
+
   it('leaves text that only looks like the start of a secret intact', async () => {
     const body = await (await fetch(`${baseUrl}/partial-lookalike`)).text();
     expect(body).toBe(`<html>${SECRET.slice(0, 12)}-not-a-secret</html>`);