Added simplified replacement + fast check tests.

Author: uhop

AGENTS.md

@@ -164,6 +164,7 @@ test('example', async t => {
 - Test files use `tape-six`: `.js` for runtime tests, `.ts` for typed tests.
 - Test file naming convention: `test-*.js` (or `test-types-*.ts`) in `tests/`.
 - Tests are configured in `package.json` under the `"tape6"` section.
+- Property-based tests live in `tests/node/test-property-parser.js` (`tape-six-fast-check`, `t.prop()` over fast-check arbitraries): chunk-boundary invariance for the JSON and JSONC parsers and the two round-trips. A new streaming invariant belongs there, not in an example-based file.
 - Test files should be directly executable: `node tests/test-foo.js`.
 
 ## Token protocol

llms-full.txt

@@ -433,18 +433,17 @@ Replaces matching subobjects with a replacement value. Needs packed keys (`keyVa
 
 Extra option:
 - `replacement` — the replacement:
-  - **function** `(stack, chunk, options) => tokens` — dynamic replacement; return `none` to remove the value.
-  - **token** — a single static token, such as `{name: 'nullValue', value: null}`.
-  - **array** — array of tokens to insert.
-  - Anything else, including `null`, removes the value.
+  - **any JSON-compatible value** — a number, string, boolean, `null`, array, or plain object — disassembled into tokens once and substituted as that JSON value (shaped by the same packing/streaming options as the parser).
+  - **token** / **array of tokens** — substituted verbatim; an empty array removes the value (compatibility), an empty JSON array is its two tokens.
+  - **function** `(stack, chunk, options) => value` — dynamic; the result is interpreted as above; return `none` to remove the value.
   - Default: `none` (removes the value, replaced by nothing).
 
 ```js
 import {replace} from 'stream-json/filters/replace.js';
 import {stringer} from 'stream-json/stringer.js';
 
 // Replace 'extra' with null
-chain([parser(), replace({filter: /^\d+\.extra\b/, replacement: [{name: 'nullValue', value: null}]}), stringer()]);
+chain([parser(), replace({filter: /^\d+\.extra\b/, replacement: null}), stringer()]);
 
 // Replace with custom function
 chain([parser(), replace({

llms.txt

@@ -123,7 +123,7 @@ e.addEventListener('keyValue', ev => console.log(ev.detail));
 All filters accept `{filter, pathSeparator, once, maxDepth, streamKeys}` options. Key-based paths and parent recreation need packed keys from upstream (the parser's default). Replayed parent keys are always packed; their streamed form mirrors upstream unless `streamKeys` says otherwise. `packKeys` on a filter is a deprecated no-op. `filter` can be a string, RegExp, or `(stack, chunk) => boolean`. `maxDepth` caps the JSON nesting depth a filter evaluates (default `1024`; a deeper token throws a `RangeError`, `Infinity` disables the limit).
 
 - **`pick(options)`** — passes only matching subobjects, discards the rest.
-- **`replace(options)`** — replaces matching subobjects. Extra option: `replacement` (function, token, or array of tokens).
+- **`replace(options)`** — replaces matching subobjects. Extra option: `replacement` (any JSON-compatible value — disassembled for you — a token or token array, or a function returning one).
 - **`ignore(options)`** — removes matching subobjects completely.
 - **`filter(options)`** — keeps matching subobjects preserving surrounding structure.
 

package-lock.json

@@ -72,9 +72,11 @@
   },
   "devDependencies": {
     "@types/node": "^26.1.2",
+    "fast-check": "^4.9.0",
     "nano-benchmark": "^1.2.0",
     "prettier": "^3.9.6",
     "tape-six": "^1.16.2",
+    "tape-six-fast-check": "^1.0.0",
     "tape-six-proc": "^1.3.1",
     "typescript": "^7.0.2"
   },

package.json

@@ -20,22 +20,23 @@ declare namespace replace {
   /** Options for `replace`, extending filter base options with a replacement value. */
   export interface ReplaceOptions extends filterBase.FilterBaseOptions {
     /**
-     * What to substitute for matched subobjects.
-     * - **function** — called with `(stack, chunk, options)`; returns tokens, or `none` to remove the value.
-     * - **Token[]** / **Many<Token>** — a static array of replacement tokens.
-     * - **Token** — a single static token, such as `{name: 'nullValue', value: null}`.
-     * - **`null`** — no replacement: the matched value is removed, like `ignore`.
-     * - Default: none (the matched value is removed).
+     * What to substitute for matched subobjects:
+     * - a **function** `(stack, chunk, options)` — called per match; its result is
+     *   interpreted like a static value below (`none` removes the value);
+     * - a **token**, a **token array**, or a `Many` of tokens — substituted verbatim;
+     * - **any other value** — a number, string, boolean, `null`, array, or plain
+     *   object — disassembled into tokens once and substituted as that JSON value,
+     *   shaped by the same packing/streaming options as the parser. An empty
+     *   array is an empty token list and removes the value (kept for
+     *   compatibility); an empty JSON array is `[{name: 'startArray'}, {name: 'endArray'}]`.
+     * - Default (option absent): none — the matched value is removed, like `ignore`.
      */
     replacement?:
-      | ((
-          stack: (string | number | null)[],
-          chunk: parser.Token,
-          options: filterBase.FilterBaseOptions
-        ) => parser.Token | parser.Token[] | Many<parser.Token> | typeof none)
+      | ((stack: (string | number | null)[], chunk: parser.Token, options: filterBase.FilterBaseOptions) => unknown)
       | parser.Token
       | parser.Token[]
       | Many<parser.Token>
+      | {}
       | null;
   }
   /** Creates a `parser() + replace()` pipeline as a flushable function. */

src/core/filters/replace.d.ts

@@ -3,20 +3,61 @@
 import {many, none, combineManyMut, getManyValues, isMany} from 'stream-chain/core';
 
 import {filterBase, makeStackDiffer} from './filter-base.js';
+import {disassembler} from '../disassembler.js';
+
+// an object with one of these names is a token; anything else is a plain value to disassemble
+const tokenNames = {
+  startObject: 1,
+  endObject: 1,
+  startArray: 1,
+  endArray: 1,
+  startKey: 1,
+  endKey: 1,
+  startString: 1,
+  endString: 1,
+  startNumber: 1,
+  endNumber: 1,
+  stringChunk: 1,
+  numberChunk: 1,
+  keyValue: 1,
+  stringValue: 1,
+  numberValue: 1,
+  nullValue: 1,
+  trueValue: 1,
+  falseValue: 1,
+  whitespace: 1,
+  comma: 1,
+  startComment: 1,
+  commentChunk: 1,
+  endComment: 1,
+  commentValue: 1
+};
+const isToken = value => value !== null && typeof value == 'object' && tokenNames[value.name] === 1;
+
 const defaultReplacement = () => none;
 
 const replace = options => {
+  const toTokens = disassembler(options); // shaped by the same packing/streaming options as the parser
+  const normalize = value => {
+    if (value === none) return none;
+    if (value !== null && typeof value == 'object') {
+      if (isMany(value)) return value;
+      if (Array.isArray(value)) {
+        if (value.every(isToken)) return many(value);
+      } else if (tokenNames[value.name] === 1) {
+        return value;
+      }
+    }
+    return many([...toTokens(value)]);
+  };
   let replacementValue = options?.replacement;
   /** @type {any} */
   let replacement = defaultReplacement;
-  switch (typeof replacementValue) {
-    case 'function':
-      replacement = replacementValue;
-      break;
-    case 'object':
-      if (Array.isArray(replacementValue)) replacementValue = many(replacementValue);
-      if (replacementValue) replacement = () => replacementValue;
-      break;
+  if (typeof replacementValue == 'function') {
+    replacement = (stack, chunk, options) => normalize(replacementValue(stack, chunk, options));
+  } else if (replacementValue !== undefined) {
+    replacementValue = normalize(replacementValue);
+    replacement = () => replacementValue;
   }
   const stackDiffer = makeStackDiffer();
   return filterBase({
@@ -25,7 +66,6 @@ const replace = options => {
     transition(stack, chunk, action, options) {
       if (action !== 'reject' && action !== 'reject-value') return stackDiffer(stack, chunk, options);
       let replacementTokens = replacement(stack, chunk, options);
-      if (Array.isArray(replacementTokens)) replacementTokens = many(replacementTokens);
       if (replacementTokens === none || (isMany(replacementTokens) && !getManyValues(replacementTokens).length)) return none;
       return combineManyMut(stackDiffer(stack, null, options), replacementTokens);
     }

src/core/filters/replace.js

@@ -0,0 +1,139 @@
+// Property-based tests (tape-six-fast-check): the streaming invariants that
+// example-based tests structurally miss — the same document split at arbitrary
+// chunk boundaries must yield the same tokens (GHSA-hqr4-qq8f-hg3x was exactly
+// this class), and the parser/assembler and disassembler/stringer pairs must
+// round-trip against JSON.parse / JSON.stringify.
+import test from 'tape-six';
+import fc from 'fast-check';
+import 'tape-six-fast-check';
+import chain from 'stream-chain';
+import {none} from 'stream-chain/core';
+import {Readable} from 'node:stream';
+
+import {parser} from '../../src/index.js';
+import {parser as jsoncParser} from '../../src/jsonc/parser.js';
+import Assembler from '../../src/assembler.js';
+import {disassembler} from '../../src/disassembler.js';
+import {stringer} from '../../src/stringer.js';
+
+const splitAt = (text, offsets) => {
+  const cuts = [...new Set(offsets.map(o => o % (text.length + 1)))].sort((a, b) => a - b);
+  const chunks = [];
+  let prev = 0;
+  for (const cut of cuts) {
+    if (cut > prev) chunks.push(text.slice(prev, cut));
+    prev = cut;
+  }
+  if (prev < text.length) chunks.push(text.slice(prev));
+  return chunks;
+};
+
+const tokensOf = (chunks, factory, options) =>
+  new Promise((resolve, reject) => {
+    const tokens = [],
+      pipeline = chain([Readable.from(chunks), factory(options)]);
+    pipeline.on('data', token => tokens.push(token));
+    pipeline.on('error', reject);
+    pipeline.on('end', () => resolve(tokens));
+  });
+
+// merge streamed pieces so two splits of the same text compare equal
+const coalesce = tokens => {
+  const out = [];
+  for (const token of tokens) {
+    const last = out[out.length - 1];
+    if (last && last.name === token.name && (token.name === 'stringChunk' || token.name === 'numberChunk' || token.name === 'commentChunk')) {
+      out[out.length - 1] = {name: last.name, value: last.value + token.value};
+    } else {
+      out.push(token);
+    }
+  }
+  return out.filter(token => token.name !== 'whitespace');
+};
+
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+const json = () => fc.jsonValue({maxDepth: 4});
+const offsets = () => fc.array(fc.nat(4096), {maxLength: 12});
+
+test('parser: chunk boundaries do not change the token stream', async t => {
+  await t.prop(
+    [json(), offsets()],
+    async (value, cuts) => {
+      const text = JSON.stringify(value);
+      return same(await tokensOf([text], parser, {streamValues: false}), await tokensOf(splitAt(text, cuts), parser, {streamValues: false}));
+    },
+    {numRuns: 100},
+    'packed tokens are split-invariant'
+  );
+  await t.prop(
+    [json(), offsets()],
+    async (value, cuts) => {
+      const text = JSON.stringify(value);
+      return same(coalesce(await tokensOf([text], parser)), coalesce(await tokensOf(splitAt(text, cuts), parser)));
+    },
+    {numRuns: 100},
+    'streamed chunks coalesce to the same tokens'
+  );
+});
+
+test('parser → assembler round-trips against JSON.parse', async t => {
+  await t.prop(
+    [json(), offsets()],
+    async (value, cuts) => {
+      const asm = new Assembler();
+      for (const token of await tokensOf(splitAt(JSON.stringify(value), cuts), parser)) asm.consume(token);
+      return same(asm.current, value);
+    },
+    {numRuns: 100},
+    'assembled value equals the original'
+  );
+});
+
+test('disassembler → stringer round-trips against JSON.stringify', async t => {
+  await t.prop(
+    [json()],
+    value => {
+      const write = stringer();
+      let text = '';
+      for (const token of disassembler()(value)) {
+        const piece = write(token);
+        if (piece !== none) text += piece;
+      }
+      const tail = write(none);
+      if (tail !== none) text += tail;
+      return same(JSON.parse(text), value);
+    },
+    {numRuns: 200},
+    'stringer output parses back to the original'
+  );
+});
+
+// JSONC: comments and whitespace inserted between tokens, then split anywhere —
+// comments may straddle chunk boundaries, including their delimiters
+const commentBody = () => fc.string({maxLength: 12}).filter(s => !s.includes('*/') && !/[\r\n]/.test(s));
+const decorate = (pretty, kinds, bodies) => {
+  let i = 0;
+  return pretty.replace(/\n/g, () => {
+    const kind = kinds[i % kinds.length],
+      body = bodies[i % bodies.length];
+    ++i;
+    return kind === 1 ? ' /*' + body + '*/\n' : kind === 2 ? ' //' + body + '\n' : '\n';
+  });
+};
+
+test('jsonc parser: comments survive arbitrary chunk boundaries', async t => {
+  await t.prop(
+    [json(), fc.array(fc.constantFrom(0, 1, 2), {minLength: 1, maxLength: 8}), fc.array(commentBody(), {minLength: 1, maxLength: 8}), offsets()],
+    async (value, kinds, bodies, cuts) => {
+      const text = decorate(JSON.stringify(value, null, 1), kinds, bodies);
+      const whole = coalesce(await tokensOf([text], jsoncParser)),
+        chunked = coalesce(await tokensOf(splitAt(text, cuts), jsoncParser));
+      if (!same(whole, chunked)) return false;
+      const asm = new Assembler();
+      for (const token of chunked) asm.consume(token);
+      return same(asm.current, value);
+    },
+    {numRuns: 100},
+    'tokens and comments are split-invariant'
+  );
+});