native: add deparse

native/README.md

@@ -93,7 +93,7 @@ supportedArchitectures:
 Drop-in replacement for `@libpg-query/parser`:
 
 ```js
-const { parse, parseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native');
+const { parse, parseSync, deparse, deparseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native');
 
 // Sync (no init needed — native loads instantly)
 const result = parseSync('SELECT id, name FROM users WHERE active = true');
@@ -111,6 +111,132 @@ const result2 = await parse('SELECT id, name FROM users WHERE active = true');
 | `fingerprintSync(sql)` / `fingerprint(sql)` | ✓ | ✓ | 16-char hex fingerprint |
 | `normalizeSync(sql)` / `normalize(sql)` | ✓ | ✓ | Normalized query string |
 | `scanSync(sql)` / `scan(sql)` | ✓ | ✓ | `ScanResult` with tokens |
+| `deparseSync(tree, opts?)` / `deparse(tree, opts?)` | ✓ | ✓ | SQL string |
+| `extractCommentsSync(sql)` / `extractComments(sql)` | ✓ | ✓ | `DeparseComment[]` |
+
+### Deparsing
+
+`deparse()` is the inverse of `parse()`, using PostgreSQL's own deparser rather than a
+reimplementation of it — so its output tracks the server's grammar, and constructs added
+in PG 18 round-trip instead of being silently dropped.
+
+```js
+const { parseSync, deparseSync } = require('@ashbyhq/libpg-query-native');
+
+const tree = parseSync('select a,b   from   t where x=1');
+deparseSync(tree);
+// SELECT a, b FROM t WHERE x = 1
+
+// Edit the tree in between to rewrite a query:
+tree.stmts[0].stmt.SelectStmt.fromClause[0].RangeVar.relname = 'other_table';
+deparseSync(tree);
+// SELECT a, b FROM other_table WHERE x = 1
+```
+
+Encoding is strict: a misspelled field or a bogus enum value throws rather than being
+dropped and deparsed into quietly wrong SQL. Trees the deparser itself rejects throw a
+`SqlError` carrying the failing C function and line.
+
+#### Formatting
+
+`prettyPrint` breaks the statement across lines. The remaining layout options are
+pretty-print options upstream, so they only take effect alongside it.
+
+```js
+deparseSync(tree, { prettyPrint: true, indentSize: 2 });
+// SELECT a, b, c
+// FROM mytable
+// WHERE
+//   x = 1
+//   AND y = 2
+```
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `prettyPrint` | `false` | Break the statement across lines |
+| `indentSize` | `4` | Spaces per indent level |
+| `maxLineLength` | `80` | Soft wrap width for lists of items |
+| `trailingNewline` | `false` | Append a newline after the statement |
+| `commasStartOfLine` | `false` | Put separating commas at the start of the line |
+| `comments` | — | Comments to weave back in (see below) |
+
+#### Comments
+
+Parse trees don't carry comments, so a parse/deparse round trip drops them. Pull them
+off the source first and hand them back:
+
+```js
+const { parseSync, deparseSync, extractCommentsSync } = require('@ashbyhq/libpg-query-native');
+
+const sql = '-- keep me\nSELECT a FROM t';
+deparseSync(parseSync(sql), { comments: extractCommentsSync(sql) });
+// -- keep me
+// SELECT a FROM t
+```
+
+Each comment carries `matchLocation` (the offset it anchors to), `newlinesBefore`,
+`newlinesAfter` and `text` — filter or rewrite the list before passing it back.
+
+#### Limits and memory
+
+`deparse()` is heavier than `parse()`, in both directions:
+
+- **Nesting depth.** Encoding is recursive, and nesting grows about one level per set
+  operation. The limit is 2000, so a chain of ~2000 `UNION`/`INTERSECT`/`EXCEPT` is the
+  ceiling; past it you get a `RangeError` naming the limit. This is a safety bound, not
+  just a quota — `deparseRawStmt` on the C side has no depth guard of its own.
+- **Peak memory.** `pg_query_deparse_protobuf()` rebuilds the entire tree as Postgres
+  `Node` structs in C, so peak allocation is proportional to tree size — on the order of
+  the parse itself. Encoding adds a transient renamed copy of the tree in the JS heap,
+  which the GC reclaims afterwards.
+
+Encoding costs about 465 ms of a deparse on a 26 MB parse tree. Switching from
+`@bufbuild/protobuf` to protobufjs cut that roughly 6× end to end; the trade is ~2 MB more
+in `node_modules`, since protobufjs (3.9 MB) is larger than `@bufbuild/protobuf` (1.9 MB),
+against a package tarball that shrank from 129 kB to 51 kB.
+
+The same allocator caveat as parsing applies, and more so: with the system allocator RSS
+ratchets across repeated deparses, and with jemalloc it stabilizes. Measured on a 26 MB
+parse tree, four deparse/settle cycles:
+
+| Allocator | after #1 | #2 | #3 | #4 |
+|-----------|---------|-----|-----|-----|
+| system | 876 MB | 898 MB | 978 MB | 980 MB (still climbing) |
+| **jemalloc** | 510 MB | 568 MB | 556 MB | **562 MB (flat)** |
+
+If you deparse large trees repeatedly, run with jemalloc.
+
+#### How the tree gets to the deparser
+
+`pg_query_deparse_protobuf()` takes a protobuf-encoded tree, but `parse()` returns JSON.
+`pg_query.proto` maps between the two with `json_name` annotations — 1,683 of them, which
+is why `SelectStmt` and `targetList` in the JSON correspond to `select_stmt` and
+`target_list` in the schema.
+
+[protobufjs's *converters* ignore `json_name`](https://github.com/protobufjs/protobuf.js/pull/1825),
+which is what made it a dead end for this historically. But its *parser* keeps the
+annotation, exposed as `Field.jsonName`, so `src/proto.ts` drives the mapping itself off
+the descriptor — a key rename, not a fork. That hand-written step is also where strictness
+lives: protobufjs silently drops unknown keys and defaults unrecognised enum names to `0`,
+both of which would yield valid-looking SQL that doesn't match the tree you passed, so the
+remap rejects them instead.
+
+The alternative, [`@bufbuild/protobuf`](https://github.com/bufbuild/protobuf-es), honours
+`json_name` natively and needs no remap — it was the original implementation here. It was
+replaced because it is reflection-driven and allocates two arrays per nested message; on a
+26 MB parse tree (~1.44M messages) that measured ~10× slower. `test/proto.test.js` pins
+the current encoder byte-for-byte against golden encodings captured from it.
+
+The schema descriptor lives in `src/gen/pg_query.json` and is committed, so `npm ci` and
+the platform builds need no protobuf toolchain. Regenerate it when the libpg_query pin
+moves:
+
+```bash
+npm run generate:proto
+```
+
+That refuses to run unless `protos/18/pg_query.proto` matches the pinned tag — a tree
+encoded against a mismatched schema would deparse into wrong SQL rather than fail loudly.
 
 ## Using jemalloc for optimal memory
 

native/package.json

@@ -24,9 +24,10 @@
     "build:ts": "tsc",
     "build": "npm run build:native && npm run build:ts",
     "clean": "make clean && rm -rf dist",
-    "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js",
+    "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/deparse.test.js test/proto.test.js",
     "benchmark": "node benchmark/memory.mjs",
-    "package-platforms": "node scripts/package-platforms.mjs"
+    "package-platforms": "node scripts/package-platforms.mjs",
+    "generate:proto": "node scripts/generate-proto.mjs"
   },
   "keywords": [
     "postgresql",
@@ -43,7 +44,8 @@
     "directory": "native"
   },
   "dependencies": {
-    "@pgsql/types": "^18.0.0"
+    "@pgsql/types": "^18.0.0",
+    "protobufjs": "^8.7.2"
   },
   "devDependencies": {
     "@types/node": "^26.0.1",

native/scripts/generate-proto.mjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+//
+// Regenerates src/gen/pg_query.json from the pg_query.proto matching the
+// pinned libpg_query tag.
+//
+// deparse() has to hand libpg_query a protobuf-encoded parse tree, but parse()
+// returns JSON. pg_query.proto maps between the two with `json_name`
+// annotations (1,683 of them). protobufjs's converters ignore those, but its
+// parser keeps them, so src/proto.ts drives the mapping itself off this
+// descriptor — see the comments there.
+//
+// A pre-parsed descriptor rather than the .proto text: Root.fromJSON is ~9 ms
+// at startup, parsing 123 KB of .proto is not.
+//
+// The output is committed, so `npm ci` and the platform builds need no
+// protobuf toolchain. Re-run this after the libpg_query pin moves:
+//
+//   npm run generate:proto
+//
+// The schema MUST match x-upstream.libpgQueryTag in package.json. A parse tree
+// encoded against a different schema will deparse into wrong SQL or fail
+// outright, so this script refuses to run if protos/NN is out of sync.
+
+import { createHash } from "node:crypto";
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import protobuf from "protobufjs";
+
+const nativeDir = join(dirname(fileURLToPath(import.meta.url)), "..");
+const repoRoot = join(nativeDir, "..");
+
+const pkg = JSON.parse(readFileSync(join(nativeDir, "package.json"), "utf8"));
+const { pgMajor, libpgQueryTag } = pkg["x-upstream"];
+
+const protoDir = join(repoRoot, "protos", pgMajor);
+const protoFile = join(protoDir, "pg_query.proto");
+
+if (!existsSync(protoFile)) {
+  console.error(`Missing ${protoFile} — run \`pnpm run fetch:protos\` at the repo root.`);
+  process.exit(1);
+}
+
+// Guard against silently generating from a stale schema.
+const upstreamUrl =
+  `https://raw.githubusercontent.com/pganalyze/libpg_query/${libpgQueryTag}/protobuf/pg_query.proto`;
+const local = readFileSync(protoFile);
+const remote = Buffer.from(await (await fetch(upstreamUrl)).arrayBuffer());
+const digest = (buf) => createHash("sha256").update(buf).digest("hex").slice(0, 12);
+
+if (!local.equals(remote)) {
+  console.error(
+    `protos/${pgMajor}/pg_query.proto does not match libpg_query ${libpgQueryTag}.\n` +
+      `  local:  ${digest(local)}\n` +
+      `  ${libpgQueryTag}: ${digest(remote)}\n` +
+      `Run \`pnpm run fetch:protos\` at the repo root, then re-run this.`
+  );
+  process.exit(1);
+}
+
+const outFile = join(nativeDir, "src", "gen", "pg_query.json");
+mkdirSync(dirname(outFile), { recursive: true });
+
+// toJSON() keeps field options, which is where json_name lives.
+const schema = protobuf.loadSync(protoFile).toJSON({ keepComments: false });
+writeFileSync(outFile, `${JSON.stringify(schema)}\n`);
+
+console.log(`✅ Generated src/gen/pg_query.json from libpg_query ${libpgQueryTag}`);