native: add deparse

native/Makefile

@@ -66,6 +66,14 @@ $(LIBPG_QUERY_DIR):
 	mkdir -p $(CACHE_DIR)
 	rm -rf $(LIBPG_QUERY_DIR).tmp
 	git clone -b $(LIBPG_QUERY_TAG) --single-branch --depth 1 $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR).tmp
+	# Local patches, applied before the move so a failed patch can't leave a
+	# poisoned cache dir. patches/*.patch documents what each one is for; a
+	# conflict on a libpg_query bump fails the build loudly, which is the
+	# desired signal that the patch needs rebasing or upstreaming succeeded.
+	# git apply rather than patch(1): the musl builds run in Alpine containers
+	# that don't ship patch, but git is guaranteed here — this recipe just ran
+	# git clone.
+	cd $(LIBPG_QUERY_DIR).tmp && for p in $(CURDIR)/patches/*.patch; do git apply $$p; done
 	mv $(LIBPG_QUERY_DIR).tmp $(LIBPG_QUERY_DIR)
 
 $(LIBPG_QUERY_LIB): $(LIBPG_QUERY_DIR)

native/README.md

@@ -93,7 +93,7 @@ supportedArchitectures:
 Drop-in replacement for `@libpg-query/parser`:
 
 ```js
-const { parse, parseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native');
+const { parse, parseSync, deparse, deparseSync, fingerprint, normalize, scan } = require('@ashbyhq/libpg-query-native');
 
 // Sync (no init needed — native loads instantly)
 const result = parseSync('SELECT id, name FROM users WHERE active = true');
@@ -111,6 +111,157 @@ const result2 = await parse('SELECT id, name FROM users WHERE active = true');
 | `fingerprintSync(sql)` / `fingerprint(sql)` | ✓ | ✓ | 16-char hex fingerprint |
 | `normalizeSync(sql)` / `normalize(sql)` | ✓ | ✓ | Normalized query string |
 | `scanSync(sql)` / `scan(sql)` | ✓ | ✓ | `ScanResult` with tokens |
+| `deparseSync(tree, opts?)` / `deparse(tree, opts?)` | ✓ | ✓ | SQL string |
+| `extractCommentsSync(sql)` / `extractComments(sql)` | ✓ | ✓ | `DeparseComment[]` |
+
+### Deparsing
+
+`deparse()` is the inverse of `parse()`, using PostgreSQL's own deparser rather than a
+reimplementation of it — so its output tracks the server's grammar, and constructs added
+in PG 18 round-trip instead of being silently dropped.
+
+```js
+const { parseSync, deparseSync } = require('@ashbyhq/libpg-query-native');
+
+const tree = parseSync('select a,b   from   t where x=1');
+deparseSync(tree);
+// SELECT a, b FROM t WHERE x = 1
+
+// Edit the tree in between to rewrite a query:
+tree.stmts[0].stmt.SelectStmt.fromClause[0].RangeVar.relname = 'other_table';
+deparseSync(tree);
+// SELECT a, b FROM other_table WHERE x = 1
+```
+
+Encoding is strict: a misspelled field or a bogus enum value throws rather than being
+dropped and deparsed into quietly wrong SQL. Trees the deparser itself rejects throw a
+`SqlError` carrying the failing C function and line.
+
+#### Formatting
+
+`prettyPrint` breaks the statement across lines. The remaining layout options are
+pretty-print options upstream, so they only take effect alongside it.
+
+```js
+const { parseSync, deparseSync } = require('@ashbyhq/libpg-query-native');
+
+const tree = parseSync('select a, b, c from mytable where x = 1 and y = 2');
+deparseSync(tree, { prettyPrint: true, indentSize: 2 });
+// SELECT a, b, c
+// FROM mytable
+// WHERE
+//   x = 1
+//   AND y = 2
+```
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `prettyPrint` | `false` | Break the statement across lines |
+| `indentSize` | `4` | Spaces per indent level |
+| `maxLineLength` | `80` | Soft wrap width for lists of items |
+| `trailingNewline` | `false` | Append a newline after the statement |
+| `commasStartOfLine` | `false` | Put separating commas at the start of the line |
+| `comments` | — | Comments to weave back in (see below) |
+
+#### Comments
+
+Parse trees don't carry comments, so a parse/deparse round trip drops them. Pull them
+off the source first and hand them back:
+
+```js
+const { parseSync, deparseSync, extractCommentsSync } = require('@ashbyhq/libpg-query-native');
+
+const sql = '-- keep me\nSELECT a FROM t';
+deparseSync(parseSync(sql), { comments: extractCommentsSync(sql) });
+// -- keep me
+//  SELECT a FROM t
+```
+
+Each comment carries `matchLocation` (the offset it anchors to), `newlinesBefore`,
+`newlinesAfter` and `text` — filter or rewrite the list before passing it back.
+
+#### Limits and memory
+
+`deparse()` is heavier than `parse()`, in both directions:
+
+- **Nesting depth.** Encoding is recursive, and nesting grows about one level per set
+  operation. The limit is 2000, so a chain of ~2000 `UNION`/`INTERSECT`/`EXCEPT` is the
+  ceiling; past it you get a `RangeError` naming the limit. This is a safety bound, not
+  just a quota — `deparseRawStmt` on the C side has no depth guard of its own.
+- **Peak memory.** `pg_query_deparse_protobuf()` rebuilds the entire tree as Postgres
+  `Node` structs in C, so peak allocation is proportional to tree size — on the order of
+  the parse itself. That is now the whole cost: encoding writes bytes directly and builds
+  no intermediate copy of the tree.
+
+Encoding is a single pass that writes protobuf bytes straight out of the parse tree —
+0.3–17 µs of a 1.0–41 µs deparse. Most of the remaining time was never the SQL
+rendering: profiling put ~90% of the C call inside protobuf-c, which walks the
+descriptor of every message it touches, and pg_query's `Node` has 271 fields with every
+value in a parse tree wrapped in one. Two patches in `patches/` remove that walk from
+both directions — `protobuf_unpack_palloc.patch` unpacks into libpg_query's memory
+context and drops protobuf-c's free pass, and `protobufc_skip_noop_field_loop.patch`
+skips the post-scan field loop for descriptors that have nothing repeated or required.
+Together they take a deparse from 8.0 µs to 3.6 µs on a simple select and from 70 µs to
+43 µs on a 100-column projection.
+
+Against `pgsql-deparser` on a parse→edit→deparse flow: 1.03× on small statements, 1.08×
+on medium — call it parity — and 0.84× on a 100-column projection. The TypeScript
+deparser pays a visitor dispatch per node, so this pulls ahead as trees get broad rather
+than deep; on ordinary statements the two are even.
+
+The allocator caveat from parsing still applies. On a 26 MB parse tree, four
+deparse/settle cycles:
+
+| Allocator | after #1 | #2 | #3 | #4 |
+|-----------|---------|-----|-----|-----|
+| system | 705 MB | 710 MB | 711 MB | **712 MB (flat)** |
+| **jemalloc** | 245 MB | 246 MB | 248 MB | **249 MB** |
+
+Neither plateau ratchets. The system-malloc figure is higher than it was before the
+patches because the unpacked structs now ride the memory context's high-water mark; on
+a sustained realistic workload (126k parse→edit→deparse ops/sec on ordinary statements)
+RSS holds at ~70 MB under both allocators. If you deparse large trees repeatedly, run
+with jemalloc.
+
+**Untrusted input.** Like every other entry point here, `deparse()` runs synchronously on
+the calling thread and has no aggregate size budget — a large tree blocks the event loop
+for the duration (a 26 MB tree is ~500 ms, the same shape as `parse()` on the SQL that
+produced it). The bounded inputs are nesting depth and `DeparseOptions.comments`, capped
+at 1,000,000 entries. If you deparse trees derived from untrusted input, apply your own
+size limit before calling, or run it off the main thread.
+
+#### How the tree gets to the deparser
+
+`pg_query_deparse_protobuf()` takes a protobuf-encoded tree, but `parse()` returns JSON.
+`pg_query.proto` maps between the two with `json_name` annotations — 1,683 of them, which
+is why `SelectStmt` and `targetList` in the JSON correspond to `select_stmt` and
+`target_list` in the schema.
+
+[protobufjs's *converters* ignore `json_name`](https://github.com/protobufjs/protobuf.js/pull/1825),
+which is what made it a dead end for this historically. But its *parser* keeps the
+annotation, exposed as `Field.jsonName`, so `src/proto.ts` drives the mapping itself off
+the descriptor — a key rename, not a fork. That hand-written step is also where strictness
+lives: protobufjs silently drops unknown keys and defaults unrecognised enum names to `0`,
+both of which would yield valid-looking SQL that doesn't match the tree you passed, so the
+remap rejects them instead.
+
+The alternative, [`@bufbuild/protobuf`](https://github.com/bufbuild/protobuf-es), honours
+`json_name` natively and needs no remap — it was the original implementation here. It was
+replaced because it is reflection-driven and allocates two arrays per nested message; on a
+26 MB parse tree (~1.44M messages) that measured ~10× slower. `test/proto.test.js` compares
+the current encoder's output byte-for-byte against bytes recorded from it, so the speedup
+cannot quietly change what libpg_query receives.
+
+The schema descriptor lives in `src/gen/pg_query.json` and is committed, so `npm ci` and
+the platform builds need no protobuf toolchain. Regenerate it when the libpg_query pin
+moves:
+
+```bash
+npm run generate:proto
+```
+
+That refuses to run unless `protos/18/pg_query.proto` matches the pinned tag — a tree
+encoded against a mismatched schema would deparse into wrong SQL rather than fail loudly.
 
 ## Using jemalloc for optimal memory
 

native/package.json

@@ -24,9 +24,11 @@
     "build:ts": "tsc",
     "build": "npm run build:native && npm run build:ts",
     "clean": "make clean && rm -rf dist",
-    "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js",
+    "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/deparse.test.js test/proto.test.js test/readme.test.js",
     "benchmark": "node benchmark/memory.mjs",
-    "package-platforms": "node scripts/package-platforms.mjs"
+    "package-platforms": "node scripts/package-platforms.mjs",
+    "generate:proto": "node scripts/generate-proto.mjs",
+    "generate:fixtures": "node scripts/generate-fixtures.mjs"
   },
   "keywords": [
     "postgresql",
@@ -43,7 +45,8 @@
     "directory": "native"
   },
   "dependencies": {
-    "@pgsql/types": "^18.0.0"
+    "@pgsql/types": "^18.0.0",
+    "protobufjs": "^8.7.2"
   },
   "devDependencies": {
     "@types/node": "^26.0.1",

native/patches/deparse_no_escape_string_syntax.patch

@@ -0,0 +1,74 @@
+Spell string constants the way Postgres does, without E'' escape syntax.
+
+deparseStringLiteral wraps any value containing a backslash in E'' and
+doubles the backslashes. The comment explains why: it is copied from
+postgres_fdw/deparse.c, which ships SQL to a remote server whose
+standard_conforming_strings it cannot see, so it picks the spelling that
+is safe under either setting.
+
+A general-purpose deparser has no remote server. Postgres' own
+parse-tree-to-SQL path, simple_quote_literal() in ruleutils.c, does the
+opposite and says so:
+
+    We form the string literal according to the prevailing setting of
+    standard_conforming_strings; we never use E''.
+
+The difference is observable. pg_get_constraintdef() on
+CHECK (c ~ '^\d+$') returns the plain literal, while deparsing the same
+parse tree here returns E'^\\d+$'. Ordinary statements therefore fail the
+textual round-trip this repo's deparse tests are built on -- among them
+SELECT regexp_replace(x, '\s+', ' '), which is unremarkable SQL.
+
+E'' is also a Postgres extension rather than standard SQL, so it does not
+survive being handed to other engines. We deparse Postgres parse trees and
+send the result to ClickHouse, which has no E prefix: it lexes the E as an
+identifier and rejects the whole query. Postgres made this same change for
+the same reason in 2006, so that pg_dump output could load into other
+databases without the backslash doubling.
+
+Keying the doubling off standard_conforming_strings rather than hardcoding
+it also honours PG_QUERY_DISABLE_STANDARD_CONFORMING_STRINGS, which this
+library already exposes and which the deparser previously ignored.
+
+Upstreaming this needs one expectation updated: deparse_tests.c pins
+CREATE DOMAIN us_postal_code with E'^\\d{5}$' inputs, copied from the
+Postgres docs, which now round-trip as plain literals. We do not run that
+suite here, so the patch is left minimal.
+
+diff --git a/src/postgres_deparse.c b/src/postgres_deparse.c
+index 58401ad..a018112 100644
+--- a/src/postgres_deparse.c
++++ b/src/postgres_deparse.c
+@@ -13,6 +13,7 @@
+ #include "nodes/nodes.h"
+ #include "nodes/parsenodes.h"
+ #include "nodes/pg_list.h"
++#include "parser/parser.h"
+ #include "utils/builtins.h"
+ #include "utils/datetime.h"
+ #include "utils/timestamp.h"
+@@ -675,19 +676,17 @@ deparseStringLiteral(DeparseState *state, const char *val)
+ 	const char *valptr;
+
+ 	/*
+-	 * Rather than making assumptions about the remote server's value of
+-	 * standard_conforming_strings, always use E'foo' syntax if there are any
+-	 * backslashes.  This will fail on remote servers before 8.1, but those
+-	 * are long out of support.
++	 * We form the string literal according to the prevailing setting of
++	 * standard_conforming_strings; we never use E''.  This matches
++	 * simple_quote_literal() in ruleutils.c, which is what Postgres itself
++	 * uses to turn a parse tree back into SQL.
+ 	 */
+-	if (strchr(val, '\\') != NULL)
+-		deparseAppendStringInfoChar(state, ESCAPE_STRING_SYNTAX);
+ 	deparseAppendStringInfoChar(state, '\'');
+ 	for (valptr = val; *valptr; valptr++)
+ 	{
+ 		char		ch = *valptr;
+
+-		if (SQL_STR_DOUBLE(ch, true))
++		if (SQL_STR_DOUBLE(ch, !standard_conforming_strings))
+ 			deparseAppendStringInfoChar(state, ch);
+ 		deparseAppendStringInfoChar(state, ch);
+ 	}

native/patches/protobuf_unpack_palloc.patch

@@ -0,0 +1,60 @@
+Unpack deparse input with palloc instead of malloc, and skip the free pass.
+
+protobuf-c's default allocator does one malloc per message and, worse,
+protobuf_c_message_free_unpacked walks every field of every message's
+descriptor to find pointers to free. pg_query's Node descriptor has 271
+fields and every value in a parse tree is wrapped in a Node, so profiling
+puts ~80% of pg_query_deparse_protobuf inside protobuf-c: 45% of samples
+in free_unpacked alone, 34% in unpack.
+
+pg_query_protobuf_to_nodes has exactly one caller, pg_query_deparse_protobuf,
+which always runs it inside a pg_query memory context -- the same context
+_readRawStmt already pallocs the Node tree into. palloc is an arena here:
+allocation is a bump, and MemoryContextDelete frees everything at once. So
+the unpacked protobuf structs can live in the context too, the free pass
+becomes unnecessary, and the 271-field descriptor walk disappears with it.
+
+Byte-level behaviour is unchanged: re-packing an arena-unpacked message
+reproduces the input exactly, and the deparse test suite pins output SQL.
+
+diff --git a/src/pg_query_readfuncs_protobuf.c b/src/pg_query_readfuncs_protobuf.c
+index d0a7e21..2993096 100644
+--- a/src/pg_query_readfuncs_protobuf.c
++++ b/src/pg_query_readfuncs_protobuf.c
+@@ -152,13 +152,27 @@ static Node * _readNode(PgQuery__Node *msg)
+ 	}
+ }
+
++static void *unpack_palloc(void *allocator_data, size_t size)
++{
++	return palloc(size);
++}
++
++static void unpack_free_noop(void *allocator_data, void *pointer)
++{
++	/* freed wholesale when the caller's memory context is deleted */
++}
++
++static ProtobufCAllocator unpack_allocator = {
++	unpack_palloc, unpack_free_noop, NULL
++};
++
+ List * pg_query_protobuf_to_nodes(PgQueryProtobuf protobuf)
+ {
+ 	PgQuery__ParseResult *result = NULL;
+ 	List * list = NULL;
+ 	size_t i = 0;
+
+-	result = pg_query__parse_result__unpack(NULL, protobuf.len, (const uint8_t *) protobuf.data);
++	result = pg_query__parse_result__unpack(&unpack_allocator, protobuf.len, (const uint8_t *) protobuf.data);
+
+ 	// TODO: Handle this by returning an error instead
+ 	Assert(result != NULL);
+@@ -171,7 +185,5 @@ List * pg_query_protobuf_to_nodes(PgQueryProtobuf protobuf)
+     for (i = 1; i < result->n_stmts; i++)
+ 		list = lappend(list, _readRawStmt(result->stmts[i]));
+
+-	pg_query__parse_result__free_unpacked(result, NULL);
+-
+ 	return list;
+ }