fix(wasm): remove the playground's memory leak and Run budget

- cycle only the request per Run, not the whole PHP module
- force Opcache's mmap shared-memory backend on
- patch Zend's chunk allocator for Emscripten's whole-mapping-only
munmap
- restore the ZEND_MM_ERROR allocator diagnostics
- recycle the wasm module in the controller as a backstop

Author: norberttech

documentation/contributing/wasm.md

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
 # Build script for Flow PHP Interactive Playground
-# Builds PHP 8.5.8
+# Builds PHP 8.5.8 as 64-bit wasm (see WASM64_MODE below) with a forced mmap Opcache shared-memory
+# backend (see php_cv_shm_mmap_anon below).
 
 set -xeu
 
@@ -15,6 +16,13 @@ echo "=== Build started at $(date) ==="
 PHP_VERSION=8.5.8
 PHP_PATH=php-$PHP_VERSION
 
+# MEMORY64=2 is wasm64 for clang/lld lowered to wasm32 by Binaryen, so PHP gets an 8-byte zend_long
+# (pack()'s q/Q/J/P, which Floe and sortBy() need) without requiring Memory64 in visitors' browsers.
+# Set via EMCC_CFLAGS, not CFLAGS, which is exported below -- after the dependencies are built.
+WASM64_MODE="-sMEMORY64=2"
+EM_TARGET=wasm64-emscripten
+export EMCC_CFLAGS="$WASM64_MODE"
+
 echo "Build libxml2 for WebAssembly"
 LIBXML2_VERSION=2.11.4
 LIBXML2_DIR=libxml2-$LIBXML2_VERSION
@@ -48,9 +56,8 @@ if [ ! -d "$LIBXML2_DIR" ]; then
 fi
 
 echo "Build libpg_query for WebAssembly"
-# Single source of truth is the pg_query extension's own pin, so the playground and the CLI can
-# never disagree about which PostgreSQL grammar parses. build.sh already consumes that directory's
-# ext/ sources below, so this adds no coupling that did not already exist.
+# Read from the pg_query extension's own pin, so the playground and the CLI cannot disagree about
+# which PostgreSQL grammar parses.
 LIBPG_QUERY_VERSION=$(sed -n 's/^LIBPG_QUERY_VERSION := //p' \
     "$PROJECT_ROOT/../src/extension/pg-query-ext/Makefile")
 
@@ -59,27 +66,22 @@ if [ -z "$LIBPG_QUERY_VERSION" ]; then
     exit 1
 fi
 
-# Version in the directory name, like libxml2/libzip/php above: a bumped pin then misses the
-# guard below and re-clones, instead of silently reusing the previous version's checkout.
-# Note this tracks a moving branch, so an upstream branch move still needs a manual rm -rf.
+# Version in the directory name so a bumped pin re-clones instead of reusing the old checkout.
+# This tracks a moving branch, so an upstream branch move still needs a manual rm -rf.
 LIBPG_QUERY_DIR=libpg_query-$LIBPG_QUERY_VERSION
 LIBPG_QUERY_INSTALL_DIR="$PROJECT_ROOT/$LIBPG_QUERY_DIR"
 
-# Guard on the built archive rather than the directory, like libzip below: a build that fails
-# part way then leaves a tree that a directory check would skip, and the missing archive would
-# only surface much later at link time.
+# Guard on the built archive, not the directory: a part-way failure leaves a tree that a directory
+# check would skip, and the missing archive would only surface at link time.
 if [ ! -f "$LIBPG_QUERY_INSTALL_DIR/libpg_query.a" ]; then
     rm -rf "$LIBPG_QUERY_DIR"
     git clone --depth=1 --branch=$LIBPG_QUERY_VERSION \
         https://github.com/pganalyze/libpg_query.git "$LIBPG_QUERY_DIR"
     cd $LIBPG_QUERY_DIR
 
-    # Build with Emscripten - override CC, AR and ARFLAGS.
-    # libpg_query 18 keeps AR and ARFLAGS separate (Makefile:71-72), unlike 17's combined
-    # AR := $(AR) rs, so folding the flags into AR gives `emar rcs rs` -- two operations, which
-    # llvm-ar rejects. Both have to come from the command line: the Makefile's `?=` never fires,
-    # because GNU make predefines ARFLAGS=-rv and `?=` only assigns when the origin is `undefined`.
-    # `rcs` rather than the default `-rv` so the archive still gets its symbol index.
+    # libpg_query 18 keeps AR and ARFLAGS separate, so folding the flags into AR gives `emar rcs rs`,
+    # which llvm-ar rejects. Both must come from the command line -- make predefines ARFLAGS, so the
+    # Makefile's `?=` never fires.
     emmake make build -j$(nproc) CC=emcc AR=emar ARFLAGS=rcs
 
     cd $PROJECT_ROOT
@@ -95,15 +97,16 @@ if [ ! -f "$LIBZIP_INSTALL_DIR/lib/libzip.a" ]; then
     # First, ensure Emscripten's zlib port is built by triggering a compile
     # This downloads and builds zlib to the Emscripten cache
     echo "int main(){return 0;}" > /tmp/zlib_test.c
-    # -flto must match CFLAGS below: emcc caches a separate port build per variant, and this
-    # script links the lto/ one. Without it only the non-LTO libz.a is produced and the check
-    # after this block fails.
+    # -flto must match CFLAGS below: emcc caches a separate port build per variant and this links
+    # the lto/ one.
     emcc -flto -sUSE_ZLIB=1 /tmp/zlib_test.c -o /tmp/zlib_test.js 2>/dev/null || true
     rm -f /tmp/zlib_test.c /tmp/zlib_test.js /tmp/zlib_test.wasm
 
     # Get Emscripten cache path and locate zlib
     EM_CACHE=$(em-config CACHE)
-    ZLIB_LIBRARY="$EM_CACHE/sysroot/lib/wasm32-emscripten/lto/libz.a"
+    # Per-target path: hardcoding wasm32 would find a real file of the wrong architecture, which the
+    # guard below cannot detect.
+    ZLIB_LIBRARY="$EM_CACHE/sysroot/lib/$EM_TARGET/lto/libz.a"
     ZLIB_INCLUDE_DIR="$EM_CACHE/sysroot/include"
 
     echo "Using zlib from Emscripten cache:"
@@ -121,11 +124,11 @@ if [ ! -f "$LIBZIP_INSTALL_DIR/lib/libzip.a" ]; then
     tar xf $LIBZIP_DIR.tar.xz
     cd $LIBZIP_DIR
 
+    # CMakeCache.txt pins the resolved zlib path and target, so a tree left from another target has
+    # to be cleared rather than reconfigured.
+    rm -rf build
     mkdir -p build && cd build
 
-    # Configure libzip for WebAssembly using CMake
-    # Provide explicit paths to Emscripten's zlib (from its ports system)
-    # Disable encryption and optional compression to minimize dependencies
     emcmake cmake .. \
         -DCMAKE_INSTALL_PREFIX=$LIBZIP_INSTALL_DIR \
         -DZLIB_LIBRARY=$ZLIB_LIBRARY \
@@ -161,14 +164,15 @@ fi
 echo "Patch php-src"
 # Re-runs reuse an already extracted $PHP_PATH, so this has to be idempotent: a cleanly applying
 # reverse patch means it is already in place. Anything else is a real failure and set -e ends here.
-PHP_PATCH="$PROJECT_ROOT/patches/php-8.5-opcache-unistd.patch"
 cd "$PHP_PATH"
 
-if patch -p1 --reverse --dry-run --force --silent <"$PHP_PATCH" >/dev/null 2>&1; then
-    echo "  already applied: $(basename "$PHP_PATCH")"
-else
-    patch -p1 --forward <"$PHP_PATCH"
-fi
+for PHP_PATCH in "$PROJECT_ROOT"/patches/*.patch; do
+    if patch -p1 --reverse --dry-run --force --silent <"$PHP_PATCH" >/dev/null 2>&1; then
+        echo "  already applied: $(basename "$PHP_PATCH")"
+    else
+        patch -p1 --forward <"$PHP_PATCH"
+    fi
+done
 
 cd "$PROJECT_ROOT"
 
@@ -191,11 +195,10 @@ cp -r "$SNAPPY_EXT_DIR" "$SNAPPY_EXT_DST"
 
 echo "Configure PHP"
 
-# Use -Oz for size optimization instead of -O3 for speed
 # -DHAVE_REALLOCARRAY=1: emcc's link probe for reallocarray() fails while its sysroot header still
-# declares the symbol, so main/php_glob.c (always built since 8.5) compiles its own static copy and
-# clashes. See https://github.com/php/php-src/issues/19152.
-export CFLAGS="-Oz -flto -fPIC -g0 -DZEND_MM_ERROR=0 -DHAVE_REALLOCARRAY=1 -I$LIBXML2_INSTALL_DIR/include/libxml2 -I$LIBPG_QUERY_INSTALL_DIR -I$LIBPG_QUERY_INSTALL_DIR/src -I$LIBZIP_INSTALL_DIR/include -sUSE_ZLIB=1"
+# declares it, so main/php_glob.c compiles a clashing static copy (php-src#19152).
+# ZEND_MM_ERROR is deliberately NOT 0 -- it gates the messages that named the chunk-alignment leak.
+export CFLAGS="-Oz -flto -fPIC -g0 -DHAVE_REALLOCARRAY=1 -I$LIBXML2_INSTALL_DIR/include/libxml2 -I$LIBPG_QUERY_INSTALL_DIR -I$LIBPG_QUERY_INSTALL_DIR/src -I$LIBZIP_INSTALL_DIR/include -sUSE_ZLIB=1"
 export CXXFLAGS="-Oz -flto -fPIC -g0 -std=c++11 -sUSE_ZLIB=1"
 export LDFLAGS="-L$LIBXML2_INSTALL_DIR/lib -L$LIBPG_QUERY_INSTALL_DIR -L$LIBZIP_INSTALL_DIR/lib -sUSE_ZLIB=1"
 
@@ -207,13 +210,7 @@ export LIBZIP_LIBS="-L$LIBZIP_INSTALL_DIR/lib -lzip"
 
 cd $PHP_PATH
 
-# Configure with extensions required by Flow PHP
-# - bcmath: required by flow-php/parquet
-# - libxml: required as base for XML extensions
-# - xml, dom, xmlreader, xmlwriter: required by flow-php/etl-adapter-xml
-# - phar, mbstring: essential PHP extensions
-# - iconv: required by symfony/polyfill-mbstring
-# - zip: required by flow-php/etl-adapter-excel (XLSX files are ZIP archives)
+# The extension set and the reason for each is in documentation/contributing/wasm.md.
 
 # Fix permissions for build scripts
 chmod +x buildconf build/config-stubs build/shtool 2>/dev/null || true
@@ -223,13 +220,14 @@ bash ./buildconf --force
 
 set +e
 
-# --disable-opcache-jit is NOT redundant next to --disable-all. Since php-src 7b4c14dc1016 opcache
-# is non-optional and --disable-all cannot reach it, but its JIT is still gated on $host_cpu -- and
-# emconfigure passes no --host, so config.guess reports the BUILD machine. On any host autoconf
-# calls JIT-capable (x86_64, aarch64, ...) emcc would be handed jit/ir/*.c and dynasm for the host
-# ISA to compile into a wasm32 binary. Same class of workaround as --disable-fiber-asm below.
-# --disable-huge-code-pages is pointless under wasm and matches the other PHP-to-wasm builds.
-# Both pass [no] as their 5th PHP_ARG_ENABLE arg, so neither is affected by --disable-all.
+# Opcache's shared-memory probes all call fork(), which Emscripten lacks, so configure would leave
+# Opcache permanently inert. Only the fork() half really fails -- Emscripten's mmap() serves
+# anonymous MAP_SHARED from linear memory and its fcntl() reports the locks taken, which is enough.
+export php_cv_shm_mmap_anon=yes
+
+# --disable-opcache-jit and --disable-huge-code-pages are NOT redundant next to --disable-all: both
+# pass [no] as their 5th PHP_ARG_ENABLE arg, so --disable-all cannot reach either. Without the JIT
+# flag emcc is handed dynasm for the build machine's ISA, since emconfigure passes no --host.
 emconfigure ./configure \
   --disable-all \
   --disable-opcache-jit \
@@ -290,9 +288,9 @@ echo "Compile pib_eval wrapper"
 emcc $CFLAGS -I . -I Zend -I main -I TSRM/ ../pib_eval.c -c -o pib_eval.o
 
 echo "Link everything together"
-emcc $CFLAGS $LDFLAGS \
+emcc $CFLAGS $LDFLAGS $WASM64_MODE \
   -s ENVIRONMENT=web \
-  -s EXPORTED_FUNCTIONS='["_pib_eval", "_pib_force_exit", "_php_embed_init", "_zend_eval_string", "_php_embed_shutdown"]' \
+  -s EXPORTED_FUNCTIONS='["_pib_eval", "_pib_force_exit", "_pib_heap_bytes"]' \
   -s EXPORTED_RUNTIME_METHODS='["ccall","FS","UTF8ToString","lengthBytesUTF8","stringToUTF8","getValue","setValue","ENV"]' \
   -s MODULARIZE=1 \
   -s EXPORT_NAME="'PHP'" \

wasm/build.sh

@@ -0,0 +1,87 @@
+Stop Zend's chunk allocator from orphaning ~2 MB of wasm memory per chunk.
+
+zend_mm_chunk_alloc_int() aligns a chunk by over-allocating and then munmap()ing
+the unwanted head and tail. Emscripten's munmap() returns EINVAL unless the length
+matches a whole mapping, so both trims fail -- and because wasm memory can never
+shrink, one ZEND_MM_CHUNK_SIZE is lost per chunk, permanently. That is what gave
+the playground a finite Run budget: 2.362 MB leaked per Run, and the tab stopped
+executing PHP after 813 Runs. It was silent because build.sh used to compile with
+-DZEND_MM_ERROR=0, which suppresses the "munmap() failed" message.
+
+posix_memalign() aligns directly, so there is nothing to trim. chunk_truncate()
+and chunk_extend() then report failure, which is the branch _WIN32 already takes
+and which callers handle by falling back to allocate-copy-free.
+
+Not reported upstream: php-src does not target Emscripten. Re-check on the next
+PHP bump.
+
+--- a/Zend/zend_alloc.c
++++ b/Zend/zend_alloc.c
+@@ -477,7 +477,9 @@
+ #endif
+ }
+ 
+-#ifndef HAVE_MREMAP
++/* zend_mm_chunk_extend() is the only non-_WIN32 caller and cannot succeed under Emscripten, so
++ * without this guard the definition is left unreferenced. */
++#if !defined(HAVE_MREMAP) && !defined(__EMSCRIPTEN__)
+ static void *zend_mm_mmap_fixed(void *addr, size_t size)
+ {
+ #ifdef _WIN32
+@@ -740,6 +742,18 @@
+ 
+ static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
+ {
++#ifdef __EMSCRIPTEN__
++	/* Emscripten's munmap() cannot release part of a mapping, so the over-allocate-then-trim path
++	 * below orphans one ZEND_MM_CHUNK_SIZE per chunk and wasm memory never shrinks. Early return
++	 * rather than #else, so the generic path stays referenced and -Wunused-function stays quiet. */
++	void *aligned;
++
++	if (posix_memalign(&aligned, alignment, size) != 0) {
++		return NULL;
++	}
++
++	return aligned;
++#endif
+ 	void *ptr = zend_mm_mmap(size);
+ 
+ 	if (ptr == NULL) {
+@@ -818,7 +832,12 @@
+ 		return;
+ 	}
+ #endif
++#ifdef __EMSCRIPTEN__
++	/* Paired with the posix_memalign() in zend_mm_chunk_alloc_int(). */
++	free(addr);
++#else
+ 	zend_mm_munmap(addr, size);
++#endif
+ }
+ 
+ static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
+@@ -832,10 +851,11 @@
+ 		}
+ 	}
+ #endif
+-#ifndef _WIN32
++#if !defined(_WIN32) && !defined(__EMSCRIPTEN__)
+ 	zend_mm_munmap((char*)addr + new_size, old_size - new_size);
+ 	return 1;
+ #else
++	/* A posix_memalign() block cannot be partially released; callers fall back to copy-and-free. */
+ 	return 0;
+ #endif
+ }
+@@ -860,9 +880,10 @@
+ 	/* Sanity check: The mapping shouldn't have moved. */
+ 	ZEND_ASSERT(ptr == addr);
+ 	return 1;
+-#elif !defined(_WIN32)
++#elif !defined(_WIN32) && !defined(__EMSCRIPTEN__)
+ 	return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
+ #else
++	/* Emscripten's mmap() rejects a non-NULL addr hint, so a fixed extension can never succeed. */
+ 	return 0;
+ #endif
+ }

wasm/patches/php-8.5-emscripten-mm-chunk-alignment.patch

@@ -3,6 +3,7 @@
 #include "Zend/zend_interfaces.h"
 #include "Zend/zend_compile.h"
 #include <emscripten.h>
+#include <emscripten/heap.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -78,7 +79,7 @@ static void pib_report_exception(zend_object *ex) {
         zend_string_release(message);
 		zend_string *file = zval_get_string(GET_PROPERTY_SILENT(&exception, ZEND_STR_FILE));
 		zend_long line = zval_get_long(GET_PROPERTY_SILENT(&exception, ZEND_STR_LINE));
-        fprintf(stderr, "At %s:%d\n", ZSTR_VAL(file), line);
+        fprintf(stderr, "At %s:" ZEND_LONG_FMT "\n", ZSTR_VAL(file), line);
         zend_string_release(file);
         /*
         // Can't get this to work at the end of execution.
@@ -97,20 +98,66 @@ static void pib_report_exception(zend_object *ex) {
     }
 }
 
-// Based on code by https://github.com/oraoto/pib with modifications.
-int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) {
-    int ret = 0;
-    // USE_ZEND_ALLOC prevents using fast shutdown.
-    // putenv("USE_ZEND_ALLOC=0");
-    php_embed_init(0, NULL);
+// One PHP *module* lifecycle per page, one *request* lifecycle per Run.
+static int pib_module_started = 0;
+
+// Everything php_embed_init() does after php_request_startup(), plus this file's own per-request
+// setup. sapi_activate() resets SG() on every request, so this runs for every request, not only
+// the first.
+static void pib_request_prepare(void) {
+    SG(headers_sent) = 1;
+    SG(request_info).no_headers = 1;
+
     pib_cli_register_file_handles();
 
     // Show fatal E_COMPILE_ERRORs and other errors properly (startup errors are normally hidden)
-    PG(display_startup_errors)=1;
-    PG(during_request_startup)=0;
+    PG(display_startup_errors) = 1;
+    PG(during_request_startup) = 0;
 
     // Enable error display to stdout
     PG(display_errors) = 1;
+}
+
+// Opcache's shared memory is allocated once, at MINIT, and sized by opcache.memory_consumption --
+// so it has to be set before php_embed_init(). ini_defaults is the SAPI's documented hook for that
+// (sapi/embed/php_embed.c:201-215); ini_entries cannot be used because php_embed_init() overwrites
+// it with its own HARDCODED_INI at :216.
+#define PIB_INI_DEFAULT(name, value) \
+    ZVAL_NEW_STR(&tmp, zend_string_init(value, sizeof(value) - 1, 1)); \
+    zend_hash_str_update(configuration_hash, name, sizeof(name) - 1, &tmp);
+
+static void pib_ini_defaults(HashTable *configuration_hash) {
+    zval tmp;
+    PIB_INI_DEFAULT("opcache.memory_consumption", "32")
+    PIB_INI_DEFAULT("opcache.interned_strings_buffer", "8")
+    PIB_INI_DEFAULT("opcache.validate_timestamps", "0")
+}
+
+#undef PIB_INI_DEFAULT
+
+double EMSCRIPTEN_KEEPALIVE pib_heap_bytes(void) {
+    return (double) emscripten_get_heap_size();
+}
+
+// Based on code by https://github.com/oraoto/pib with modifications.
+int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) {
+    int ret = 0;
+    // USE_ZEND_ALLOC prevents using fast shutdown.
+    // putenv("USE_ZEND_ALLOC=0");
+    if (!pib_module_started) {
+        php_embed_module.ini_defaults = pib_ini_defaults;
+
+        // The shipped version ignored this return value, so a failed init fell through into
+        // zend_first_try on a dead engine and reported nothing at all.
+        if (php_embed_init(0, NULL) == FAILURE) {
+            fprintf(stderr, "Fatal error: failed to initialize the PHP engine\n");
+            fflush(stderr);
+            return FAILURE;
+        }
+
+        pib_module_started = 1;
+        pib_request_prepare();
+    }
 
     zend_first_try {
         // Set error_reporting to E_ALL
@@ -148,9 +195,19 @@ int EMSCRIPTEN_KEEPALIVE pib_eval(char *code) {
         }
         ret = EG(exit_status);
     } zend_end_try();
-    php_embed_shutdown();
+
+    php_request_shutdown((void *) 0);
+
     fflush(stdout);
     fflush(stderr);
+
+    if (php_request_startup() == FAILURE) {
+        pib_module_started = 0;
+        return FAILURE;
+    }
+
+    pib_request_prepare();
+
     return ret;
 }