fix(proxy): tune macOS libmalloc and trim allocator pages so long-lived RSS stays bounded (#2879)

## Summary

Fixes #2820.

Prevents long-lived macOS proxies from retaining every largest transient
request-body allocation in libmalloc. The reporter’s production A/B
isolated the allocator behavior and verified the two pre-main libmalloc
knobs; this PR applies them through a one-time Darwin-only re-exec and
adds periodic per-worker pressure relief.

- `MallocAggressiveMadvise=1` returns freed pages eagerly.
- `MallocLargeCache=0` disables the large-allocation death-row cache.
- Operator-set allocator variables are preserved;
`HEADROOM_MALLOC_TUNING=0` is the kill switch.
- Periodic trim defaults on only for macOS, runs off the event loop,
performs no forced Python GC, validates its interval, and is
retained/cancelled through the app lifecycle.
- Non-Darwin behavior remains unchanged unless explicitly enabled.
- Semantically rebased onto current `main`, retaining startup dependency
validation, MCP SDK v1 compatibility, and all newer proxy behavior.

## Verification

- 147 proxy CLI/config/malloc/MCP-contract tests pass; 1 platform skip.
- Ruff check and formatting clean; `git diff --check` clean.
- The reporter’s macOS A/B reduced dirty empty malloc regions to zero
and lowered steady/startup RSS; the control flow and shutdown lifecycle
are covered locally.

## Safety

The re-exec is Darwin-only, PID-preserving, loop-guarded, and opt-out.
The trim task is per worker because allocator state is per process, and
shutdown cancels it explicitly.

Author: abhay-codes07

headroom/cli/proxy.py

@@ -113,6 +113,35 @@ def _get_env_bool_optional(name: str) -> bool | None:
     return _get_env_bool(name, False)
 
 
+# libmalloc reads these before main() runs, so they cannot be set from inside
+# the current process — the proxy re-execs itself once to apply them. Without
+# them, freed pages from large concurrent request bodies stay resident
+# (``vmmap`` shows whole "MALLOC_LARGE (empty)" regions) and long-lived proxy
+# RSS only ratchets upward (#2820). Vars the operator already set are left
+# untouched; HEADROOM_MALLOC_TUNING=0 disables the re-exec entirely.
+_MALLOC_TUNING = {
+    "MallocAggressiveMadvise": "1",  # madvise freed pages back to the OS eagerly
+    "MallocLargeCache": "0",  # no death-row cache for freed large allocations
+}
+
+
+def _reexec_with_malloc_tuning() -> None:
+    if sys.platform != "darwin":
+        return
+    if not _get_env_bool("HEADROOM_MALLOC_TUNING", True):
+        return
+    if os.environ.get("_HEADROOM_MALLOC_TUNED") == "1":
+        return
+    missing = {k: v for k, v in _MALLOC_TUNING.items() if k not in os.environ}
+    # Set the loop guard before the re-exec so the replacement process (which
+    # inherits this environment) skips this path instead of re-execing forever.
+    os.environ["_HEADROOM_MALLOC_TUNED"] = "1"
+    if not missing:
+        return
+    os.environ.update(missing)
+    os.execv(sys.executable, [sys.executable, "-m", "headroom.cli", *sys.argv[1:]])
+
+
 def _get_env_int_optional(name: str) -> int | None:
     val = os.environ.get(name)
     if val is None or val == "":
@@ -1065,6 +1094,7 @@ def proxy(
     Usage with OpenAI-compatible clients:
         OPENAI_BASE_URL=http://localhost:8787/v1 your-app
     """
+    _reexec_with_malloc_tuning()
     ensure_proxy_dependencies()
 
     # Import here to avoid slow startup
@@ -1261,6 +1291,10 @@ def proxy(
         rate_limit_requests_per_minute=rpm if rpm is not None else 60,
         rate_limit_tokens_per_minute=tpm if tpm is not None else 100_000,
         compress_user_messages=_get_env_bool("HEADROOM_COMPRESS_USER_MESSAGES", False),
+        periodic_malloc_trim_enabled=_get_env_bool(
+            "HEADROOM_MALLOC_TRIM", sys.platform == "darwin"
+        ),
+        malloc_trim_interval_seconds=_get_env_int("HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS", 60),
         min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", 500),
         max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", 50),
         exclude_tools=_parse_exclude_tools(None) or None,

headroom/proxy/malloc_trim.py

@@ -0,0 +1,135 @@
+"""Return freed-but-retained allocator pages to the OS on long-lived proxies.
+
+Large concurrent Anthropic bodies (0.5-1 MB of JSON parsed, deep-copied and
+re-serialized per in-flight request) drive libmalloc and pymalloc to a
+high-water mark that is never returned to the OS: after a burst the malloc
+zones keep entire regions resident but empty (``vmmap`` lists them as
+``MALLOC_LARGE (empty)`` / ``MALLOC_SMALL (empty)``), so process RSS only
+ratchets upward. Over a multi-day proxy lifetime under Claude Code traffic
+this reaches double-digit GB and starves the host.
+
+Neither runtime returns these pages on its own. macOS exposes
+``malloc_zone_pressure_relief(NULL, 0)`` to purge every zone's free pages;
+glibc has ``malloc_trim(0)``. ``trim()`` calls that entry point directly: it is
+a C call that releases the GIL and reclaims whatever is already on the
+allocator's free lists. It deliberately does not run a Python ``gc.collect()``
+-- a full cyclic collection holds the GIL, and this periodic task runs off the
+event-loop thread precisely so it cannot stall request handling; freeing cyclic
+garbage is left to CPython's own automatic collection.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ctypes
+import logging
+import sys
+import time
+
+logger = logging.getLogger(__name__)
+
+# Interval bounds for the periodic trim task. A non-positive interval would make
+# ``asyncio.sleep`` return immediately and spin a continuous collect/trim loop,
+# so anything below the minimum falls back to the default.
+_DEFAULT_TRIM_INTERVAL_SECONDS = 60
+_MIN_TRIM_INTERVAL_SECONDS = 1
+
+# Lazily resolved (platform_tag, foreign_function | None). ``None`` function
+# means the platform has no supported trim call and trim() is a no-op.
+_relief: tuple[str, object | None] | None = None
+
+
+def _resolve() -> tuple[str, object | None]:
+    global _relief
+    if _relief is not None:
+        return _relief
+    try:
+        libc = ctypes.CDLL(None)
+        if sys.platform == "darwin":
+            fn = libc.malloc_zone_pressure_relief
+            fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t]
+            fn.restype = ctypes.c_size_t
+            _relief = ("darwin", fn)
+        else:
+            fn = libc.malloc_trim
+            fn.argtypes = [ctypes.c_size_t]
+            fn.restype = ctypes.c_int
+            _relief = ("glibc", fn)
+    except (OSError, AttributeError):
+        _relief = ("unsupported", None)
+    return _relief
+
+
+def trim() -> int:
+    """Return allocator free pages to the OS.
+
+    Calls the platform's allocator pressure-relief entry point
+    (``malloc_zone_pressure_relief`` on macOS, ``malloc_trim`` on glibc). This
+    is a C call that releases the GIL for its duration and reclaims pages
+    already on the allocator's free lists. It deliberately does *not* run a
+    Python ``gc.collect()`` (a full cyclic collection holds the GIL); cyclic
+    garbage is left to CPython's automatic collection, so this off-thread
+    periodic task never holds the GIL for a full-heap traversal.
+
+    Returns the number of bytes freed on macOS (glibc's ``malloc_trim``
+    reports only success, so 0 is returned there and on unsupported
+    platforms).
+    """
+    kind, fn = _resolve()
+    if fn is None:
+        return 0
+    if kind == "darwin":
+        return int(fn(None, 0))  # type: ignore[operator]
+    fn(0)  # type: ignore[operator]
+    return 0
+
+
+async def trim_periodically(interval_seconds: int = 60) -> None:
+    """Background task that periodically returns allocator free pages to the OS.
+
+    Runs in every worker process (allocator state is per-process). The trim is
+    the platform's allocator pressure-relief C call
+    (``malloc_zone_pressure_relief``/``malloc_trim``), dispatched via
+    ``asyncio.to_thread`` so it runs off the event-loop thread. Because it is a
+    C call that releases the GIL and runs no Python ``gc.collect()``, it holds
+    the GIL only as briefly as the to_thread hand-off, so a slow purge on a
+    large heap does not stall request handling. The task exits immediately on
+    platforms with no supported trim call, so it is a true no-op there.
+
+    Args:
+        interval_seconds: How often to trim (default: 60 seconds). A value below
+            ``_MIN_TRIM_INTERVAL_SECONDS`` (which would busy-loop) falls back to
+            the default.
+    """
+    _, fn = _resolve()
+    if fn is None:
+        # No supported allocator-trim call on this platform (Windows, musl, ...);
+        # do not spin a wakeup task that can only ever no-op.
+        logger.debug("MallocTrim: no supported trim on %s; task disabled", sys.platform)
+        return
+
+    if interval_seconds < _MIN_TRIM_INTERVAL_SECONDS:
+        logger.warning(
+            "MallocTrim: interval %ss is below the %ds minimum; using default %ds",
+            interval_seconds,
+            _MIN_TRIM_INTERVAL_SECONDS,
+            _DEFAULT_TRIM_INTERVAL_SECONDS,
+        )
+        interval_seconds = _DEFAULT_TRIM_INTERVAL_SECONDS
+
+    while True:
+        await asyncio.sleep(interval_seconds)
+        try:
+            start = time.perf_counter()
+            # Off the event-loop thread: the C-level purge can pause for a while
+            # on a large heap, and that pause must not stall proxy traffic.
+            freed = await asyncio.to_thread(trim)
+            elapsed_ms = (time.perf_counter() - start) * 1000
+            log = logger.info if freed >= (16 << 20) else logger.debug
+            log(
+                "MallocTrim: returned %.1f MB to OS in %.0f ms",
+                freed / 1048576,
+                elapsed_ms,
+            )
+        except Exception as e:
+            logger.debug("MallocTrim failed: %s", e)

headroom/proxy/models.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import logging
+import sys
 from dataclasses import InitVar, dataclass, field
 from datetime import datetime
 from typing import Any, Literal
@@ -439,6 +440,17 @@ class ProxyConfig:
     # Env: HEADROOM_PERIODIC_TOIN_STATS=0.
     periodic_toin_stats_enabled: bool = True
 
+    # Periodic allocator trim. Long-lived proxies processing large concurrent
+    # request bodies ratchet RSS through freed-but-retained allocator pages;
+    # this returns them to the OS (malloc_zone_pressure_relief on macOS,
+    # malloc_trim on glibc). Default-on only on macOS, where the retained-page
+    # ratchet is the documented failure (#2820); an opt-in elsewhere via
+    # HEADROOM_MALLOC_TRIM=1 so glibc deployments do not silently take on a
+    # once-a-minute allocator purge they did not ask for. Envs:
+    # HEADROOM_MALLOC_TRIM=0/1, HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS.
+    periodic_malloc_trim_enabled: bool = field(default_factory=lambda: sys.platform == "darwin")
+    malloc_trim_interval_seconds: int = 60
+
     # Stateless mode — disable all filesystem writes for read-only / container deployments
     stateless: bool = False
 

headroom/proxy/server.py

@@ -144,6 +144,7 @@
 )
 from headroom.proxy.loop_callback_failure_policy import is_known_websocket_callback_failure
 from headroom.proxy.loopback_guard import is_loopback_host
+from headroom.proxy.malloc_trim import trim_periodically
 from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
 
 # Data models (extracted to headroom/proxy/models.py for maintainability)
@@ -2601,6 +2602,7 @@ async def lifespan(app: FastAPI):  # type: ignore[no-untyped-def]
         app.state.ready = False
         app.state.startup_error = None
         app.state.periodic_toin_stats_task = None
+        app.state.periodic_malloc_trim_task = None
 
         try:
             try:
@@ -2611,6 +2613,12 @@ async def lifespan(app: FastAPI):  # type: ignore[no-untyped-def]
                     app.state.periodic_toin_stats_task = asyncio.create_task(
                         _log_toin_stats_periodically()
                     )
+                # Per-worker on purpose: allocator state is per-process, so
+                # every worker must trim its own zones (no beacon-owner gate).
+                if config.periodic_malloc_trim_enabled:
+                    app.state.periodic_malloc_trim_task = asyncio.create_task(
+                        trim_periodically(config.malloc_trim_interval_seconds)
+                    )
                 if proxy.usage_reporter:
                     await proxy.usage_reporter.start(proxy)
                 if proxy.traffic_learner:
@@ -2670,6 +2678,16 @@ async def _timed(coro: Any, *, label: str, timeout: float) -> None:
                 )
                 app.state.periodic_toin_stats_task = None
 
+            periodic_malloc_trim_task = app.state.periodic_malloc_trim_task
+            if periodic_malloc_trim_task is not None:
+                periodic_malloc_trim_task.cancel()
+                await _timed(
+                    asyncio.gather(periodic_malloc_trim_task, return_exceptions=True),
+                    label="periodic_malloc_trim.stop",
+                    timeout=3.0,
+                )
+                app.state.periodic_malloc_trim_task = None
+
             if _cc_reconciler is not None:
                 await _timed(_cc_reconciler.stop(), label="cc_reconciler.stop", timeout=3.0)
             if _beacon_is_owner[0]:
@@ -5160,6 +5178,10 @@ def _proxy_config_from_env() -> ProxyConfig:
         http2=_get_env_bool("HEADROOM_HTTP2", True),
         http_proxy=os.environ.get("HEADROOM_HTTP_PROXY") or None,
         periodic_toin_stats_enabled=_get_env_bool("HEADROOM_PERIODIC_TOIN_STATS", True),
+        periodic_malloc_trim_enabled=_get_env_bool(
+            "HEADROOM_MALLOC_TRIM", sys.platform == "darwin"
+        ),
+        malloc_trim_interval_seconds=_get_env_int("HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS", 60),
         proxy_token=os.environ.get("HEADROOM_PROXY_TOKEN") or None,
         offline=_get_env_bool("HEADROOM_OFFLINE", False),
         # Default mode is CACHE (Headroom's coding posture): delta-only compression