fix(tests): add GIL-proof watchdog and restore logger global state

Two Group 5 release-CI fixes for the py3.13 freeze-at-98% (run 31235996603,
both retry attempts):

- tests/conftest.py: arm faulthandler.dump_traceback_later(120s, exit=True)
  around each test. pytest-timeout's thread method (timeout=90) needs the GIL
  to fire, so a C-level hang that holds the GIL freezes the worker silently
  until the 50-min step wall; faulthandler's C watchdog needs no GIL, dumps
  every thread's stack, and hard-exits the worker so xdist replaces it and
  --reruns retries the test. The stderr fd is dup'd at pytest_configure time
  because pytest's fd-level capture otherwise swallows the dump (which is why
  the 3.12 thread dumps never reached CI logs). LANGFLOW_TEST_HARD_TIMEOUT=0
  disables it for debugger sessions.

- tests/unit/test_logger.py: module-scoped autouse fixture restoring the
  process-global logging state the module rewires (structlog config, root
  handlers/level, named-logger levels, lfx file-handler global). In every
  observed freeze the wedged worker had just finished this module and hung in
  its next full-app test (test_login.py::
  test_session_endpoint_rejects_expired_external_token).

Author: erichare

.secrets.baseline

@@ -1088,15 +1088,15 @@
         "filename": "src/backend/tests/conftest.py",
         "hashed_secret": "8bb6118f8fd6935ad0876a3be34a717d32708ffd",
         "is_verified": false,
-        "line_number": 559,
+        "line_number": 608,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "src/backend/tests/conftest.py",
         "hashed_secret": "61fbb5a12cd7b1f1fe1624120089efc0cd299e43",
         "is_verified": false,
-        "line_number": 769,
+        "line_number": 818,
         "is_secret": false
       }
     ],
@@ -7242,5 +7242,5 @@
       }
     ]
   },
-  "generated_at": "2026-08-05T23:30:16Z"
+  "generated_at": "2026-08-08T15:38:52Z"
 }

pyproject.toml

@@ -249,6 +249,9 @@ ignore-regex = '.*(Stati Uniti|Tense=Pres).*'
 # 50-min step wall. A 90s cap fires below the ~127s stall, and timeout_method="thread" dumps ALL
 # thread stacks (faulthandler) — naming the exact blocking call even when it is a leaked background
 # task. Revert to timeout=150 / timeout_method="signal" once the stall is identified and bounded.
+# NOTE: the thread method's timer needs the GIL, so a C-level hang that holds the GIL defeats it
+# (observed: release-1.11.3 py3.13 Group 5 froze 25+ min with no dump). A second, GIL-proof
+# faulthandler watchdog (120s, dump + exit) backstops it in src/backend/tests/conftest.py.
 timeout = 90
 timeout_method = "thread"
 minversion = "6.0"

src/backend/tests/conftest.py

@@ -1,7 +1,9 @@
 import asyncio
+import faulthandler
 import json
 import os
 import shutil
+import sys
 
 # we need to import tmpdir
 import tempfile
@@ -165,7 +167,54 @@ def blockbuster(request):
             yield bb
 
 
+# Hard, GIL-proof per-test watchdog. pytest-timeout's thread method (pyproject
+# `timeout = 90`) arms a *Python* timer thread, which needs the GIL to run its
+# callback -- a hang inside a C call that never releases the GIL (observed on
+# the release-1.11.3 py3.13 Group 5 job: a worker froze for 25+ minutes in
+# test_login.py::test_session_endpoint_rejects_expired_external_token with no
+# dump) silently defeats it and the job burns to the CI step wall.
+# faulthandler.dump_traceback_later() instead uses a C-level watchdog thread
+# that needs no GIL: it dumps every thread's stack to the real stderr fd
+# (inherited by xdist workers, so it lands in the CI log) and, with exit=True,
+# hard-exits the wedged worker -- xdist then reports "node down", replaces the
+# worker, and --reruns retries the test on the fresh one. The 120s default sits
+# above pytest-timeout's 90s so the soft watchdog (clean per-test failure)
+# always gets first shot; this only fires when that one *couldn't* run.
+# Disable locally for debugger sessions with LANGFLOW_TEST_HARD_TIMEOUT=0.
+_HARD_TIMEOUT_S = float(os.getenv("LANGFLOW_TEST_HARD_TIMEOUT", "120"))
+
+# Real-stderr fd, dup'd at pytest_configure time. pytest's fd-level capture
+# redirects fd 2 into a per-test temp file that is discarded when the process
+# hard-exits, so a dump armed against sys.__stderr__ at *test* time vanishes
+# (which is also why pytest-timeout's thread dumps never showed in CI logs).
+# At configure time fd 2 still points at the process's original stderr -- in an
+# xdist worker that fd is inherited from the controller, so dumps written to
+# the dup land in the CI step log. Same strategy as pytest's builtin
+# faulthandler plugin.
+_watchdog_stderr_fd: int | None = None
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_runtest_protocol(item, nextitem):  # noqa: ARG001
+    if _HARD_TIMEOUT_S <= 0 or _watchdog_stderr_fd is None:
+        return (yield)
+    faulthandler.dump_traceback_later(_HARD_TIMEOUT_S, file=_watchdog_stderr_fd, exit=True)
+    try:
+        return (yield)
+    finally:
+        faulthandler.cancel_dump_traceback_later()
+
+
 def pytest_configure(config):
+    global _watchdog_stderr_fd  # noqa: PLW0603
+    if _HARD_TIMEOUT_S > 0 and _watchdog_stderr_fd is None:
+        with suppress(AttributeError, ValueError, OSError):
+            try:
+                fd = sys.stderr.fileno()
+            except (AttributeError, ValueError, OSError):
+                fd = sys.__stderr__.fileno()
+            _watchdog_stderr_fd = os.dup(fd)
+
     config.addinivalue_line("markers", "noclient: don't create a client for this test")
     config.addinivalue_line("markers", "load_flows: load the flows for this test")
     config.addinivalue_line("markers", "api_key_required: run only if the api key is set in the environment variables")

src/backend/tests/unit/test_logger.py

@@ -11,6 +11,7 @@
 
 import builtins
 import contextlib
+import importlib
 import json
 import logging
 import os
@@ -35,6 +36,67 @@
 from loguru import logger as loguru_logger
 
 
+@pytest.fixture(scope="module", autouse=True)
+def _restore_process_logging_state():
+    """Restore every process-global logging object this module mutates.
+
+    The tests here call ``configure()`` dozens of times and edit
+    ``logging.root.handlers`` directly, which rewires *process-wide* state:
+    the structlog config, stdlib root handlers (InterceptHandler /
+    RotatingFileHandler pointed at since-deleted temp dirs), named-logger
+    levels (via LANGFLOW_LOG_LEVELS), and the ``lfx.log.logger`` module
+    globals. Under xdist that poisoned state leaks into whatever test the
+    worker picks up next -- observed on the release-1.11.3 CI run, where the
+    worker that had just finished this module hung indefinitely inside the
+    next test needing the full-app ``client`` fixture
+    (test_login.py::test_session_endpoint_rejects_expired_external_token),
+    freezing the whole Group 5 job to the 50-min step wall on py3.13. This
+    fixture snapshots the pre-module state and puts it back at module exit.
+    """
+    # ``import lfx.log.logger as ...`` would bind the *logger object* that
+    # ``lfx.log``'s ``__init__`` re-exports under the same name, not the module.
+    _lfx_log = importlib.import_module("lfx.log.logger")
+
+    orig_structlog_config = dict(structlog.get_config())
+    orig_root_handlers = logging.root.handlers[:]
+    orig_root_level = logging.root.level
+    orig_file_handler = _lfx_log._file_handler
+    orig_logger_levels = {
+        name: lg.level for name, lg in logging.Logger.manager.loggerDict.items() if isinstance(lg, logging.Logger)
+    }
+
+    yield
+
+    # structlog: reinstall the exact pre-module config (processor chain,
+    # wrapper_class -- which also carries configure()'s change fingerprint).
+    structlog.configure(**orig_structlog_config)
+
+    # stdlib root: drop handlers the module added (closing file handlers so
+    # they stop pointing into deleted temp dirs), then restore the original
+    # handler list and level. If the module's configure() calls replaced the
+    # lfx-managed file handler, the original one was already closed by
+    # setup_log_file(), so it must not be reinstalled.
+    current_file_handler = _lfx_log._file_handler
+    for handler in logging.root.handlers[:]:
+        if handler not in orig_root_handlers:
+            logging.root.removeHandler(handler)
+            if isinstance(handler, logging.handlers.RotatingFileHandler):
+                with contextlib.suppress(OSError, ValueError):
+                    handler.close()
+    restored_handlers = orig_root_handlers
+    if current_file_handler is not orig_file_handler:
+        restored_handlers = [h for h in orig_root_handlers if h is not orig_file_handler]
+        _lfx_log._file_handler = None
+    logging.root.handlers[:] = restored_handlers
+    logging.root.setLevel(orig_root_level)
+
+    # Named loggers whose levels configure() changed via LANGFLOW_LOG_LEVELS.
+    for name, level in orig_logger_levels.items():
+        lg = logging.Logger.manager.loggerDict.get(name)
+        if isinstance(lg, logging.Logger) and lg.level != level:
+            lg.setLevel(level)
+
+
 class TestConfigure:
     """Test suite for the configure() function."""