cache models inbetween test runs

.github/workflows/ci.yml

@@ -1,11 +1,27 @@
 name: CI
 
 on:
+  # Only run the (expensive) test matrix when files that can affect the tests or
+  # package change. Docs, license, images and other non-code changes are skipped.
+  # This is an allowlist: anything test-relevant added later (e.g. a new data dir
+  # or a root conftest.py) must be added here or CI will silently skip it.
   push:
     branches: ["main"]
+    paths:
+      - "src/**"                    # birdnet, birdnet_tests (incl. TEST_FILES), birdnet_benchmark
+      - "example/**"                # test data: test_issue29/55 load example/soundscape.wav
+      - "pyproject.toml"            # deps + tox + pytest/ruff/mypy config
+      - "MANIFEST.in"               # packaging -> build/twine check step
+      - ".github/workflows/ci.yml"  # the workflow itself
 
   pull_request:
     branches: ["main"]
+    paths:
+      - "src/**"
+      - "example/**"
+      - "pyproject.toml"
+      - "MANIFEST.in"
+      - ".github/workflows/ci.yml"
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -21,12 +37,28 @@ jobs:
   run-tests:
     runs-on: ${{ matrix.os }}
     continue-on-error: false
-    # Backstop for a wedged run: the fork/TensorFlow deadlock hangs in native code
-    # holding the GIL, which pytest-timeout cannot interrupt in either method, so the
-    # job-level limit is what stops it. Slowest healthy job is ~32 min; 50 leaves
-    # headroom while capping a hang's cost.
+    # Backstop for a wedged job. The library defaults to the "spawn" start method
+    # (see birdnet.core.start_method), which removes the fork-after-TensorFlow
+    # deadlock from the general suite; the isolated fork lanes can still burn up to
+    # the 600 s per-test timeout each when that inherent deadlock fires. Budget is
+    # set by the fattest cell, ubuntu 3.12, which runs three tox envs (py312 +
+    # py312-repro + py312-coverage, ~45 min healthy): 40 was proven too tight when
+    # a repro fork-lane deadlock pushed it over.
     timeout-minutes: 50
 
+    env:
+      # Fixed, cacheable location for birdnet's model/label downloads so the ~11 min
+      # "load_model" download step is skipped on a cache hit. Placed under the
+      # workspace because the `runner` context is not available in job-level env;
+      # this is safe since checkout runs once, before the cache is restored, and is
+      # never re-run. tox forwards this var to the pytest subprocess via `passenv`.
+      BIRDNET_APP_DATA: ${{ github.workspace }}/.birdnet-app-data
+      # Enables the GIL-immune hang watchdog (see birdnet_tests/conftest.py): a
+      # wedged test that survives pytest-timeout gets its thread stacks dumped
+      # here and the process hard-killed shortly after its timeout, instead of
+      # silently burning the job budget. Uploaded as an artifact on failure.
+      BIRDNET_TEST_WATCHDOG_DIR: ${{ github.workspace }}/.watchdog-dumps
+
     strategy:
       fail-fast: false
       matrix:
@@ -73,10 +105,31 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # Provides the `uv` binary and persists uv's wheel cache across runs, so the
+      # large TensorFlow/torch/cuda downloads are not repeated every run.
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
+      # Cache the downloaded models/labels. The files are identical across OS and
+      # Python, so we key only on the download *profile* (full vs the TF-free py3.14
+      # surface) plus the model definitions (their URLs/sizes). This keeps the cache
+      # to ~2 entries (~3.6 GB) instead of one ~3 GB copy per (os, python), which
+      # overran the 10 GB repo cache limit and thrashed. A changed definition busts
+      # the key; restore-keys then falls back to the last cache of the same profile
+      # and only the stale models are re-downloaded.
+      - name: Cache downloaded models
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.BIRDNET_APP_DATA }}
+          key: birdnet-models-${{ matrix.python-version == '3.14' && 'notf' || 'full' }}-${{ hashFiles('src/birdnet/**/models/**/*.py') }}
+          restore-keys: |
+            birdnet-models-${{ matrix.python-version == '3.14' && 'notf' || 'full' }}-
+
       - name: Install dependencies
-        run: |
-          python -m pip install uv --upgrade
-          python -m uv pip install --system .[tests]
+        run: uv pip install --system '.[tests]'
 
       - name: Run tests py311
         if: matrix.python-version == '3.11'
@@ -148,3 +201,24 @@ jobs:
         with:
           name: birdnet-log
           path: ${{ steps.extract-log-path.outputs.log_paths }}
+
+      # Diagnostic for silent test-process deaths: rules the kernel OOM killer
+      # in or out directly in the job log. (Checked for the fork-lane wedges:
+      # no OOM kills logged — those are the inherent fork-after-TF child
+      # deadlock, ended by pytest-timeout at the per-test timeout.)
+      - name: Check for OOM kills (Linux)
+        if: failure() && startsWith(matrix.os, 'ubuntu')
+        run: sudo dmesg | grep -iE "out of memory|oom-kill|killed process" || echo "no OOM kills logged"
+
+      # Stack dumps written by the hang watchdog (see BIRDNET_TEST_WATCHDOG_DIR
+      # above); on a wedge they show every thread's stack at kill time.
+      # include-hidden-files is required: upload-artifact skips dotfiles by
+      # default, and the dump dir is hidden (.watchdog-dumps).
+      - name: Upload hang watchdog dumps
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: watchdog-dumps-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: .watchdog-dumps/
+          if-no-files-found: ignore
+          include-hidden-files: true

docs/general.rst

@@ -30,6 +30,33 @@ A *Producer* loads only as much audio as the buffer can hold, keeping RAM usage
 * **Model Backends** – Each worker loads its own instance of the inference model. On the CPU, both **TFLite** and **Protocol Buffers** (Protobuf) models can be used; Protobuf models can optionally run on the GPU.
 * **Best Practice for CPU Inference** – For CPU-only execution on Linux, the number of *Worker* processes should not exceed the number of physical cores, as oversubscription typically leads to reduced performance. When running TFLite, keep the batch size to one (1); larger batches offer no throughput benefit.
 
+Multiprocessing start method
+----------------------------
+
+The pipeline creates its processes with the ``spawn`` start method by default on
+**all** platforms — it does *not* inherit Linux's platform default of ``fork``.
+Forking a process after TensorFlow has started its multi-threaded runtime can
+deadlock the child, so a plain ``model.predict(...)`` is safe out of the box
+even when TensorFlow is already loaded.
+
+The default can be overridden, in order of precedence:
+
+1. Set the ``BIRDNET_START_METHOD`` environment variable to ``spawn``,
+   ``forkserver`` or ``fork``.
+2. Fix the method globally in your application **before** using birdnet, e.g.
+   ``multiprocessing.set_start_method("fork")`` — an explicitly chosen method
+   is always honored.
+
+With ``fork`` (explicit opt-in), workers inherit a model loaded in the parent
+process via copy-on-write, which avoids per-worker model loading — but you are
+responsible for calling the pipeline before TensorFlow spawns threads, or
+accepting the deadlock risk. With ``spawn`` or ``forkserver``, each worker
+loads its model itself.
+
+Because the default is ``spawn``, the standard Python rule for scripts applies
+on every platform (it always did on macOS and Windows): entry-point code must
+be guarded with ``if __name__ == "__main__":``.
+
 Known limitations
 ----
 

pyproject.toml

@@ -70,7 +70,11 @@ repro = [
   "pandas == 2.3.3",
   "psutil == 7.1.3",
   "pyarrow == 22.0.0",
-  "tensorflow == 2.20.0 ; sys_platform == 'linux' or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'windows' and platform_machine == 'amd64')",
+  # 2.20.0 wedged indefinitely inside native eager execution on macOS arm64 when a
+  # process mixed tflite-interpreter, keras and SavedModel use under parallel test
+  # load (worker stack: quick_execute never returning during a pb predict). 2.21.0
+  # with the identical remaining pins does not hang; only h5py moved with it.
+  "tensorflow == 2.21.0 ; sys_platform == 'linux' or (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'windows' and platform_machine == 'amd64')",
   "ai-edge-litert == 2.0.3 ; python_version < '3.13' and sys_platform == 'linux'",
   # Indirect dependencies
   "absl-py == 2.3.1",
@@ -83,7 +87,7 @@ repro = [
   "gast == 0.6.0",
   "google-pasta == 0.2.0",
   "grpcio == 1.76.0",
-  "h5py == 3.15.1",
+  "h5py == 3.14.0", # resolved by tensorflow 2.21.0
   "idna == 3.11",
   "keras == 3.12.0",
   "libclang == 18.1.1",
@@ -182,7 +186,13 @@ namespaces = true
 [tool.pytest.ini_options]
 log_cli = true
 log_level = "DEBUG"
-timeout = 300
+# Hang backstop, not a speed budget. 600 because the slowest healthy tests (TF
+# SavedModel "pb" loads, acoustic v3 onnx predict) take ~140-250 s on an idle dev
+# machine and CI runners are slower and run several xdist workers concurrently:
+# at 300 they were killed at exactly 300 s under contention, which the thread
+# timeout method turns into an opaque xdist "node down: Not properly terminated"
+# worker crash. A genuine hang still dies here long before the job-level limit.
+timeout = 600
 # The signal method cannot interrupt a thread blocked in a C call, so a deadlocked
 # fork child let the run hang until the job limit killed it hours later. The thread
 # method kills the process instead, turning such hangs into a failure with a stack
@@ -197,6 +207,7 @@ markers = [
   "gpu: tests requiring a GPU to run and to be run sequentially",
   "repro: tests requiring exact package versions to reproduce results",
   "no_tf: tests for the TensorFlow-free surface, run when TF is absent (e.g. Python 3.14)",
+  "fork: tests that force the fork start method; run in a dedicated serial (-n 1) lane in a fresh process so a fork-after-TensorFlow deadlock cannot wedge the parallel general-phase workers (the library itself defaults to spawn, see birdnet.core.start_method)",
 ]
 
 [tool.ruff]
@@ -248,6 +259,12 @@ package = wheel
 allowlist_externals = uv, pyproject-build
 install_command = uv pip install {opts} {packages}
 skip_missing_interpreters = true
+# Forward the model-cache location and the hang-watchdog dir (set in CI) into
+# the test subprocess; tox strips unlisted env vars, which would defeat the
+# model cache and silently disable the watchdog.
+passenv =
+  BIRDNET_APP_DATA
+  BIRDNET_TEST_WATCHDOG_DIR
 
 deps = 
   .[tests,and-cuda,pt,onnx]
@@ -256,9 +273,20 @@ commands =
   # ruff check src/birdnet_tests
   # mypy src/birdnet
   pytest -m "not repro and load_model" -n auto
-  pytest -m "not repro and not load_model and (not litert and not gpu)" -n auto
-  pytest -m "not repro and not load_model and litert" -n auto
+  pytest -m "not repro and not load_model and (not litert and not gpu and not fork)" -n auto
+  pytest -m "not repro and not load_model and litert and not fork" -n auto
   pytest -m "not repro and not load_model and gpu" -n 1
+  # Fork tests run last, serially, in a fresh process (see the `fork` marker and
+  # conftest.py). A fork-after-TensorFlow deadlock here can only wedge this isolated
+  # lane, not the parallel general phase, whose results are already recorded. The
+  # litert/non-litert split mirrors the phases above (litert can't load after TF).
+  # -n 0 (in-process, no xdist worker) on purpose: when a forked child wedges and
+  # pytest-timeout kills the test process, an xdist worker's timeout report is lost
+  # with the worker, and the orphaned child keeps the worker's execnet socket open
+  # so the controller sits blind until the watchdog's idle deadline. In-process,
+  # the timeout report lands in the log and the run ends at the test timeout.
+  pytest -m "not repro and not load_model and fork and not litert and not gpu" -n 0
+  pytest -m "not repro and not load_model and fork and litert and not gpu" -n 0
   # build and check package
   # note `python -m build` uses the isolated build folder which results in import error:
   # 'No module named build.__main__; 'build' is a package and cannot be directly executed'
@@ -271,9 +299,12 @@ deps =
 commands = 
   pytest -m "load_model" -n auto
   # run normal tests including repro tests
-  pytest -m "not load_model and (not litert and not gpu)" -n auto
+  pytest -m "not load_model and (not litert and not gpu and not fork)" -n auto
   # run litert tests including repro tests
-  pytest -m "not load_model and litert" -n auto
+  pytest -m "not load_model and litert and not fork" -n auto
+  # fork tests isolated + serial (see the `fork` marker and conftest.py)
+  pytest -m "not load_model and fork and not litert and not gpu" -n 0
+  pytest -m "not load_model and fork and litert and not gpu" -n 0
   # gpu tests are not supported to be run with repro environment
   pyproject-build -o dist/
   python -m twine check dist/*
@@ -294,9 +325,12 @@ deps =
 commands =
   coverage erase
   pytest -m "not repro and load_model" -n auto --cov=src/birdnet --cov-append --cov-report=
-  pytest -m "not repro and not load_model and (not litert and not gpu)" -n auto --cov=src/birdnet --cov-append --cov-report=
-  pytest -m "not repro and not load_model and litert" -n auto --cov=src/birdnet --cov-append --cov-report=
+  pytest -m "not repro and not load_model and (not litert and not gpu and not fork)" -n auto --cov=src/birdnet --cov-append --cov-report=
+  pytest -m "not repro and not load_model and litert and not fork" -n auto --cov=src/birdnet --cov-append --cov-report=
   pytest -m "not repro and not load_model and gpu" -n 1 --cov=src/birdnet --cov-append --cov-report=
+  # fork tests isolated + serial (see the `fork` marker and conftest.py)
+  pytest -m "not repro and not load_model and fork and not litert and not gpu" -n 0 --cov=src/birdnet --cov-append --cov-report=
+  pytest -m "not repro and not load_model and fork and litert and not gpu" -n 0 --cov=src/birdnet --cov-append --cov-report=
   coverage report
 """
 

src/birdnet/acoustic/inference/configs.py

@@ -2,7 +2,7 @@
 
 import multiprocessing
 from collections.abc import Callable, Collection, Iterable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal, TypeVar
 
@@ -15,6 +15,7 @@
 from birdnet.acoustic.inference.core.tensor import AcousticTensorBase
 from birdnet.core.backends import VersionedBackendProtocol
 from birdnet.core.base import ResultBase
+from birdnet.core.start_method import resolve_start_method
 from birdnet.globals import ACOUSTIC_MODEL_VERSIONS
 from birdnet.utils.helper import (
   SF_FORMATS,
@@ -339,6 +340,12 @@ class InferenceConfig:
   processing_conf: ProcessingConfig
   filtering_conf: FilteringConfig
   output_conf: OutputConfig
+  # Effective multiprocessing start method for every process, queue and
+  # synchronization primitive of this session's pipeline (see
+  # resolve_start_method for the resolution rules). Resolved once at config
+  # creation so parent and children always agree on it; stored as a string so
+  # the config stays picklable.
+  start_method: str = field(default_factory=resolve_start_method)
 
   @classmethod
   def validate_input_audio(

src/birdnet/acoustic/inference/core/benchmarking.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import inspect
-import multiprocessing as mp
 import platform
 from collections import OrderedDict
 from dataclasses import asdict, dataclass
@@ -168,9 +167,9 @@ def hw_cpu_logical_cores(self) -> int:
   def hw_ram_GiB(self) -> float:
     return psutil.virtual_memory().total / 1024**3
 
-  @property
-  def sw_start_method(self) -> str:
-    return mp.get_start_method()
+  # The session's effective start method (see InferenceConfig.start_method);
+  # the global mp.get_start_method() may differ from what the pipeline used.
+  sw_start_method: str
 
   # Software
   @property

src/birdnet/acoustic/inference/core/encoding/encoding_worker.py

@@ -38,6 +38,7 @@ def __init__(
     start_signal: Event,
     finish_signal: Event,
     end_event: Event,
+    start_method: str,
   ) -> None:
     super().__init__(
       session_id=session_id,
@@ -66,6 +67,7 @@ def __init__(
       start_signal=start_signal,
       finish_signal=finish_signal,
       end_event=end_event,
+      start_method=start_method,
     )
 
   def _get_block(

src/birdnet/acoustic/inference/core/logs.py

@@ -1,6 +1,4 @@
 import logging
-import multiprocessing as mp
-import multiprocessing.synchronize
 from logging.handlers import QueueHandler
 from multiprocessing import Queue
 
@@ -63,23 +61,30 @@ def __init__(
     name: str,
     logging_queue: Queue,
     logging_level: int,
+    start_method: str,
   ) -> None:
     self.__logger: logging.Logger | None = None
     self.__logging_queue = logging_queue
     self.__logging_level = logging_level
     self.__local_queue_handler: QueueHandler | None = None
     self.__name = name
     self.__session_id = session_id
+    # The session's effective start method is passed in instead of read from
+    # mp.get_start_method(): the global default can differ from the context the
+    # pipeline actually uses (e.g. library default "forkserver" while the
+    # global still says "fork"), and these branches must match the context the
+    # process was really created with.
+    self.__start_method = start_method
     self._session_hash = get_session_id_hash(session_id)
 
   def _init_logging(self) -> None:
-    if mp.get_start_method() in ("spawn", "forkserver"):
+    if self.__start_method in ("spawn", "forkserver"):
       init_session_logger(self.__session_id, self.__logging_level)
       self.__local_queue_handler = add_session_queue_handler(
         self.__session_id, self.__logging_queue
       )
     else:
-      assert mp.get_start_method() == "fork"
+      assert self.__start_method == "fork"
       assert session_queue_handler_exists(self.__session_id, self.__logging_queue)
     self.__logger = get_logger_from_session(self.__session_id, self.__name)
     self.__logger.debug(
@@ -91,11 +96,11 @@ def _uninit_logging(self) -> None:
     self.__logger.debug(
       f"Uninitializing logging for session {self._session_hash} -> {self.__name}."
     )
-    if mp.get_start_method() in ("spawn", "forkserver"):
+    if self.__start_method in ("spawn", "forkserver"):
       assert self.__local_queue_handler is not None
       remove_session_queue_handler(self.__session_id, self.__local_queue_handler)
     else:
-      assert mp.get_start_method() == "fork"
+      assert self.__start_method == "fork"
       assert self.__local_queue_handler is None
     self.__local_queue_handler = None
     self.__logger = None

src/birdnet/acoustic/inference/core/perf_tracker.py

@@ -136,8 +136,9 @@ def __init__(
     start_signal: Event,
     finish_signal: Event,
     start: float,
+    start_method: str,
   ) -> None:
-    super().__init__(session_id, __name__, logging_queue, logging_level)
+    super().__init__(session_id, __name__, logging_queue, logging_level, start_method)
 
     n_last_batch_stats = 10
     n_last_seconds_live_stats = 5.0