Merge remote-tracking branch 'origin/main' into geomodel-3.0

Author: Josef-Haupt

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.16] - 2026-05-09
+
+### Added
+
+- Add support for overriding BirdNET’s application-data directory via an environment variable `BIRDNET_APP_DATA`, enabling users to place downloaded models/benchmarks in a custom location (useful for deployments with restricted home directories or shared storage).
+
+### Bugfixes
+
+- Fixed acoustic inference session being aborted on macOS when stats were enabled: hardened parent/child memory tracking against `psutil.AccessDenied`, and replaced the two tracked semaphores with a wrapper that mirrors the count into shared memory so `get_value()` works on macOS (#39)
+- Fixed float16 quantization of segment timestamps in prediction results, which caused up to ±0.05 s drift in CSV/DataFrame/Parquet output (#38, #42). Also closed an analogous hole in encoding results where a hop duration that is exactly representable in float16 (e.g. hop=1.5) could still produce drifting accumulated timestamps. Timestamps are now always materialized at >= float32 precision at the source.
+
 ## [0.2.15] - 2026-05-02
 
 ### Bugfixes
@@ -264,7 +275,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial release
 
-[Unreleased]: https://github.com/birdnet-team/birdnet/compare/v0.2.15...HEAD
+[Unreleased]: https://github.com/birdnet-team/birdnet/compare/v0.2.16...HEAD
+[0.2.16]: https://github.com/birdnet-team/birdnet/compare/v0.2.15...v0.2.16
 [0.2.15]: https://github.com/birdnet-team/birdnet/compare/v0.2.14...v0.2.15
 [0.2.14]: https://github.com/birdnet-team/birdnet/compare/v0.2.13...v0.2.14
 [0.2.13]: https://github.com/birdnet-team/birdnet/compare/v0.2.12...v0.2.13

docs/conf.py

@@ -9,7 +9,7 @@
 project = "birdnet"
 copyright = "2026, Stefan Taubert"
 author = "Stefan Taubert"
-release = "0.2.15"
+release = "0.2.16"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

docs/general.rst

@@ -29,3 +29,25 @@ A *Producer* loads only as much audio as the buffer can hold, keeping RAM usage
 * **Buffer Size** – By default, the buffer is set to twice the *Worker* count, ensuring that every *Worker* always has a pre-loaded batch to process and thus avoids idle time.
 * **Model Backends** – Each worker loads its own instance of the inference model. On the CPU, both **TFLite** and **Protocol Buffers** (Protobuf) models can be used; Protobuf models can optionally run on the GPU.
 * **Best Practice for CPU Inference** – For CPU-only execution on Linux, the number of *Worker* processes should not exceed the number of physical cores, as oversubscription typically leads to reduced performance. When running TFLite, keep the batch size to one (1); larger batches offer no throughput benefit.
+
+Known limitations
+----
+
+**End-time precision on the last segment of short files (≤ ~34 minutes).**
+For memory efficiency, per-file durations are stored in the smallest float
+dtype that covers their magnitude: ``float16`` for files up to 2\ :sup:`11` ≈
+2048 s, ``float32`` for files up to 2\ :sup:`24` s (~194 days), ``float64``
+beyond. The stored duration is used as the upper clamp when computing the
+``end_time`` of the *last* segment of each file. Inside the float16 range
+this rounding is visible: the largest representable float16 below ``X`` may
+differ from ``X`` by up to one ULP — about 0.06 s near 128 s, 0.25 s near
+1024 s, and 0.5 s near 2048 s. The error appears only on the very last
+segment per file and only when the actual file duration is not exactly
+representable in float16 (integer-second durations up to 2048 s are
+exact). For files of one hour or longer the storage dtype is float32, where
+the equivalent ULP is below 4 ms even at 12 h, so the effect is not
+observable in practice.
+
+All other timestamps (``start_time`` and ``end_time`` of every segment that
+does not hit the clamp) are computed at ≥ float32 precision regardless of
+file length.

docs/setup.rst

@@ -281,6 +281,14 @@ All BirdNET data (models, benchmarks) is stored in the application-data director
 - **macOS:** ``~/Library/Application Support/birdnet``
 - **Windows:** ``%APPDATA%/birdnet``
 
+The default location can be overridden by setting the ``BIRDNET_APP_DATA`` environment variable to any absolute path before the ``birdnet`` package is imported. ::
+
+  # Windows pre-execution script
+  set BIRDNET_APP_DATA=C:\Program Files\BirdNET-Analyzer\birdnet-data
+
+  # Linux / macOS pre-execution script
+  export BIRDNET_APP_DATA=/opt/birdnet-analyzer/birdnet-data
+
 Why is Python 3.10 not supported?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "birdnet"
-version = "0.2.15"
+version = "0.2.16"
 description = "A Python library for identifying bird species by their sounds."
 readme = "README.md"
 requires-python = ">=3.11, <3.14"

src/birdnet/acoustic/inference/core/encoding/encoding_result.py

@@ -21,7 +21,6 @@
   format_input_for_csv,
   get_uint_dtype,
   hms_centis_fast,
-  upgrade_float_dtype_for_value,
 )
 
 if TYPE_CHECKING:
@@ -135,11 +134,11 @@ def to_structured_array(self) -> np.ndarray:
     embeddings_selected = self.embeddings[valid_file_idx, valid_seg_idx]
 
     hop_duration_s = self.hop_duration_s
-    # Upgrade the storage dtype for the output if it cannot represent hop
-    # exactly, otherwise rounding accumulates across segments.
-    time_dtype = upgrade_float_dtype_for_value(
-      self._input_durations.dtype, hop_duration_s
-    )
+    # Force at least float32 for timing columns. The bulk _input_durations
+    # array is stored in a magnitude-based dtype (float16 for files <= 2**11 s),
+    # which is too coarse for accumulated i*hop products and would also produce
+    # Arrow halffloat that some implementations (e.g. R) cannot read.
+    time_dtype = np.result_type(self._input_durations.dtype, np.float32)
 
     dtype = [
       (VAR_INPUT, self._input_dtype),

src/birdnet/acoustic/inference/core/perf_tracker.py

@@ -10,14 +10,15 @@
 from collections.abc import Callable
 from dataclasses import dataclass
 from multiprocessing import Queue, shared_memory
-from multiprocessing.synchronize import Event, Semaphore
+from multiprocessing.synchronize import Event
 from queue import Empty
 
 import numpy as np
 import psutil
 
 import birdnet.acoustic.inference.core.logs as bn_logging
 from birdnet.acoustic.inference.core.shm import RingField
+from birdnet.acoustic.inference.core.sync import CountedSemaphore
 from birdnet.globals import READABLE_FLAG, READING_FLAG, WRITABLE_FLAG
 
 
@@ -124,8 +125,8 @@ def __init__(
     logging_queue: Queue,
     logging_level: int,
     perf_res: Queue,
-    sem_active_workers: Semaphore,
-    sem_filled_slots: Semaphore,
+    sem_active_workers: CountedSemaphore,
+    sem_filled_slots: CountedSemaphore,
     segment_size_s: float,
     parent_process_id: int,
     rf_flags: RingField,
@@ -275,20 +276,42 @@ def reset(self) -> None:
     self._prd_speed_xrt_tracker.reset()
     self._prd_speed_seg_per_s_tracker.reset()
 
+  @staticmethod
+  def _safe_proc_memory(proc: psutil.Process) -> float | None:
+    try:
+      return float(proc.memory_full_info().uss)
+    except (psutil.AccessDenied, PermissionError):
+      pass
+    except psutil.NoSuchProcess:
+      return None
+    try:
+      return float(proc.memory_info().rss)
+    except (psutil.NoSuchProcess, psutil.AccessDenied, PermissionError):
+      return None
+
   def _track_memory_usage(self) -> None:
     if self._parent_process is None:
-      self._parent_process = psutil.Process(self._parent_process_id)
-    memory_usage: float = self._parent_process.memory_full_info().uss
-    for child in self._parent_process.children(recursive=True):
       try:
-        memory_usage += child.memory_full_info().uss
-      except psutil.NoSuchProcess:
-        continue
-      except psutil.AccessDenied:
-        continue
-
-    mem_usage_MiB = memory_usage / 1024**2
-    self._memory_usage_MiB_tracker.add_value(mem_usage_MiB)
+        self._parent_process = psutil.Process(self._parent_process_id)
+      except (psutil.NoSuchProcess, psutil.AccessDenied, PermissionError):
+        return
+
+    parent_mem = self._safe_proc_memory(self._parent_process)
+    if parent_mem is None:
+      return
+
+    total = parent_mem
+    try:
+      children = self._parent_process.children(recursive=True)
+    except (psutil.AccessDenied, PermissionError, psutil.NoSuchProcess):
+      children = []
+
+    for child in children:
+      child_mem = self._safe_proc_memory(child)
+      if child_mem is not None:
+        total += child_mem
+
+    self._memory_usage_MiB_tracker.add_value(total / 1024**2)
 
   @property
   def wall_time(self) -> float:

src/birdnet/acoustic/inference/core/prediction/prediction_result.py

@@ -2,7 +2,7 @@
 
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
 
 import numpy as np
 from ordered_set import OrderedSet
@@ -153,10 +153,15 @@ def to_structured_array(self) -> np.ndarray:
     del valid_mask
 
     n_predictions = len(valid_indices[0])
+    # Force at least float32 for timing columns. The bulk _input_durations
+    # array is stored in a magnitude-based dtype (float16 for files <= 2**11 s),
+    # which is too coarse for accumulated i*hop products and would also produce
+    # Arrow halffloat that some implementations (e.g. R) cannot read.
+    time_dtype = np.result_type(self._input_durations.dtype, np.float32)
     dtype = [
       (VAR_INPUT, self._input_dtype),
-      (VAR_START_TIME, self._input_durations.dtype),
-      (VAR_END_TIME, self._input_durations.dtype),
+      (VAR_START_TIME, time_dtype),
+      (VAR_END_TIME, time_dtype),
       (VAR_SPECIES_NAME, object),
       (VAR_CONFIDENCE, self._species_probs.dtype),
     ]
@@ -191,15 +196,15 @@ def to_structured_array(self) -> np.ndarray:
     del sort_indices
 
     hop_duration_s = self.hop_duration_s
-    start_times = chunk_idx_flat.astype(self._input_durations.dtype) * hop_duration_s
+    start_times = chunk_idx_flat.astype(time_dtype) * hop_duration_s
     del hop_duration_s
     del chunk_idx_flat
 
     structured_array[VAR_START_TIME] = start_times
     structured_array[VAR_END_TIME] = np.minimum(
       start_times
       + apply_speed_to_duration(self._segment_duration_s[0], self._speed[0]),
-      self._input_durations[file_idx_flat],
+      self._input_durations[file_idx_flat].astype(time_dtype),
     )
     del start_times
     structured_array[VAR_INPUT] = self._inputs[file_idx_flat]

src/birdnet/acoustic/inference/core/producer.py

@@ -19,6 +19,7 @@
 
 import birdnet.acoustic.inference.core.logs as bn_logging
 from birdnet.acoustic.inference.core.shm import RingField
+from birdnet.acoustic.inference.core.sync import CountedSemaphore
 from birdnet.globals import (
   READABLE_FLAG,
   READING_FLAG,
@@ -54,7 +55,7 @@ def __init__(
     rf_batch_sizes: RingField,
     rf_flags: RingField,
     sem_free_slots: Semaphore,
-    sem_filled_slots: Semaphore,
+    sem_filled_slots: CountedSemaphore,
     max_segment_idx_ptr: ctypes.c_uint8
     | ctypes.c_uint16
     | ctypes.c_uint32

src/birdnet/acoustic/inference/core/sync.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import multiprocessing as mp
+from multiprocessing.sharedctypes import Synchronized
+from types import TracebackType
+
+
+class CountedSemaphore:
+  """
+  Drop-in replacement for ``mp.Semaphore`` whose ``get_value()`` works on
+  macOS by mirroring acquire/release into a shared counter.
+  """
+
+  def __init__(self, initial: int = 0) -> None:
+    self._sem = mp.Semaphore(initial)
+    self._counter: Synchronized = mp.Value("i", initial)
+
+  def acquire(self, block: bool = True, timeout: float | None = None) -> bool:
+    acquired = self._sem.acquire(block, timeout)
+    if acquired:
+      with self._counter.get_lock():
+        self._counter.value -= 1
+    return acquired
+
+  def release(self) -> None:
+    with self._counter.get_lock():
+      self._counter.value += 1
+    self._sem.release()
+
+  def get_value(self) -> int:
+    return self._counter.value
+
+  def __enter__(self) -> bool:
+    return self.acquire()
+
+  def __exit__(
+    self,
+    exc_type: type[BaseException] | None,
+    exc: BaseException | None,
+    tb: TracebackType | None,
+  ) -> None:
+    self.release()