initial to_structured_array

Author: Josef-Haupt

.gitignore

@@ -18,6 +18,7 @@ src/birdnet_debug
 src/birdnet_v1
 src/birdnet_v1_tests
 src/birdnet_v2_old
+playground.*
 
 # VSCode
 .vscode

pyproject.toml

@@ -180,7 +180,7 @@ norecursedirs = ["src/birdnet_v1_tests"]
 markers = [
   "load_model: tests that download all models, should be run before other tests",
   "litert: tests requiring ai_edge_litert backend which can not be loaded after tf is imported (raises ImportError) which happens on parallel test runs",
-  "gpu: tests requiring a GPU to run and to be runned sequentially",
+  "gpu: tests requiring a GPU to run and to be run sequentially",
   "repro: tests requiring exact package versions to reproduce results",
 ]
 

src/birdnet/acoustic/inference/core/consumer.py

@@ -4,8 +4,8 @@
 from multiprocessing.synchronize import Event
 from queue import Empty
 
-from birdnet.acoustic.inference.core.tensor import AcousticTensorBase
 from birdnet.acoustic.inference.core.logs import get_logger_from_session
+from birdnet.acoustic.inference.core.tensor import AcousticTensorBase
 
 
 class Consumer:

src/birdnet/acoustic/inference/core/encoding/encoding_result.py

@@ -7,8 +7,19 @@
 from birdnet.acoustic.inference.core.encoding.encoding_tensor import (
   AcousticEncodingTensor,
 )
-from birdnet.acoustic.inference.core.result_base import AcousticResultBase
-from birdnet.utils.helper import get_uint_dtype
+from birdnet.acoustic.inference.core.result_base import (
+  VAR_END_TIME,
+  VAR_INPUT,
+  VAR_START_TIME,
+  AcousticResultBase,
+)
+from birdnet.utils.helper import (
+  apply_speed_to_duration,
+  get_hop_duration_s,
+  get_uint_dtype,
+)
+
+VAR_EMBEDDING = "embedding"
 
 NP_EMB_KEY = "embeddings"
 NP_EMB_MASKED_KEY = "embeddings_masked"
@@ -78,6 +89,65 @@ def emd_dim(self) -> int:
   def max_n_segments(self) -> int:
     return self._embeddings.shape[1]
 
+  def test() -> None:
+    pass
+
+  def to_structured_array(self) -> np.ndarray:
+    valid_mask_per_segment = ~(self._embeddings_masked).all(axis=2)
+    valid_file_idx, valid_seg_idx = np.where(valid_mask_per_segment)
+    n_embeddings = len(valid_file_idx)
+
+    embeddings_selected = self.embeddings[valid_file_idx, valid_seg_idx]
+
+    dtype = [
+      (VAR_INPUT, self._input_dtype),
+      (VAR_START_TIME, self._input_durations.dtype),
+      (VAR_END_TIME, self._input_durations.dtype),
+      (VAR_EMBEDDING, self._embeddings.dtype, self.emd_dim),
+    ]
+
+    structured_array = np.empty(n_embeddings, dtype=dtype)
+    del dtype
+
+    if n_embeddings == 0:
+      return structured_array
+    del n_embeddings
+
+    sort_keys = (
+      valid_seg_idx,
+      valid_file_idx,
+    )
+    sort_indices = np.lexsort(sort_keys)
+    del sort_keys
+
+    file_idx_flat = valid_file_idx[sort_indices]
+    chunk_idx_flat = valid_seg_idx[sort_indices]
+    emb_flat = embeddings_selected[sort_indices]
+    del embeddings_selected
+    del sort_indices
+
+    hop_duration_s = get_hop_duration_s(
+      self._segment_duration_s[0], self._overlap_duration_s[0], self._speed[0]
+    )
+    start_times = chunk_idx_flat.astype(self._input_durations.dtype) * hop_duration_s
+    del hop_duration_s
+    del chunk_idx_flat
+
+    structured_array[VAR_START_TIME] = start_times
+    structured_array[VAR_END_TIME] = np.minimum(
+      start_times
+      + apply_speed_to_duration(self._segment_duration_s[0], self._speed[0]),
+      self._input_durations[file_idx_flat],
+    )
+    del start_times
+    structured_array[VAR_INPUT] = self._inputs[file_idx_flat]
+    del file_idx_flat
+
+    structured_array[VAR_EMBEDDING] = emb_flat
+    del emb_flat
+
+    return structured_array
+
   def unprocessable_inputs(self) -> np.ndarray:
     return self._unprocessable_inputs
 

src/birdnet/acoustic/inference/core/encoding/encoding_tensor.py

@@ -5,8 +5,8 @@
 import numpy as np
 from numpy.typing import DTypeLike
 
-from birdnet.acoustic.inference.core.tensor import AcousticTensorBase
 from birdnet.acoustic.inference.core.logs import get_logger_from_session
+from birdnet.acoustic.inference.core.tensor import AcousticTensorBase
 
 
 class AcousticEncodingTensor(AcousticTensorBase):

src/birdnet/acoustic/inference/core/prediction/prediction_result.py

@@ -11,7 +11,12 @@
 from birdnet.acoustic.inference.core.prediction.prediction_tensor import (
   AcousticPredictionTensor,
 )
-from birdnet.acoustic.inference.core.result_base import AcousticResultBase
+from birdnet.acoustic.inference.core.result_base import (
+  VAR_END_TIME,
+  VAR_INPUT,
+  VAR_START_TIME,
+  AcousticResultBase,
+)
 from birdnet.utils.helper import (
   apply_speed_to_duration,
   get_hop_duration_s,
@@ -22,9 +27,6 @@
   import pandas as pd
   import pyarrow as pa
 
-VAR_INPUT = "input"
-VAR_START_TIME = "start_time"
-VAR_END_TIME = "end_time"
 VAR_SPECIES_NAME = "species_name"
 VAR_CONFIDENCE = "confidence"
 
@@ -145,10 +147,6 @@ def _set_extra_load_data(cls, data: dict[str, np.ndarray]) -> None:
     cls._species_list = data[NP_SPECIES_LIST_KEY]
     cls._unprocessable_inputs = data[NP_UNPROCESSABLE_INPUTS_KEY]
 
-  @property
-  def _input_dtype(self) -> type:
-    return self._inputs.dtype
-
   def to_structured_array(self) -> np.ndarray:
     valid_mask = ~self._species_masked
     valid_indices = np.where(valid_mask)

src/birdnet/acoustic/inference/core/prediction/prediction_worker.py

@@ -8,9 +8,9 @@
 import numpy as np
 from numpy.typing import DTypeLike
 
+from birdnet.acoustic.inference.core.shm import RingField
 from birdnet.acoustic.inference.core.worker import WorkerBase
 from birdnet.core.backends import BackendLoader, BatchT
-from birdnet.acoustic.inference.core.shm import RingField
 from birdnet.utils.helper import flat_sigmoid_logaddexp_fast, get_uint_dtype
 
 if TYPE_CHECKING:

src/birdnet/acoustic/inference/core/result_base.py

@@ -11,6 +11,10 @@
 from birdnet.core.base import ResultBase
 from birdnet.utils.helper import get_float_dtype, get_hash, get_uint_dtype
 
+VAR_INPUT = "input"
+VAR_START_TIME = "start_time"
+VAR_END_TIME = "end_time"
+
 NP_INPUTS_KEY = "inputs"
 NP_INPUT_DURATIONS_KEY = "input_durations"
 NP_SEGMENT_DURATION_S_KEY = "segment_duration_s"
@@ -62,6 +66,10 @@ def __init__(
     self._model_fmax = np.array([model_fmax], dtype=get_uint_dtype(model_fmax))
     self._model_sr = np.array([model_sr], dtype=get_uint_dtype(model_sr))
 
+  @property
+  def _input_dtype(self) -> type:
+    return self._inputs.dtype
+
   @property
   def segment_duration_s(self) -> float:
     return float(self._segment_duration_s[0])
@@ -146,7 +154,7 @@ def __init__(self) -> None:
   def __enter__(self) -> Self: ...
 
   @abstractmethod
-  def __exit__(self, *args): ...
+  def __exit__(self, *args) -> None: ...
 
   @abstractmethod
   def run(self, *args, **kwargs) -> ResultBase: ...

src/birdnet/acoustic/inference/core/worker.py

@@ -13,14 +13,14 @@
 from numpy.typing import DTypeLike
 
 import birdnet.acoustic.inference.core.logs as bn_logging
+from birdnet.acoustic.inference.core.shm import RingField
 from birdnet.core.backends import BackendLoader, BatchT, VersionedBackendProtocol
 from birdnet.globals import (
   READABLE_FLAG,
   READING_FLAG,
   WRITABLE_FLAG,
   WRITING_FLAG,
 )
-from birdnet.acoustic.inference.core.shm import RingField
 
 if TYPE_CHECKING:
   pass

src/birdnet/acoustic/inference/session.py

@@ -142,7 +142,7 @@ def _run(self, inputs: list[Path] | list[tuple[np.ndarray, int]]) -> ResultType:
       self._resources.producer_resources.unprocessed_inputs
     )
 
-    if is_file_input := any(isinstance(inp, Path) for inp in inputs):
+    if any(isinstance(inp, Path) for inp in inputs):
       assert all(isinstance(inp, Path) for inp in inputs)
       inputs = cast(list[Path], inputs)
       result = self._strategy.create_files_result(