multiple changes

Author: stefantaubert

.github/workflows/ci.yml

@@ -11,7 +11,53 @@ on:
   workflow_dispatch:
 
 jobs:
+  build-and-dl-models:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os:
+          [
+            ubuntu-slim,
+            ubuntu-24.04,
+            ubuntu-24.04-arm,
+            macos-15,
+            macos-15-intel,
+            windows-2025,
+          ]
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install uv build --upgrade
+          python -m uv pip install twine build --upgrade
+          python -m uv pip install --system .[tests]
+
+      # - name: Run linters
+      #   run: |
+      #     ruff check src/birdnet
+      #     ruff check src/birdnet_tests
+
+      # - name: Type checking
+      #   run: mypy
+
+      - name: Download models
+        run: |
+          pytest -m "load_model" -n auto
+
   build-and-test-no-litert:
+    needs: build-and-dl-models
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -62,6 +108,7 @@ jobs:
           python -m twine check dist/*
 
   build-and-test-with-litert:
+    needs: build-and-dl-models
     runs-on: ${{ matrix.os }}
 
     strategy:

CHANGELOG.md

@@ -13,11 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed #17: Issue on macOS with too long ring buffer names
 - Fixed #19: `queue.qsize()` is not used anymore
 - Fixed issue with hanging session because of logging
+- Fixed issue with downloading same model simultaneously
 
 ### Changed
 
 - Improved prediction speed, esp. for half-precision models (+10 seg/s)
 - Lowered dependencies
+- Update `ai-edge-litert` to version 2.0.3 on `repro`
+- Better download progress indication of model files
+- Model loading in tests is done before running other tests
 
 ### Added
 

README.md

@@ -23,7 +23,7 @@ pip install birdnet --user
 pip install birdnet[and-cuda] --user
 
 # For edge devices (e.g., Raspberry Pi)
-# Note: Does NOT work on Windows and not on Python 3.13 yet
+# Note: Does NOT work on Windows, Intel-based Macs and not on Python 3.13 yet
 pip install birdnet[litert] --user
 ```
 

pyproject.toml

@@ -63,7 +63,7 @@ repro = [
   "psutil == 7.1.3",
   "pyarrow == 22.0.0",
   "tensorflow == 2.20.0",
-  "ai-edge-litert == 2.0.2; sys_platform == 'linux' or sys_platform == 'darwin'",
+  "ai-edge-litert == 2.0.3; sys_platform == 'linux' or sys_platform == 'darwin'",
 ]
 tests = [
   "uv >= 0.9.7",
@@ -101,6 +101,7 @@ log_level = "DEBUG"
 testpaths = ["src/birdnet_tests"]
 norecursedirs = ["src/birdnet_v1_tests"]
 markers = [
+  "load_model: tests that download all models, should be run before other tests",
   "litert: tests requiring ai_edge_litert backend which can not be loaded after tf is imported (raises ImportError) which happens on parallel test runs",
   "gpu: tests requiring a GPU to run and to be runned sequentially",
   "repro: tests requiring exact package versions to reproduce results",
@@ -159,24 +160,27 @@ deps =
 commands = 
   # ruff check src/birdnet
   # ruff check src/birdnet_tests
-  pytest -m "not repro and (not litert and not gpu)" -n auto
-  pytest -m "not repro and litert" -n auto
-  pytest -m "not repro and gpu" -n 1
+  pytest -m "load_model" -n auto
+  pytest -m "not load_model and not repro and (not litert and not gpu)" -n auto
+  pytest -m "not load_model and not repro and litert" -n auto
+  pytest -m "not load_model and not repro and gpu" -n 1
 
 [testenv:py312-repro]
 deps =
   .[tests,repro]
 commands = 
-  pytest -m "repro and (not litert and not gpu)" -n auto
-  pytest -m "repro and litert" -n auto
-  pytest -m "repro and gpu" -n 1
+  pytest -m "load_model" -n auto
+  pytest -m "not load_model repro and (not litert and not gpu)" -n auto
+  pytest -m "not load_model repro and litert" -n auto
+  pytest -m "not load_model repro and gpu" -n 1
 
 [testenv:py313]
 deps =
   .[tests,and-cuda]
 commands = 
-  pytest -m "not repro and (not litert and not gpu)" -n auto
-  pytest -m "not repro and gpu" -n 1
+  pytest -m "load_model" -n auto
+  pytest -m "not load_model not repro and (not litert and not gpu)" -n auto
+  pytest -m "not load_model not repro and gpu" -n 1
 """
 
 [build-system]

src/birdnet/acoustic_models/v2_4/pb.py

@@ -46,7 +46,7 @@ def _check_acoustic_model_available(cls) -> bool:
     return model_is_downloaded
 
   @classmethod
-  def _download_acoustic_model(cls) -> None:
+  def _download_model(cls) -> None:
     dl_url = "https://zenodo.org/records/15050749/files/BirdNET_v2.4_protobuf.zip"
     dl_size = 124522908
 
@@ -56,10 +56,10 @@ def _download_acoustic_model(cls) -> None:
         dl_url,
         zip_download_path,
         download_size=dl_size,
-        description="Downloading model",
+        description="Downloading acoustic model v2.4 (pb)",
       )
 
-      print("Extracting models...")
+      print("Extracting...")
       extract_dir = Path(temp_dir) / "extracted"
 
       with zipfile.ZipFile(zip_download_path, "r") as zip_ref:
@@ -70,19 +70,21 @@ def _download_acoustic_model(cls) -> None:
 
       acoustic_model_dir, acoustic_lang_dir = cls._get_paths()
       acoustic_model_dir.parent.mkdir(parents=True, exist_ok=True)
+      shutil.rmtree(acoustic_model_dir, ignore_errors=True)
       shutil.move(acoustic_model_dl_dir, acoustic_model_dir)
 
       acoustic_lang_dir.parent.mkdir(parents=True, exist_ok=True)
+      shutil.rmtree(acoustic_lang_dir, ignore_errors=True)
       shutil.move(species_dl_dir, acoustic_lang_dir)
-      print("Models extracted.")
+      print("Extracted.")
 
   @classmethod
   def get_model_path_and_labels(
     cls,
     lang: str,
   ) -> tuple[Path, OrderedSet[str]]:
     if not cls._check_acoustic_model_available():
-      cls._download_acoustic_model()
+      cls._download_model()
     assert cls._check_acoustic_model_available()
 
     model_dir, langs_path = cls._get_paths()

src/birdnet/acoustic_models/v2_4/tf.py

@@ -82,14 +82,14 @@ def _check_acoustic_model_available(cls, precision: MODEL_PRECISIONS) -> bool:
     return all((lang_dir / f"{lang}.txt").is_file() for lang in cls.AVAILABLE_LANGUAGES)
 
   @classmethod
-  def _download_acoustic_model(cls, precision: MODEL_PRECISIONS) -> None:
+  def _download_model(cls, precision: MODEL_PRECISIONS) -> None:
     with tempfile.TemporaryDirectory(prefix="birdnet_download") as temp_dir:
       zip_download_path = Path(temp_dir) / "download.zip"
       download_file_tqdm(
         models[precision].dl_url,
         zip_download_path,
         download_size=models[precision].dl_size,
-        description="Downloading model",
+        description=f"Downloading acoustic model v2.4 (tf, {precision})",
       )
 
       extract_dir = Path(temp_dir) / "extracted"
@@ -114,7 +114,7 @@ def get_model_path_and_labels(
   ) -> tuple[Path, OrderedSet[str]]:
     assert lang in cls.AVAILABLE_LANGUAGES
     if not cls._check_acoustic_model_available(precision):
-      cls._download_acoustic_model(precision)
+      cls._download_model(precision)
     assert cls._check_acoustic_model_available(precision)
 
     model_path, langs_path = cls._get_paths(precision)

src/birdnet/geo_models/v2_4/pb.py

@@ -43,7 +43,7 @@ def _check_geo_model_available(cls) -> bool:
     return model_is_downloaded
 
   @classmethod
-  def _download_geo_model(cls) -> None:
+  def _download_model(cls) -> None:
     dl_url = "https://zenodo.org/records/15050749/files/BirdNET_v2.4_protobuf.zip"
     dl_size = 124522908
 
@@ -53,10 +53,10 @@ def _download_geo_model(cls) -> None:
         dl_url,
         zip_download_path,
         download_size=dl_size,
-        description="Downloading model",
+        description="Downloading geo model v2.4 (pb)",
       )
 
-      print("Extracting models...")
+      print("Extracting...")
       extract_dir = Path(temp_dir) / "extracted"
 
       with zipfile.ZipFile(zip_download_path, "r") as zip_ref:
@@ -67,19 +67,21 @@ def _download_geo_model(cls) -> None:
 
       geo_model_dir, geo_lang_dir = cls._get_paths()
       geo_model_dir.parent.mkdir(parents=True, exist_ok=True)
+      shutil.rmtree(geo_model_dir, ignore_errors=True)
       shutil.move(geo_model_dl_dir, geo_model_dir)
 
       geo_lang_dir.parent.mkdir(parents=True, exist_ok=True)
+      shutil.rmtree(geo_lang_dir, ignore_errors=True)
       shutil.move(species_dl_dir, geo_lang_dir)
-      print("Models extracted.")
+      print("Extracted.")
 
   @classmethod
   def get_model_path_and_labels(
     cls,
     lang: str,
   ) -> tuple[Path, OrderedSet[str]]:
     if not cls._check_geo_model_available():
-      cls._download_geo_model()
+      cls._download_model()
     assert cls._check_geo_model_available()
 
     model_dir, langs_path = cls._get_paths()

src/birdnet/geo_models/v2_4/tf.py

@@ -57,14 +57,14 @@ def _check_geo_model_available(cls) -> bool:
     return all((lang_dir / f"{lang}.txt").is_file() for lang in cls.AVAILABLE_LANGUAGES)
 
   @classmethod
-  def _download_geo_model(cls) -> None:
+  def _download_model(cls) -> None:
     with tempfile.TemporaryDirectory(prefix="birdnet_download") as temp_dir:
       zip_download_path = Path(temp_dir) / "download.zip"
       download_file_tqdm(
         cls._model_info.dl_url,
         zip_download_path,
         download_size=cls._model_info.dl_size,
-        description="Downloading model",
+        description="Downloading geo model v2.4 (tf)",
       )
 
       extract_dir = Path(temp_dir) / "extracted"
@@ -87,7 +87,7 @@ def _download_geo_model(cls) -> None:
   def get_model_path_and_labels(cls, lang: str) -> tuple[Path, OrderedSet[str]]:
     assert lang in cls.AVAILABLE_LANGUAGES
     if not cls._check_geo_model_available():
-      cls._download_geo_model()
+      cls._download_model()
     assert cls._check_geo_model_available()
 
     model_path, langs_path = cls._get_paths()

src/birdnet_tests/acoustic_models/v2_4/pb_py/test_download_acoustic_model.py

@@ -0,0 +1,11 @@
+from birdnet.acoustic_models.v2_4.pb import AcousticPBDownloaderV2_4
+
+
+def xtest_double_download() -> None:
+  # takes too long to run normally
+  AcousticPBDownloaderV2_4._download_model()
+  AcousticPBDownloaderV2_4._download_model()
+
+
+if __name__ == "__main__":
+  xtest_double_download()

src/birdnet_tests/acoustic_models/v2_4/tf_py/test_download_acoustic_tf_model.py

@@ -0,0 +1,11 @@
+from birdnet.acoustic_models.v2_4.tf import AcousticTFDownloaderV2_4
+
+
+def xtest_double_download() -> None:
+  # takes too long to run normally
+  AcousticTFDownloaderV2_4._download_model("fp32")
+  AcousticTFDownloaderV2_4._download_model("fp32")
+
+
+if __name__ == "__main__":
+  xtest_double_download()