Fix run benchmark

.github/scripts/find_benchmarks.py

@@ -84,6 +84,13 @@ def filter_changed_dirs(dirs: list[str], changed_files: set[str]) -> list[str]:
 
 
 def main() -> None:
+
+    import argparse
+    parser = argparse.ArgumentParser(description='Find benchmarks in sub-repo')
+    parser.add_argument('--all', action="store_true",
+                        help='Force to run all benchmarks')
+    args = parser.parse_args()
+
     root = Path.cwd()
     repo = Repo(root)
 
@@ -93,7 +100,7 @@ def main() -> None:
     # Get reference range for filtering
     ref_range = get_ref_range(repo)
 
-    if ref_range:
+    if ref_range and not args.all:
         base, head = ref_range
         changed_files = get_changed_files(repo, base, head)
         filtered_dirs = filter_changed_dirs(all_dirs, changed_files)

.github/workflows/benchopt_test.yml

@@ -1,4 +1,4 @@
-name: Tests
+name: Test benchopt
 
 on:
   push:

.github/workflows/deepinv_run_hf.yml

@@ -27,7 +27,7 @@ jobs:
         run: pip install GitPython
       - name: Find benchmark directories
         id: find-dirs
-        run: python .github/scripts/find_benchmarks.py
+        run: python .github/scripts/find_benchmarks.py --all
         env:
           GITHUB_SHA: ${{ github.sha }}
           GITHUB_EVENT_NAME: ${{ github.event_name }}

.github/workflows/test_run_benchmak.yml

@@ -0,0 +1,59 @@
+name: Test deepinv_bench
+
+on:
+  push:
+    branches:
+      - main
+  create:
+    tags:
+      - '**'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      CONDA_ENV: 'run_env'
+      BENCHOPT_CONDA_CMD: 'mamba'
+    defaults:
+      run:
+        # Need to use this shell to get conda working properly.
+        # See https://github.com/marketplace/actions/setup-miniconda#important
+        shell: bash -l {0}
+
+    steps:
+    - uses: actions/checkout@v6
+    - name: Get Date
+      id: get-date
+      run: |
+        echo "date=$(/bin/date -u "+%Y%m")" >> $GITHUB_OUTPUT
+      shell: bash
+    - name: Cache benchopt runs and conda packages
+      id: cache-benchopt
+      uses: actions/cache@v5
+      with:
+        path: ~/conda_pkgs_dir
+        # The cache is unique for each OS and we invalidate it every month
+        # to avoid stale caches.
+        key: ${{ runner.os }}-${{ steps.get-date.outputs.date }}
+
+    - name: Setup Conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-update-conda: true
+        miniforge-version: latest
+        mamba-version: "*"
+        use-mamba: true
+        channels: conda-forge
+        python-version: 3.12
+        activate-environment: ${{ env.CONDA_ENV }}
+        use-only-tar-bz2: true # Needed for package caching
+
+    - name: Install deepinv_bench and its dependencies
+      run: |
+        mamba install -yq pip
+        pip install -e .[test]
+
+    - name: Run tests
+      run: pytest -vs

deepinv_bench/run.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import deepinv as dinv
 
-BENCHMARK_ROOT = Path(__file__).parent
+BENCHMARK_ROOT = Path(__file__).parent / "benchmarks"
 
 
 def run_benchmark(
@@ -28,22 +28,29 @@ def run_benchmark(
     """
     model_name = model_name or str(model.__class__.__name__)
 
-    if not isinstance(model, (dinv.models.Reconstructor, torch.nn.Module)):
-        raise ValueError(
-            "Model should be an instance of "
-            "deepinv.models.Reconstructor or torch.nn.Module"
-        )
-
     try:
         benchmark = benchopt.benchmark.Benchmark(
             BENCHMARK_ROOT / benchmark_name
         )
     except Exception:
+        all_benchmarks = "\n-".join([
+            p.name for p in BENCHMARK_ROOT.iterdir()
+            if p.is_dir() and not p.name.startswith(".")
+            and "template" not in p.name
+        ])
         raise ValueError(
-            f"Could not find benchmark: {benchmark_name}. "
-            f"Consider updating deepinv_bench by running \n\n"
-            f"pip install --upgrade --force-reinstall --no-deps "
-            f"git+https://github.com/deepinv/benchmarks.git#egg=deepinv_bench"
+            f"Could not find benchmark: {benchmark_name}.\n"
+            f"Available benchmarks are:\n-{all_benchmarks}\n\n"
+            "If the requested benchmark is not present, consider updating "
+            "deepinv_bench by running \n\n"
+            "pip install --upgrade --force-reinstall --no-deps "
+            "git+https://github.com/deepinv/benchmarks.git"
+        )
+
+    if not isinstance(model, (dinv.models.Reconstructor, torch.nn.Module)):
+        raise ValueError(
+            "Model should be an instance of "
+            "deepinv.models.Reconstructor or torch.nn.Module"
         )
 
     objectives = benchmark.check_objective_filters([])

deepinv_bench/tests/test_run_benchmark.py

@@ -0,0 +1,33 @@
+import pytest
+import deepinv as dinv
+
+from deepinv_bench import run_benchmark
+
+TEST_BENCH = "cbsd500_gaussian_denoising"
+
+
+def test_run_benchmark():
+    """Make sure that the run_benchmark comand is properly running"""
+    my_solver = dinv.models.DnCNN()
+    results = run_benchmark(my_solver, TEST_BENCH, debug=True)
+    assert isinstance(results, dict), "results should be a dict"
+    for col in ["PSNR", "NIQE"]:
+        assert col in results, f"{col} not found in results"
+        assert f"{col}_std" in results, f"{col}_std not found in results"
+
+
+def test_run_invalid_benchmark():
+    """Make sure that the run_benchmark comand is properly running"""
+
+    bench = "invalid_bench"
+    err_msg = f"Could not find benchmark: {bench}"
+    with pytest.raises(ValueError, match=err_msg):
+        run_benchmark(None, "invalid_bench")
+
+
+def test_run_invalid_method():
+    """Make sure that the run_benchmark comand is properly running"""
+
+    err_msg = "Model should be an instance of deepinv.models.Reconstructor"
+    with pytest.raises(ValueError, match=err_msg):
+        run_benchmark(None, TEST_BENCH)

pyproject.toml

@@ -11,6 +11,13 @@ dependencies = [
     "benchopt>=1.8.0",
 ]
 
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "datasets",
+    "pyiqa",
+]
+
 [tool.setuptools.packages.find]
 namespaces = false