fix(benchmarking): report how many queries the RAG benchmark actually scored

benchmarking/rag/lib/metrics.py

@@ -21,7 +21,7 @@ def retrieval_metrics(
     qrels: dict[str, dict[str, int]],
     results: dict[str, dict[str, float]],
     k_values: list[int] | None = None,
-) -> dict[str, float]:
+) -> dict[str, float | int]:
     """Compute retrieval metrics using pytrec_eval.
 
     Args:
@@ -30,7 +30,11 @@ def retrieval_metrics(
         k_values: Cutoff values for nDCG, recall, MAP. Default [5, 10].
 
     Returns:
-        Aggregated metrics dict, e.g. {"ndcg_cut_10": 0.45, "recall_10": 0.78, ...}
+        Aggregated metrics dict, e.g. {"ndcg_cut_10": 0.45, "recall_10": 0.78, ...},
+        plus "num_scored_queries" (queries the averages were taken over) and
+        "num_missing_queries" (queries in qrels that got no result, usually
+        because the run errored past them). Callers already set their own
+        "num_queries", so these use distinct names.
     """
     if k_values is None:
         k_values = [5, 10]
@@ -44,7 +48,10 @@ def retrieval_metrics(
     # Filter to queries present in both qrels and results
     common_qids = set(qrels.keys()) & set(results.keys())
     if not common_qids:
-        return dict.fromkeys(metrics_set, 0.0)
+        empty = dict.fromkeys(metrics_set, 0.0)
+        empty["num_scored_queries"] = 0
+        empty["num_missing_queries"] = len(qrels)
+        return empty
 
     filtered_qrels = {qid: qrels[qid] for qid in common_qids}
     filtered_results = {qid: results[qid] for qid in common_qids}
@@ -58,6 +65,12 @@ def retrieval_metrics(
         values = [per_query[qid][metric] for qid in per_query if metric in per_query[qid]]
         aggregated[metric] = sum(values) / len(values) if values else 0.0
 
+    # Report what the averages were taken over. Queries the run never produced a
+    # result for are absent from common_qids, so without these two numbers a
+    # partial run and a complete one are indistinguishable in the output.
+    aggregated["num_scored_queries"] = len(common_qids)
+    aggregated["num_missing_queries"] = len(set(qrels.keys()) - common_qids)
+
     return aggregated
 
 
@@ -91,7 +104,7 @@ def per_query_retrieval_metrics(
 def answer_metrics(
     predictions: dict[str, str],
     ground_truths: dict[str, str | list[str]],
-) -> dict[str, float]:
+) -> dict[str, float | int]:
     """Compute aggregated answer quality metrics.
 
     Uses HuggingFace `evaluate` (SQuAD metric) for EM/F1 and `rouge-score` for ROUGE-L.
@@ -105,7 +118,13 @@ def answer_metrics(
     """
     common_qids = sorted(set(predictions.keys()) & set(ground_truths.keys()))
     if not common_qids:
-        return {"exact_match": 0.0, "f1": 0.0, "rouge_l": 0.0, "num_queries": 0}
+        return {
+            "exact_match": 0.0,
+            "f1": 0.0,
+            "rouge_l": 0.0,
+            "num_queries": 0,
+            "num_missing_queries": len(ground_truths),
+        }
 
     # Format for HuggingFace squad metric
     hf_predictions = []
@@ -139,4 +158,5 @@ def answer_metrics(
         "f1": squad_results["f1"] / 100.0,
         "rouge_l": sum(rouge_scores) / len(rouge_scores),
         "num_queries": len(common_qids),
+        "num_missing_queries": len(set(ground_truths.keys()) - set(common_qids)),
     }

tests/unit/test_rag_benchmark_metrics_coverage.py

@@ -0,0 +1,69 @@
+# Copyright (c) The OGX Contributors.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Coverage reporting for the RAG benchmark metrics.
+
+The benchmark runners skip a conversation when a query or ingestion raises, so
+those query ids never reach the metric functions. The averages are taken over
+the queries that did arrive, which means a partial run and a complete run look
+the same in the output unless the counts are reported alongside.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("pytrec_eval")
+pytest.importorskip("evaluate")
+
+# `benchmarking/` is not a package and is not on the path by default.
+_BENCHMARKING_ROOT = Path(__file__).resolve().parents[2]
+if str(_BENCHMARKING_ROOT) not in sys.path:
+    sys.path.insert(0, str(_BENCHMARKING_ROOT))
+
+from benchmarking.rag.lib.metrics import retrieval_metrics  # noqa: E402
+
+
+def _qrels(n: int) -> dict[str, dict[str, int]]:
+    return {f"q{i}": {f"d{i}": 1} for i in range(n)}
+
+
+def _results(qids: list[str]) -> dict[str, dict[str, float]]:
+    return {qid: {f"d{qid[1:]}": 1.0} for qid in qids}
+
+
+def test_reports_counts_when_all_queries_present():
+    metrics = retrieval_metrics(_qrels(3), _results(["q0", "q1", "q2"]), k_values=[5])
+    assert metrics["num_scored_queries"] == 3
+    assert metrics["num_missing_queries"] == 0
+
+
+def test_missing_queries_are_counted_not_hidden():
+    # Two of five queries errored during the run, so they never reached the metrics.
+    metrics = retrieval_metrics(_qrels(5), _results(["q0", "q1", "q2"]), k_values=[5])
+    assert metrics["num_scored_queries"] == 3
+    assert metrics["num_missing_queries"] == 2
+
+
+def test_partial_run_is_distinguishable_from_complete_run():
+    complete = retrieval_metrics(_qrels(4), _results(["q0", "q1", "q2", "q3"]), k_values=[5])
+    partial = retrieval_metrics(_qrels(4), _results(["q0", "q1"]), k_values=[5])
+    # Both score 1.0 on the queries they saw; only the counts tell them apart.
+    assert complete["ndcg_cut_5"] == partial["ndcg_cut_5"]
+    assert (complete["num_scored_queries"], complete["num_missing_queries"]) != (
+        partial["num_scored_queries"],
+        partial["num_missing_queries"],
+    )
+
+
+def test_no_results_at_all_reports_every_query_missing():
+    metrics = retrieval_metrics(_qrels(3), {}, k_values=[5])
+    assert metrics["num_scored_queries"] == 0
+    assert metrics["num_missing_queries"] == 3
+    assert metrics["ndcg_cut_5"] == 0.0