fix: disable json_mode for QA eval LLM, add list-format unwrap safety net

Author: Alex-Hunterz

benchmarks/locomo/locomo_runner.py

@@ -166,6 +166,7 @@ def _init_eval_llm(self) -> None:
         self._eval_llm = create_llm(
             self.config.eval_model,
             thinking_level=self.config.qa_thinking_level,
+            json_mode=False,
         )
         level = self.config.qa_thinking_level or "model-default"
         logger.info("Eval LLM: %s (thinking=%s)", self.config.eval_model, level)

vektori/models/gemini.py

@@ -43,10 +43,12 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         initial_backoff: float = DEFAULT_INITIAL_BACKOFF,
         max_backoff: float = DEFAULT_MAX_BACKOFF,
+        json_mode: bool = True,
     ) -> None:
         self.model = model or DEFAULT_MODEL
         self._api_key = api_key
         self._thinking_level = thinking_level  # None = use model-based default
+        self._json_mode = json_mode
         self._client = None
         self.max_retries = max_retries
         self.initial_backoff = initial_backoff
@@ -94,10 +96,9 @@ async def generate(self, prompt: str, max_tokens: int | None = None) -> str:
 
         client = self._get_client()
 
-        config_kwargs: dict[str, Any] = {
-            "temperature": 0.1,
-            "response_mime_type": "application/json",
-        }
+        config_kwargs: dict[str, Any] = {"temperature": 0.1}
+        if self._json_mode:
+            config_kwargs["response_mime_type"] = "application/json"
         if max_tokens is not None:
             config_kwargs["max_output_tokens"] = max_tokens
         thinking = self._thinking_config()

vektori/qa/generator.py

@@ -108,12 +108,14 @@ async def generate_answer(
     )
     try:
         answer = (await llm.generate(prompt, max_tokens=max_tokens)).strip()
-        # Unwrap {"answer": "..."} JSON that some models (e.g. gemini) occasionally return
+        # Unwrap JSON-wrapped answers (safety net — should not trigger after json_mode fix)
         try:
             import json as _json
             parsed = _json.loads(answer)
             if isinstance(parsed, dict) and "answer" in parsed:
                 answer = str(parsed["answer"]).strip()
+            elif isinstance(parsed, list) and parsed and isinstance(parsed[0], dict) and "answer" in parsed[0]:
+                answer = str(parsed[0]["answer"]).strip()
         except Exception:
             # Fallback: regex extraction handles malformed/truncated JSON
             m = re.search(r'"answer"\s*:\s*"((?:[^"\\]|\\.)*)"', answer)