Skip to content

Commit e234df4

Browse files
MaximilianAnzingerMaximilianAnzinger
committed
clean cli output
1 parent 08f1850 commit e234df4

4 files changed

Lines changed: 142 additions & 30 deletions

File tree

‎config/logos_test.yaml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
benchmark:
2+
name: "logos_test"
3+
version: "1.0.0"
4+
runs: 1
5+
6+
evaluators:
7+
qwen38_27b:
8+
provider: "openai_compatible"
9+
model: "Qwen/Qwen3.8-27B"
10+
base_url: "https://logos.aet.cit.tum.de/v1"
11+
temperature: 1.0
12+
max_tokens: 2000
13+
qwen36_35b:
14+
provider: "openai_compatible"
15+
model: "Qwen/Qwen3.6-35B-A3B"
16+
base_url: "https://logos.aet.cit.tum.de/v1"
17+
temperature: 1.0
18+
max_tokens: 2000
19+
llama33_70b:
20+
provider: "openai_compatible"
21+
model: "RedHatAI/Llama-3.3-70B-Instruct-quantized.w4a16"
22+
base_url: "https://logos.aet.cit.tum.de/v1"
23+
temperature: 1.0
24+
max_tokens: 2000
25+
gptoss120b:
26+
provider: "openai_compatible"
27+
model: "openai/gpt-oss-120b"
28+
base_url: "https://logos.aet.cit.tum.de/v1"
29+
temperature: 1.0
30+
max_tokens: 8000
31+
gemma3_12b:
32+
provider: "openai_compatible"
33+
model: "google/gemma-3-12b-it"
34+
base_url: "https://logos.aet.cit.tum.de/v1"
35+
temperature: 1.0
36+
max_tokens: 2000
37+
gemma3_4b:
38+
provider: "openai_compatible"
39+
model: "google/gemma-3-4b-it"
40+
base_url: "https://logos.aet.cit.tum.de/v1"
41+
temperature: 1.0
42+
max_tokens: 2000
43+
llama31_8b:
44+
provider: "openai_compatible"
45+
model: "meta-llama/Llama-3.1-8B-Instruct"
46+
base_url: "https://logos.aet.cit.tum.de/v1"
47+
temperature: 1.0
48+
max_tokens: 2000
49+
phi4reason:
50+
provider: "openai_compatible"
51+
model: "microsoft/Phi-4-reasoning"
52+
base_url: "https://logos.aet.cit.tum.de/v1"
53+
temperature: 1.0
54+
max_tokens: 8000
55+
56+
metrics:
57+
- name: "accuracy"
58+
version: "1.1"
59+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
60+
enabled: true
61+
- name: "answer_key_correctness"
62+
version: "1.0"
63+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
64+
enabled: true
65+
- name: "objective_alignment"
66+
version: "1.0"
67+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
68+
enabled: true
69+
- name: "cognitive_level"
70+
version: "1.0"
71+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
72+
enabled: true
73+
- name: "clarity"
74+
version: "2.0"
75+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
76+
enabled: true
77+
- name: "distractor_quality"
78+
version: "2.0"
79+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
80+
enabled: true
81+
- name: "homogeneous_options"
82+
version: "2.0"
83+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
84+
enabled: true
85+
- name: "absence_of_cueing"
86+
version: "1.0"
87+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
88+
enabled: true
89+
- name: "grammatical_correctness"
90+
version: "2.0"
91+
evaluators: ["qwen38_27b", "qwen36_35b", "llama33_70b", "gptoss120b", "gemma3_12b", "gemma3_4b", "llama31_8b", "phi4reason"]
92+
enabled: true
93+
94+
inputs:
95+
quiz_directory: "data/quizzes/example_quiz.json"
96+
source_directory: "data/inputs"
97+
instructions_directory: "data/instructions"
98+
99+
outputs:
100+
results_directory: "data/results/logos_test"
101+
aggregate: true

‎main.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,8 +237,8 @@ def safe_get(item: Any, key: str, default: Any = None) -> Any:
237237
else:
238238
summary_str = summary
239239

240-
# Print the FULL summary (Statistics + Reasoning)
241-
logger.info("\n%s", summary_str)
240+
# Print stats-only to CLI; full summary goes to file
241+
logger.info("\n%s", summary)
242242

243243
# Save summary to text file
244244
summary_file = run_dir / "summary.txt"

‎src/metrics/base.py‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
"""Base metric interface."""
22

33
import json
4+
import logging
45
from abc import ABC, abstractmethod
56
from collections.abc import Callable
67
from dataclasses import dataclass
78
from enum import Enum
89
from typing import Any, Optional
910

11+
logger = logging.getLogger(__name__)
12+
1013
from pydantic import BaseModel, Field
1114

1215
from ..models.instruction import QuizInstructions
@@ -203,10 +206,11 @@ def adjust_difficulty_for_instructions(
203206

204207
if low <= raw_score <= high:
205208
# Already in band — no adjustment needed
206-
print(
207-
f"\n[Difficulty Band Check — {self.name}]\n"
208-
f" Requested: {requested_difficulty} ({low}–{high})\n"
209-
f" Raw score: {raw_score} — within band, no adjustment."
209+
logger.debug(
210+
"\n[Difficulty Band Check — %s]\n"
211+
" Requested: %s (%s–%s)\n"
212+
" Raw score: %s — within band, no adjustment.",
213+
self.name, requested_difficulty, low, high, raw_score,
210214
)
211215
return raw_score
212216

@@ -215,11 +219,12 @@ def adjust_difficulty_for_instructions(
215219
penalty = round(min(distance * 0.5, 30.0), 1) # cap penalty at 30pts
216220
adjusted = round(max(0.0, min(100.0, raw_score - penalty)), 1)
217221

218-
print(
219-
f"\n[Difficulty Band Check — {self.name}]\n"
220-
f" Requested: {requested_difficulty} ({low}–{high})\n"
221-
f" Raw score: {raw_score} — outside band by {distance:.1f} pts\n"
222-
f" Penalty: -{penalty} → {adjusted}"
222+
logger.debug(
223+
"\n[Difficulty Band Check — %s]\n"
224+
" Requested: %s (%s–%s)\n"
225+
" Raw score: %s — outside band by %.1f pts\n"
226+
" Penalty: -%s → %s",
227+
self.name, requested_difficulty, low, high, raw_score, distance, penalty, adjusted,
223228
)
224229
return adjusted
225230

@@ -350,11 +355,12 @@ def adjust_score_for_custom_prompt(
350355
adjusted = raw_score + adjustment
351356
final = round(max(0.0, min(100.0, adjusted)), 1)
352357

353-
print(
354-
f"\n[Instruction Adjustment — {self.name}]\n"
355-
f" Relevant: {relevant}\n"
356-
f" Adjustment: {adjustment:+.2f} ({raw_score:.1f} → {final:.1f})\n"
357-
f" Reasoning: {reasoning}"
358+
logger.debug(
359+
"\n[Instruction Adjustment — %s]\n"
360+
" Relevant: %s\n"
361+
" Adjustment: %+.2f (%.1f → %.1f)\n"
362+
" Reasoning: %s",
363+
self.name, relevant, adjustment, raw_score, final, reasoning,
358364
)
359365

360366
return final

‎src/runners/benchmark.py‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -280,12 +280,14 @@ def _check_difficulty_compliance(
280280
in_band = low <= mean_difficulty <= high
281281

282282
if in_band:
283-
print(
284-
f"\n[Difficulty Compliance — {quiz_id}]"
285-
f"\n Requested : {instructions.difficulty} (band {low}–{high})"
286-
f"\n Mean score: {mean_difficulty} ✓ within band"
287-
f"\n Questions : {len(difficulty_scores)} scored"
288-
f"\n Adjusted : {mean_difficulty} (no penalty)"
283+
self.logger.debug(
284+
"\n[Difficulty Compliance — %s]"
285+
"\n Requested : %s (band %s–%s)"
286+
"\n Mean score: %s ✓ within band"
287+
"\n Questions : %s scored"
288+
"\n Adjusted : %s (no penalty)",
289+
quiz_id, instructions.difficulty, low, high,
290+
mean_difficulty, len(difficulty_scores), mean_difficulty,
289291
)
290292
return mean_difficulty
291293

@@ -294,14 +296,17 @@ def _check_difficulty_compliance(
294296
penalty = round(min(distance * 0.5, 30.0), 1)
295297
adjusted = round(max(0.0, min(100.0, mean_difficulty - penalty)), 1)
296298

297-
print(
298-
f"\n[Difficulty Compliance — {quiz_id}]"
299-
f"\n Requested : {instructions.difficulty} (band {low}–{high})"
300-
f"\n Mean score: {mean_difficulty} ✗ outside band by {distance:.1f} pts"
301-
f"\n Penalty : -{penalty} → adjusted mean = {adjusted}"
302-
f"\n Questions : {len(difficulty_scores)} scored"
303-
f"\n Note : Quiz overall difficulty does not match the "
304-
f"'{instructions.difficulty}' instruction."
299+
self.logger.debug(
300+
"\n[Difficulty Compliance — %s]"
301+
"\n Requested : %s (band %s–%s)"
302+
"\n Mean score: %s ✗ outside band by %.1f pts"
303+
"\n Penalty : -%s → adjusted mean = %s"
304+
"\n Questions : %s scored"
305+
"\n Note : Quiz overall difficulty does not match the "
306+
"'%s' instruction.",
307+
quiz_id, instructions.difficulty, low, high,
308+
mean_difficulty, distance, penalty, adjusted,
309+
len(difficulty_scores), instructions.difficulty,
305310
)
306311
return adjusted
307312

0 commit comments

Comments
 (0)