Skip to content

Commit e4f6edb

Browse files
athrael-sojuclaude
athrael-soju
and
claude
committed
Add VultronRetrieverPrime-Qwen3.5-8B (v2t175) ViDoRe V1/V2/V3 results
Result JSONs for athrael-soju/VultronRetrieverPrime-Qwen3.5-8B at revision e8f3104b743a04b0d5f715b67117d687ae99ce51: ViDoRe V1 0.9208 / V2 0.6818 / V3 0.6472. Depends on the ModelMeta PR to embeddings-benchmark/mteb merging first. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent a2dba76 commit e4f6edb

23 files changed

Lines changed: 11333 additions & 0 deletions

‎results/athrael-soju__VultronRetrieverPrime-Qwen3.5-8B/e8f3104b743a04b0d5f715b67117d687ae99ce51/Vidore2BioMedicalLecturesRetrieval.json‎

Lines changed: 632 additions & 0 deletions
Large diffs are not rendered by default.

‎results/athrael-soju__VultronRetrieverPrime-Qwen3.5-8B/e8f3104b743a04b0d5f715b67117d687ae99ce51/Vidore2ESGReportsHLRetrieval.json‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd",
3+
"task_name": "Vidore2ESGReportsHLRetrieval",
4+
"mteb_version": "2.12.30",
5+
"scores": {
6+
"test": [
7+
{
8+
"ndcg_at_1": 0.75641,
9+
"ndcg_at_3": 0.75915,
10+
"ndcg_at_5": 0.76818,
11+
"ndcg_at_10": 0.80639,
12+
"ndcg_at_20": 0.81873,
13+
"ndcg_at_100": 0.83604,
14+
"ndcg_at_1000": 0.84029,
15+
"map_at_1": 0.53622,
16+
"map_at_3": 0.67532,
17+
"map_at_5": 0.70486,
18+
"map_at_10": 0.73985,
19+
"map_at_20": 0.7495,
20+
"map_at_100": 0.75583,
21+
"map_at_1000": 0.75669,
22+
"recall_at_1": 0.53622,
23+
"recall_at_3": 0.73846,
24+
"recall_at_5": 0.79822,
25+
"recall_at_10": 0.88701,
26+
"recall_at_20": 0.9253,
27+
"recall_at_100": 0.9845,
28+
"recall_at_1000": 1.0,
29+
"accuracy": 0.53622,
30+
"precision_at_1": 0.76923,
31+
"precision_at_3": 0.42308,
32+
"precision_at_5": 0.29615,
33+
"precision_at_10": 0.18462,
34+
"precision_at_20": 0.10096,
35+
"precision_at_100": 0.02346,
36+
"precision_at_1000": 0.00246,
37+
"mrr_at_1": 0.769231,
38+
"mrr_at_3": 0.820513,
39+
"mrr_at_5": 0.833974,
40+
"mrr_at_10": 0.839583,
41+
"mrr_at_20": 0.839583,
42+
"mrr_at_100": 0.840385,
43+
"mrr_at_1000": 0.840385,
44+
"nauc_ndcg_at_1_max": 0.02257,
45+
"nauc_ndcg_at_1_std": 0.072554,
46+
"nauc_ndcg_at_1_diff1": 0.456427,
47+
"nauc_ndcg_at_3_max": 0.075638,
48+
"nauc_ndcg_at_3_std": 0.086418,
49+
"nauc_ndcg_at_3_diff1": 0.412317,
50+
"nauc_ndcg_at_5_max": 0.151632,
51+
"nauc_ndcg_at_5_std": 0.154338,
52+
"nauc_ndcg_at_5_diff1": 0.412797,
53+
"nauc_ndcg_at_10_max": 0.078521,
54+
"nauc_ndcg_at_10_std": 0.095996,
55+
"nauc_ndcg_at_10_diff1": 0.420301,
56+
"nauc_ndcg_at_20_max": 0.069217,
57+
"nauc_ndcg_at_20_std": 0.096684,
58+
"nauc_ndcg_at_20_diff1": 0.408037,
59+
"nauc_ndcg_at_100_max": 0.09558,
60+
"nauc_ndcg_at_100_std": 0.109941,
61+
"nauc_ndcg_at_100_diff1": 0.436216,
62+
"nauc_ndcg_at_1000_max": 0.076056,
63+
"nauc_ndcg_at_1000_std": 0.091724,
64+
"nauc_ndcg_at_1000_diff1": 0.427715,
65+
"nauc_map_at_1_max": 0.14478,
66+
"nauc_map_at_1_std": 0.138696,
67+
"nauc_map_at_1_diff1": 0.643127,
68+
"nauc_map_at_3_max": 0.140109,
69+
"nauc_map_at_3_std": 0.095476,
70+
"nauc_map_at_3_diff1": 0.51127,
71+
"nauc_map_at_5_max": 0.16692,
72+
"nauc_map_at_5_std": 0.132572,
73+
"nauc_map_at_5_diff1": 0.478307,
74+
"nauc_map_at_10_max": 0.122137,
75+
"nauc_map_at_10_std": 0.113015,
76+
"nauc_map_at_10_diff1": 0.469934,
77+
"nauc_map_at_20_max": 0.122028,
78+
"nauc_map_at_20_std": 0.120395,
79+
"nauc_map_at_20_diff1": 0.461753,
80+
"nauc_map_at_100_max": 0.127869,
81+
"nauc_map_at_100_std": 0.124148,
82+
"nauc_map_at_100_diff1": 0.469666,
83+
"nauc_map_at_1000_max": 0.12553,
84+
"nauc_map_at_1000_std": 0.121906,
85+
"nauc_map_at_1000_diff1": 0.46863,
86+
"nauc_recall_at_1_max": 0.14478,
87+
"nauc_recall_at_1_std": 0.138696,
88+
"nauc_recall_at_1_diff1": 0.643127,
89+
"nauc_recall_at_3_max": 0.103413,
90+
"nauc_recall_at_3_std": 0.058046,
91+
"nauc_recall_at_3_diff1": 0.433193,
92+
"nauc_recall_at_5_max": 0.282423,
93+
"nauc_recall_at_5_std": 0.237024,
94+
"nauc_recall_at_5_diff1": 0.344814,
95+
"nauc_recall_at_10_max": 0.127487,
96+
"nauc_recall_at_10_std": 0.124381,
97+
"nauc_recall_at_10_diff1": 0.293845,
98+
"nauc_recall_at_20_max": 0.048624,
99+
"nauc_recall_at_20_std": 0.133206,
100+
"nauc_recall_at_20_diff1": 0.210096,
101+
"nauc_recall_at_100_max": 0.728583,
102+
"nauc_recall_at_100_std": 0.696287,
103+
"nauc_recall_at_100_diff1": 0.634903,
104+
"nauc_recall_at_1000_max": NaN,
105+
"nauc_recall_at_1000_std": NaN,
106+
"nauc_recall_at_1000_diff1": NaN,
107+
"nauc_precision_at_1_max": -0.021133,
108+
"nauc_precision_at_1_std": 0.041578,
109+
"nauc_precision_at_1_diff1": 0.510921,
110+
"nauc_precision_at_3_max": -0.12607,
111+
"nauc_precision_at_3_std": -0.09897,
112+
"nauc_precision_at_3_diff1": -0.321264,
113+
"nauc_precision_at_5_max": -0.090139,
114+
"nauc_precision_at_5_std": -0.025555,
115+
"nauc_precision_at_5_diff1": -0.40814,
116+
"nauc_precision_at_10_max": -0.161573,
117+
"nauc_precision_at_10_std": -0.068457,
118+
"nauc_precision_at_10_diff1": -0.403227,
119+
"nauc_precision_at_20_max": -0.17807,
120+
"nauc_precision_at_20_std": -0.071288,
121+
"nauc_precision_at_20_diff1": -0.41566,
122+
"nauc_precision_at_100_max": -0.113711,
123+
"nauc_precision_at_100_std": -0.057832,
124+
"nauc_precision_at_100_diff1": -0.378828,
125+
"nauc_precision_at_1000_max": -0.143126,
126+
"nauc_precision_at_1000_std": -0.09151,
127+
"nauc_precision_at_1000_diff1": -0.383785,
128+
"nauc_mrr_at_1_max": -0.021133,
129+
"nauc_mrr_at_1_std": 0.041578,
130+
"nauc_mrr_at_1_diff1": 0.510921,
131+
"nauc_mrr_at_3_max": -0.12478,
132+
"nauc_mrr_at_3_std": -0.030478,
133+
"nauc_mrr_at_3_diff1": 0.460717,
134+
"nauc_mrr_at_5_max": -0.086403,
135+
"nauc_mrr_at_5_std": -0.002389,
136+
"nauc_mrr_at_5_diff1": 0.458123,
137+
"nauc_mrr_at_10_max": -0.089625,
138+
"nauc_mrr_at_10_std": -0.005529,
139+
"nauc_mrr_at_10_diff1": 0.457826,
140+
"nauc_mrr_at_20_max": -0.089625,
141+
"nauc_mrr_at_20_std": -0.005529,
142+
"nauc_mrr_at_20_diff1": 0.457826,
143+
"nauc_mrr_at_100_max": -0.089187,
144+
"nauc_mrr_at_100_std": -0.004631,
145+
"nauc_mrr_at_100_diff1": 0.461256,
146+
"nauc_mrr_at_1000_max": -0.089187,
147+
"nauc_mrr_at_1000_std": -0.004631,
148+
"nauc_mrr_at_1000_diff1": 0.461256,
149+
"hit_rate_at_1": 0.76923,
150+
"hit_rate_at_3": 0.88462,
151+
"hit_rate_at_5": 0.94231,
152+
"hit_rate_at_10": 0.98077,
153+
"hit_rate_at_20": 0.98077,
154+
"hit_rate_at_100": 1.0,
155+
"hit_rate_at_1000": 1.0,
156+
"main_score": 0.76818,
157+
"hf_subset": "default",
158+
"languages": [
159+
"eng-Latn"
160+
]
161+
}
162+
]
163+
},
164+
"evaluation_time": 181.85765743255615,
165+
"kg_co2_emissions": null,
166+
"date": 1781787815.691921
167+
}

0 commit comments

Comments
 (0)