anserini/src/main/resources/regression/dl21-doc-segmented.yaml at master · yuki617/anserini · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
---
corpus: msmarco-v2-doc-segmented
corpus_path: collections/msmarco/msmarco_v2_doc_segmented/

index_path: indexes/lucene-index.msmarco-v2-doc-segmented/
collection_class: MsMarcoV2DocCollection
generator_class: DefaultLuceneDocumentGenerator
index_threads: 18
index_options: -storePositions -storeDocvectors -storeRaw
index_stats:
  documents: 124131414
  documents (non-empty): 124131414
  total terms: 24780915974

metrics:
  - metric: MAP@100
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -c -M 100 -m map
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: no
  - metric: MRR@100
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -c -M 100 -m recip_rank
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: true
  - metric: nDCG@10
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -c -m ndcg_cut.10
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: true
  - metric: R@100
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -c -m recall.100
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: false
  - metric: R@1000
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -c -m recall.1000
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: false

topic_reader: TsvInt
topic_root: src/main/resources/topics-and-qrels/
qrels_root: src/main/resources/topics-and-qrels/
topics:
  - name: "[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)"
    id: dl21
    path: topics.dl21.txt
    qrel: qrels.dl21-doc.txt

models:
  - name: bm25-default
    display: BM25 (default)
    params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25
    results:
      MAP@100:
        - 0.2436
      MRR@100:
        - 0.8937
      nDCG@10:
        - 0.5776
      R@100:
        - 0.3478
      R@1000:
        - 0.6930
  - name: bm25-default+rm3
    display: +RM3
    params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rm3
    results:
      MAP@100:
        - 0.2933
      MRR@100:
        - 0.9018
      nDCG@10:
        - 0.6185
      R@100:
        - 0.3892
      R@1000:
        - 0.7694
  - name: bm25-default+rocchio
    display: +Rocchio
    params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rocchio
    results:
      MAP@100:
        - 0.2974
      MRR@100:
        - 0.9099
      nDCG@10:
        - 0.6067
      R@100:
        - 0.3957
      R@1000:
        - 0.7740
  - name: bm25-default+ax
    display: +Ax
    params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -axiom -axiom.deterministic -rerankCutoff 20
    results:
      MAP@100:
        - 0.2808
      MRR@100:
        - 0.9221
      nDCG@10:
        - 0.5840
      R@100:
        - 0.3884
      R@1000:
        - 0.7934
  - name: bm25-default+prf
    display: +PRF
    params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -bm25prf
    results:
      MAP@100:
        - 0.2729
      MRR@100:
        - 0.9146
      nDCG@10:
        - 0.5936
      R@100:
        - 0.3778
      R@1000:
        - 0.7423