simple-evals-continue/eval_builder.py at main · 0xjunhao/simple-evals-continue · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
from .evals.mmlu_tr_eval import MMLUTrEval
from .evals.swebench_lite_eval import SWEBenchLiteEval
from .evals.browsecomp_eval import BrowseCompEval
from .evals.drop_eval import DropEval
from .evals.gpqa_eval import GPQAEval
from .evals.healthbench_eval import HealthBenchEval
from .evals.healthbench_meta_eval import HealthBenchMetaEval
from .evals.math_eval import MathEval
from .evals.mgsm_eval import MGSMEval
from .evals.mmlu_eval import MMLUEval
from .evals.humaneval_eval import HumanEval
from .evals.simpleqa_eval import SimpleQAEval


class EvalBuilder:
    """Builds evaluation instances based on the selected evaluation name."""
    SUPPORTED_EVALS = [
        "mmlu",
        "math",
        "gpqa",
        "mgsm",
        "drop",
        "humaneval",
        "simpleqa",
        "browsecomp",
        "healthbench",
        "healthbench_hard",
        "healthbench_consensus",
        "healthbench_meta",
        "swebench_verified",
        "mmlu_tr",
    ]

    def __init__(self, args, equality_checker, grading_sampler):
        self.args = args
        self.equality_checker = equality_checker
        self.grading_sampler = grading_sampler

    def build(self, eval_name):
        args = self.args
        equality_checker = self.equality_checker
        grading_sampler = self.grading_sampler
        debug_mode = args.debug
        num_examples = (
            args.examples if args.examples is not None else (
                5 if debug_mode else None)
        )
        match eval_name:
            case "mmlu":
                return MMLUEval(num_examples)
            case "math":
                return MathEval(
                    equality_checker=equality_checker,
                    num_examples=num_examples,
                    n_repeats=1 if debug_mode else args.n_repeats or 10,
                )
            case "gpqa":
                return GPQAEval(
                    n_repeats=1 if debug_mode else args.n_repeats or 10,
                    num_examples=num_examples,
                )
            case "mgsm":
                return MGSMEval(
                    num_examples_per_lang=10 if debug_mode else num_examples or 250
                )
            case "drop":
                return DropEval(
                    num_examples=10 if debug_mode else num_examples,
                    train_samples_per_prompt=3,
                )
            case "humaneval":
                return HumanEval(num_examples=10 if debug_mode else num_examples)
            case "simpleqa":
                return SimpleQAEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                )
            case "browsecomp":
                return BrowseCompEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                )
            case "healthbench":
                return HealthBenchEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                    n_repeats=args.n_repeats or 1,
                    n_threads=args.n_threads or 1,
                    subset_name=None,
                )
            case "healthbench_hard":
                return HealthBenchEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                    n_repeats=args.n_repeats or 1,
                    n_threads=args.n_threads or 1,
                    subset_name="hard",
                )
            case "healthbench_consensus":
                return HealthBenchEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                    n_repeats=args.n_repeats or 1,
                    n_threads=args.n_threads or 1,
                    subset_name="consensus",
                )
            case "healthbench_meta":
                return HealthBenchMetaEval(
                    grader_model=grading_sampler,
                    num_examples=10 if debug_mode else num_examples,
                    n_repeats=args.n_repeats or 1,
                    n_threads=args.n_threads or 1,
                )
            case "swebench_lite":
                return SWEBenchLiteEval(
                    num_examples=num_examples,
                )
            case "mmlu_tr":
                return MMLUTrEval(num_examples)
            case _:
                raise Exception(f"Unrecognized eval type: {eval_name}")