eidolons/evals/results/2026-07-03-kupo-keep-suite-bare-standard.scorecard.json at main · Rynaro/eidolons · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
{
  "schema_version": "1.0",
  "suite": "kupo-keep-suite",
  "arm": {
    "label": "bare-standard",
    "fix_hook": "/home/rynaro/workspace/oss/agents/eidolons/evals/hooks/keep-bare.sh",
    "env": {
      "EIDOLONS_EVAL_MODEL": "sonnet",
      "EIDOLONS_SANDBOX_MODEL_TIER": "standard"
    },
    "control": true
  },
  "started_at": "2026-07-03T18:10:46Z",
  "k": 3,
  "tasks": [
    {
      "id": "keep-import-path-fix",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-rename-call-consistency",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-lockfile-pin-bump",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-config-key-edit",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-one-line-arithmetic",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-grep-replace-multi",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-json-syntax-fix",
      "resolved": false,
      "passes": 0,
      "attempts": 3
    },
    {
      "id": "keep-multi-block-edit",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-json-value-edit",
      "resolved": false,
      "passes": 0,
      "attempts": 3
    },
    {
      "id": "keep-env-key-rename",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-relative-path-fix",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    },
    {
      "id": "keep-typo-fix",
      "resolved": true,
      "passes": 3,
      "attempts": 3
    }
  ],
  "resolved_rate": 0.8333333333333334,
  "pass_k_rate": 0.8333333333333334,
  "harness": {
    "nexus_version": "1.49.0",
    "smoke": false
  }
}