MCPg/benchmarks/results/forced-nl2sql-comparison.json at 9719d2ef0693c4585ae818a5db0a100be3f75050 · devopam/MCPg · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
{
  "trials": [
    {
      "task_id": "top_revenue_category",
      "arm": "baseline",
      "trial": 0,
      "tokens_in": 6392,
      "tokens_out": 600,
      "turns": 5,
      "tool_calls": 4,
      "passed": true,
      "final_answer": "The **Home** category generated the most revenue overall, with total sales of $956,135.87 (95,613,587 cents), calculated from `mcpg_demo.order_items.quantity * order_items.unit_price_cents` joined to `mcpg_demo.products.category`.",
      "error": null,
      "hidden_tokens_in": 0,
      "hidden_tokens_out": 0
    },
    {
      "task_id": "top_revenue_category",
      "arm": "mcpg",
      "trial": 0,
      "tokens_in": 18858,
      "tokens_out": 2389,
      "turns": 6,
      "tool_calls": 17,
      "passed": false,
      "final_answer": "I wasn't able to locate any populated schema in this database. I checked `public` (empty) as well as a wide range of common schema names (sales, ecommerce, retail, store, shop, app, northwind, dvdrental, main, analytics, business, orders, production, data) \u2014 all reported no tables. System catalogs (`information_schema`, `pg_catalog`) are blocked from querying, and I don't have a tool to list schemas directly.\n\nI'm unable to answer the question because no accessible schema in this database currently contains any tables (or the correct schema name is not one I guessed). Could you tell me the specific schema name where the product/sales/category tables live, or confirm this is the intended database?",
      "error": null,
      "hidden_tokens_in": 6948,
      "hidden_tokens_out": 584
    },
    {
      "task_id": "top_customer_lifetime_spend",
      "arm": "baseline",
      "trial": 0,
      "tokens_in": 6824,
      "tokens_out": 667,
      "turns": 5,
      "tool_calls": 4,
      "passed": true,
      "final_answer": "The single highest-spending customer is **Liam Okafor** (customer_id 1 in `mcpg_demo.customers`), with a lifetime order total of $128,513.85 (12,851,385 cents), computed by summing `mcpg_demo.orders.total_cents` grouped by customer \u2014 well ahead of the next highest customer, Chen Eriksson (~$62,467.57).",
      "error": null,
      "hidden_tokens_in": 0,
      "hidden_tokens_out": 0
    },
    {
      "task_id": "top_customer_lifetime_spend",
      "arm": "mcpg",
      "trial": 0,
      "tokens_in": 29177,
      "tokens_out": 2194,
      "turns": 8,
      "tool_calls": 12,
      "passed": true,
      "final_answer": "The highest-spending customer is **Liam Okafor**, with a total lifetime order value of **$128,513.85** (12,851,385 cents).\n\nThis was determined from `mcpg_demo.customers` (column `full_name`) joined to `mcpg_demo.orders` (column `total_cents`, keyed on `customer_id`), summing `total_cents` per customer and taking the top result.",
      "error": null,
      "hidden_tokens_in": 5612,
      "hidden_tokens_out": 920
    }
  ],
  "aggregate": {
    "mcpg": {
      "trials": 2,
      "mean_total_tokens": 33341.0,
      "mean_tool_calls": 14.5,
      "mean_turns": 7.0,
      "correctness": 0.5
    },
    "baseline": {
      "trials": 2,
      "mean_total_tokens": 7241.5,
      "mean_tool_calls": 4.0,
      "mean_turns": 5.0,
      "correctness": 1.0
    },
    "token_ratio": 0.21719504513961788,
    "errored": 0,
    "per_task": [
      {
        "task_id": "top_revenue_category",
        "mcpg": {
          "mean_total_tokens": 28779.0,
          "mean_tool_calls": 17.0,
          "correctness": 0.0
        },
        "baseline": {
          "mean_total_tokens": 6992.0,
          "mean_tool_calls": 4.0,
          "correctness": 1.0
        }
      },
      {
        "task_id": "top_customer_lifetime_spend",
        "mcpg": {
          "mean_total_tokens": 37903.0,
          "mean_tool_calls": 12.0,
          "correctness": 1.0
        },
        "baseline": {
          "mean_total_tokens": 7491.0,
          "mean_tool_calls": 4.0,
          "correctness": 1.0
        }
      }
    ]
  }
}