benchmarks/results/schema.json at master · computesdk/benchmarks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://www.computesdk.com/schemas/benchmark-result.json",
  "title": "ComputeSDK Benchmark Result",
  "description": "Schema for benchmark result files produced by ComputeSDK Benchmarks (e.g., results/sequential_tti/latest.json).",
  "type": "object",
  "required": ["version", "timestamp", "results"],
  "additionalProperties": false,
  "properties": {
    "version": {
      "type": "string",
      "description": "Schema/data version of the result file."
    },
    "timestamp": {
      "type": "string",
      "format": "date-time",
      "description": "ISO 8601 timestamp when this result set was produced."
    },
    "environment": {
      "type": "object",
      "description": "Context about where and how the benchmark ran (runner, region, CI, etc.).",
      "additionalProperties": true
    },
    "config": {
      "type": "object",
      "description": "Benchmark configuration used for this run (iterations, concurrency, file sizes, models, etc.).",
      "additionalProperties": true
    },
    "results": {
      "type": "array",
      "description": "One entry per provider/mode combination tested.",
      "items": {
        "$ref": "#/definitions/providerResult"
      }
    }
  },
  "definitions": {
    "nestedMetricMap": {
      "type": "object",
      "description": "A map from sub-metric names (e.g., action types) to metric objects.",
      "additionalProperties": {
        "$ref": "#/definitions/metric"
      },
      "minProperties": 1
    },
    "metric": {
      "type": "object",
      "description": "A single performance metric with percentile statistics.",
      "required": ["median"],
      "additionalProperties": false,
      "properties": {
        "median": {
          "type": "number",
          "description": "Median value across all successful iterations."
        },
        "p95": {
          "type": "number",
          "description": "95th percentile across all successful iterations."
        },
        "p99": {
          "type": "number",
          "description": "99th percentile across all successful iterations."
        },
        "min": {
          "type": "number",
          "description": "Minimum value observed."
        },
        "max": {
          "type": "number",
          "description": "Maximum value observed."
        },
        "avg": {
          "type": "number",
          "description": "Arithmetic mean across all successful iterations."
        }
      }
    },
    "providerResult": {
      "type": "object",
      "required": ["provider", "summary"],
      "additionalProperties": true,
      "properties": {
        "provider": {
          "type": "string",
          "description": "Provider or service being benchmarked."
        },
        "mode": {
          "type": "string",
          "description": "Test mode for this result (e.g., sequential, staggered, burst, cold, warm)."
        },
        "model": {
          "type": "string",
          "description": "Model name, used for AI-gateway benchmarks."
        },
        "bucket": {
          "type": "string",
          "description": "Bucket identifier, used for storage benchmarks."
        },
        "fileSizeBytes": {
          "type": "integer",
          "description": "File size in bytes, used for storage benchmarks."
        },
        "skipped": {
          "type": "boolean",
          "description": "Whether this provider was skipped for this run."
        },
        "skipReason": {
          "type": "string",
          "description": "Reason the provider was skipped, if applicable."
        },
        "iterations": {
          "type": "array",
          "description": "Per-iteration raw measurements and metadata.",
          "items": {
            "type": "object",
            "additionalProperties": true
          }
        },
        "summary": {
          "type": "object",
          "description": "Aggregated metrics for this provider result. Top-level keys are metric names (e.g., ttiMs, uploadMs, totalMs). Most values are metric objects; some suites also group related metrics under a single key (e.g., perActionType maps action names to metric objects).",
          "additionalProperties": {
            "anyOf": [
              {
                "$ref": "#/definitions/metric"
              },
              {
                "$ref": "#/definitions/nestedMetricMap"
              }
            ]
          },
          "minProperties": 1
        },
        "compositeScore": {
          "type": "number",
          "description": "Composite 0-100 score blending timing metrics and success rate, when applicable."
        },
        "successRate": {
          "type": "number",
          "minimum": 0,
          "maximum": 1,
          "description": "Fraction of iterations that succeeded (0.0 to 1.0)."
        }
      }
    }
  }
}