feat: add parameter usage metrics for Responses API (ogx-ai#5255)

Fixed ogx-ai#5256

# What does this PR do?
<!-- Provide a short summary of what this PR does and why. Link to
relevant issues if applicable. -->

<!-- If resolving an issue, uncomment and update the line below -->

Add OpenTelemetry metrics to track which optional parameters callers
explicitly provide in Responses API (`POST /v1/responses`) requests.
This helps understand API usage patterns and prioritize OpenAI parity
work. introduced new counter metric
`llama_stack.responses.parameter_usage_total` with `operation` and
`parameter` attributes

For a request like:
```bash
curl -X POST http://localhost:8321/v1/responses \
  -H 'Content-Type: application/json' \
  -d '{
    "input": "What is Python?",
    "model": "ollama/llama3.2:3b-instruct-fp16",
    "temperature": 0.7,
    "stream": true,
    "tools": [],
    "instructions": "Be concise"
  }'
```

The following Prometheus metrics are emitted:
```
llama_stack_llama_stack_responses_parameter_usage_total{operation="create_response", parameter="temperature"} 1
llama_stack_llama_stack_responses_parameter_usage_total{operation="create_response", parameter="stream"} 1
llama_stack_llama_stack_responses_parameter_usage_total{operation="create_response", parameter="tools"} 1
llama_stack_llama_stack_responses_parameter_usage_total{operation="create_response", parameter="instructions"} 1
```

```console
(llama-stack) gualiu@gualiu-mac llama-stack % curl -s 'http://localhost:9090/api/v1/query?query=llama_stack_llama_stack_responses_parameter_usage_total' | jq
{
  "status": "success",
  "data": {
    "resultType": "vector",
    "result": [
      {
        "metric": {
          "__name__": "llama_stack_llama_stack_responses_parameter_usage_total",
          "exported_job": "llama-stack-server",
          "instance": "otel-collector:9464",
          "job": "otel-collector",
          "operation": "create_response",
          "otel_scope_name": "llama_stack.responses",
          "otel_scope_version": "1.0.0",
          "parameter": "instructions"
        },
        "value": [
          1774293626.183,
          "7"
        ]
      },
      {
        "metric": {
          "__name__": "llama_stack_llama_stack_responses_parameter_usage_total",
          "exported_job": "llama-stack-server",
          "instance": "otel-collector:9464",
          "job": "otel-collector",
          "operation": "create_response",
          "otel_scope_name": "llama_stack.responses",
          "otel_scope_version": "1.0.0",
          "parameter": "stream"
        },
        "value": [
          1774293626.183,
          "7"
        ]
      },
      {
        "metric": {
          "__name__": "llama_stack_llama_stack_responses_parameter_usage_total",
          "exported_job": "llama-stack-server",
          "instance": "otel-collector:9464",
          "job": "otel-collector",
          "operation": "create_response",
          "otel_scope_name": "llama_stack.responses",
          "otel_scope_version": "1.0.0",
          "parameter": "temperature"
        },
        "value": [
          1774293626.183,
          "4"
        ]
      },
      {
        "metric": {
          "__name__": "llama_stack_llama_stack_responses_parameter_usage_total",
          "exported_job": "llama-stack-server",
          "instance": "otel-collector:9464",
          "job": "otel-collector",
          "operation": "create_response",
          "otel_scope_name": "llama_stack.responses",
          "otel_scope_version": "1.0.0",
          "parameter": "tools"
        },
        "value": [
          1774293626.183,
          "4"
        ]
      }
    ]
  }
}
```

Co-authored-by: Sébastien Han <seb@redhat.com>

Author: gyliu513

scripts/telemetry/llama-stack-responses-metrics.json

@@ -0,0 +1,384 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 1,
+    "id": null,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+        "id": 100,
+        "title": "Overview",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "thresholds" },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                { "color": "green", "value": null }
+              ]
+            },
+            "unit": "short"
+          },
+          "overrides": []
+        },
+        "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 },
+        "id": 1,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": ["lastNotNull"],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto"
+        },
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "expr": "sum(llama_stack_llama_stack_responses_parameter_usage_total)",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Total Parameter Usage Events",
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "thresholds" },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                { "color": "green", "value": null }
+              ]
+            },
+            "unit": "short"
+          },
+          "overrides": []
+        },
+        "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 },
+        "id": 2,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "none",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": ["lastNotNull"],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto"
+        },
+        "pluginVersion": "10.2.3",
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "expr": "count(count by (parameter) (llama_stack_llama_stack_responses_parameter_usage_total))",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Distinct Parameters Used",
+        "type": "stat"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "thresholds" },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                { "color": "green", "value": null }
+              ]
+            },
+            "unit": "short"
+          },
+          "overrides": []
+        },
+        "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 },
+        "id": 3,
+        "options": {
+          "colorMode": "value",
+          "graphMode": "area",
+          "justifyMode": "auto",
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": ["lastNotNull"],
+            "fields": "",
+            "values": false
+          },
+          "textMode": "auto"
+        },
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "expr": "sum(llama_stack_llama_stack_responses_parameter_usage_total{parameter=\"stream\"})",
+            "legendFormat": "__auto",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Streaming Requests",
+        "type": "stat"
+      },
+      {
+        "collapsed": false,
+        "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+        "id": 101,
+        "title": "Parameter Usage Distribution",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "palette-classic" },
+            "custom": {
+              "hideFrom": { "tooltip": false, "viz": false, "legend": false }
+            },
+            "mappings": []
+          },
+          "overrides": []
+        },
+        "gridPos": { "h": 10, "w": 12, "x": 0, "y": 6 },
+        "id": 4,
+        "options": {
+          "displayLabels": ["name", "percent"],
+          "legend": {
+            "displayMode": "table",
+            "placement": "right",
+            "showLegend": true,
+            "values": ["value", "percent"]
+          },
+          "pieType": "donut",
+          "reduceOptions": {
+            "calcs": ["lastNotNull"],
+            "fields": "",
+            "values": false
+          },
+          "tooltip": { "mode": "single", "sort": "none" }
+        },
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "expr": "sum by (parameter) (llama_stack_llama_stack_responses_parameter_usage_total)",
+            "legendFormat": "{{parameter}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Parameter Usage Distribution",
+        "type": "piechart"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "palette-classic" },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "bars",
+              "fillOpacity": 80,
+              "gradientMode": "none",
+              "hideFrom": { "tooltip": false, "viz": false, "legend": false },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": { "type": "linear" },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": { "group": "A", "mode": "normal" },
+              "thresholdsStyle": { "mode": "off" }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [{ "color": "green", "value": null }]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": { "h": 10, "w": 12, "x": 12, "y": 6 },
+        "id": 5,
+        "options": {
+          "legend": {
+            "calcs": ["last"],
+            "displayMode": "table",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": { "mode": "multi", "sort": "desc" }
+        },
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "expr": "sum by (operation, parameter) (rate(llama_stack_llama_stack_responses_parameter_usage_total[5m]))",
+            "legendFormat": "{{operation}}/{{parameter}}",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Parameter Usage Rate (per second)",
+        "type": "timeseries"
+      },
+      {
+        "collapsed": false,
+        "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
+        "id": 102,
+        "title": "Details",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": { "mode": "thresholds" },
+            "custom": {
+              "align": "auto",
+              "cellOptions": { "type": "auto" },
+              "inspect": false
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [{ "color": "green", "value": null }]
+            }
+          },
+          "overrides": [
+            {
+              "matcher": { "id": "byName", "options": "Value" },
+              "properties": [{ "id": "displayName", "value": "Count" }]
+            }
+          ]
+        },
+        "gridPos": { "h": 10, "w": 24, "x": 0, "y": 17 },
+        "id": 6,
+        "options": {
+          "cellHeight": "sm",
+          "footer": {
+            "countRows": false,
+            "fields": "",
+            "reducer": ["sum"],
+            "show": false
+          },
+          "showHeader": true,
+          "sortBy": [
+            { "desc": true, "displayName": "Count" }
+          ]
+        },
+        "pluginVersion": "10.2.3",
+        "targets": [
+          {
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "editorMode": "code",
+            "exemplar": false,
+            "expr": "sum by (operation, parameter) (llama_stack_llama_stack_responses_parameter_usage_total)",
+            "format": "table",
+            "instant": true,
+            "legendFormat": "__auto",
+            "range": false,
+            "refId": "A"
+          }
+        ],
+        "title": "Parameter Usage Counts",
+        "transformations": [
+          {
+            "id": "organize",
+            "options": {
+              "excludeByName": { "Time": true },
+              "indexByName": {},
+              "renameByName": {
+                "operation": "Operation",
+                "parameter": "Parameter"
+              }
+            }
+          }
+        ],
+        "type": "table"
+      }
+    ],
+    "refresh": "5s",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [
+      "llama-stack",
+      "metrics",
+      "responses"
+    ],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-15m",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Llama Stack - Responses API Parameter Usage",
+    "uid": "llama-stack-responses-metrics",
+    "version": 0,
+    "weekStart": ""
+  }

scripts/telemetry/setup_telemetry.sh

@@ -140,6 +140,7 @@ $CONTAINER_RUNTIME run -d --name grafana \
   -v "$SCRIPT_DIR/llama-stack-tool-runtime-metrics.json:/etc/grafana/provisioning/dashboards/llama-stack-tool-runtime-metrics.json:Z" \
   -v "$SCRIPT_DIR/llama-stack-vector-io-metrics.json:/etc/grafana/provisioning/dashboards/llama-stack-vector-io-metrics.json:Z" \
   -v "$SCRIPT_DIR/llama-stack-request-metrics.json:/etc/grafana/provisioning/dashboards/llama-stack-request-metrics.json:Z" \
+  -v "$SCRIPT_DIR/llama-stack-responses-metrics.json:/etc/grafana/provisioning/dashboards/llama-stack-responses-metrics.json:Z" \
   docker.io/grafana/grafana:11.0.0
 
 # Wait for services to start