Skip to content

Commit 1ca9849

Browse files
jhayniffyjhayniffy
authored
Merge pull request #167 from obajecollinsmicheal-cmd/feat/observability-metrics-logging
feat: add Prometheus metrics, structured JSON logging, and request ID…
2 parents 3970e7e + 10d4488 commit 1ca9849

19 files changed

Lines changed: 1043 additions & 70 deletions

‎.github/workflows/ci.yml‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,18 +70,6 @@ jobs:
7070
run:
7171
working-directory: backend
7272

73-
services:
74-
redis:
75-
image: redis:7-alpine
76-
ports:
77-
- 6379:6379
78-
options: >-
79-
--health-cmd "redis-cli ping"
80-
--health-interval 5s
81-
--health-timeout 3s
82-
--health-retries 5
83-
--health-start-period 5s
84-
8573
env:
8674
REDIS_HOST: 127.0.0.1
8775
REDIS_PORT: 6379
@@ -90,6 +78,14 @@ jobs:
9078
steps:
9179
- uses: actions/checkout@v4
9280

81+
# Install Redis directly on the runner — avoids Docker Hub rate limits entirely
82+
- name: Start Redis
83+
run: |
84+
sudo apt-get update -qq
85+
sudo apt-get install -y redis-server
86+
sudo systemctl start redis-server
87+
redis-cli ping
88+
9389
- uses: actions/setup-node@v4
9490
with:
9591
node-version: 22

‎backend/docs/grafana-dashboard.json‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
{
2+
"__inputs": [
3+
{
4+
"name": "DS_PROMETHEUS",
5+
"label": "Prometheus",
6+
"description": "",
7+
"type": "datasource",
8+
"pluginId": "prometheus",
9+
"pluginName": "Prometheus"
10+
}
11+
],
12+
"title": "NiffyInsure API",
13+
"uid": "niffyinsure-api",
14+
"schemaVersion": 38,
15+
"version": 1,
16+
"refresh": "30s",
17+
"panels": [
18+
{
19+
"id": 1,
20+
"title": "Request Rate (req/s)",
21+
"type": "timeseries",
22+
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
23+
"targets": [
24+
{
25+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
26+
"expr": "sum(rate(http_requests_total{app=\"niffyinsure-api\"}[1m])) by (route, method)",
27+
"legendFormat": "{{method}} {{route}}"
28+
}
29+
]
30+
},
31+
{
32+
"id": 2,
33+
"title": "HTTP Latency p50 / p95 / p99 (s)",
34+
"type": "timeseries",
35+
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
36+
"targets": [
37+
{
38+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
39+
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{app=\"niffyinsure-api\"}[5m])) by (le, route))",
40+
"legendFormat": "p50 {{route}}"
41+
},
42+
{
43+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
44+
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{app=\"niffyinsure-api\"}[5m])) by (le, route))",
45+
"legendFormat": "p95 {{route}}"
46+
},
47+
{
48+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
49+
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{app=\"niffyinsure-api\"}[5m])) by (le, route))",
50+
"legendFormat": "p99 {{route}}"
51+
}
52+
]
53+
},
54+
{
55+
"id": 3,
56+
"title": "5xx Error Rate",
57+
"type": "timeseries",
58+
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
59+
"targets": [
60+
{
61+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
62+
"expr": "sum(rate(http_5xx_errors_total{app=\"niffyinsure-api\"}[5m])) by (route)",
63+
"legendFormat": "5xx {{route}}"
64+
}
65+
],
66+
"fieldConfig": {
67+
"defaults": {
68+
"thresholds": {
69+
"mode": "absolute",
70+
"steps": [
71+
{ "color": "green", "value": null },
72+
{ "color": "red", "value": 0.1 }
73+
]
74+
}
75+
}
76+
}
77+
},
78+
{
79+
"id": 4,
80+
"title": "RPC Call Rate by Method",
81+
"type": "timeseries",
82+
"gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
83+
"targets": [
84+
{
85+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
86+
"expr": "sum(rate(rpc_calls_total{app=\"niffyinsure-api\"}[1m])) by (rpc_method, status)",
87+
"legendFormat": "{{rpc_method}} ({{status}})"
88+
}
89+
]
90+
},
91+
{
92+
"id": 5,
93+
"title": "RPC Error Rate",
94+
"type": "timeseries",
95+
"gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
96+
"targets": [
97+
{
98+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
99+
"expr": "sum(rate(rpc_errors_total{app=\"niffyinsure-api\"}[5m])) by (rpc_method, error_type)",
100+
"legendFormat": "{{rpc_method}} / {{error_type}}"
101+
}
102+
]
103+
},
104+
{
105+
"id": 6,
106+
"title": "RPC Latency p95 (s)",
107+
"type": "timeseries",
108+
"gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 },
109+
"targets": [
110+
{
111+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
112+
"expr": "histogram_quantile(0.95, sum(rate(rpc_call_duration_seconds_bucket{app=\"niffyinsure-api\"}[5m])) by (le, rpc_method))",
113+
"legendFormat": "p95 {{rpc_method}}"
114+
}
115+
]
116+
},
117+
{
118+
"id": 7,
119+
"title": "Node.js Heap Used (MB)",
120+
"type": "timeseries",
121+
"gridPos": { "x": 0, "y": 24, "w": 12, "h": 8 },
122+
"targets": [
123+
{
124+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
125+
"expr": "nodejs_heap_size_used_bytes{app=\"niffyinsure-api\"} / 1024 / 1024",
126+
"legendFormat": "heap used MB"
127+
}
128+
]
129+
},
130+
{
131+
"id": 8,
132+
"title": "Event Loop Lag (s)",
133+
"type": "timeseries",
134+
"gridPos": { "x": 12, "y": 24, "w": 12, "h": 8 },
135+
"targets": [
136+
{
137+
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
138+
"expr": "nodejs_eventloop_lag_seconds{app=\"niffyinsure-api\"}",
139+
"legendFormat": "event loop lag"
140+
}
141+
]
142+
}
143+
]
144+
}

‎backend/docs/observability.md‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Observability Guide
2+
3+
## Metrics — `/metrics`
4+
5+
The `/metrics` endpoint (not prefixed with `/api`) exposes Prometheus text format.
6+
Restrict it at the ingress/firewall level — it must not be publicly reachable.
7+
8+
### HTTP metrics
9+
10+
| Metric | Type | Labels | Description |
11+
|---|---|---|---|
12+
| `http_request_duration_seconds` | Histogram | `method`, `route`, `status_code` | Request latency. Buckets: 10 ms → 10 s |
13+
| `http_requests_total` | Counter | `method`, `route`, `status_code` | Total requests |
14+
| `http_5xx_errors_total` | Counter | `method`, `route` | 5xx responses only |
15+
16+
### RPC metrics
17+
18+
| Metric | Type | Labels | Description |
19+
|---|---|---|---|
20+
| `rpc_call_duration_seconds` | Histogram | `rpc_method`, `status` | Soroban RPC call latency |
21+
| `rpc_calls_total` | Counter | `rpc_method`, `status` | Total RPC calls |
22+
| `rpc_errors_total` | Counter | `rpc_method`, `error_type` | RPC errors by type |
23+
24+
`rpc_method` values: `simulate_generate_premium`, `build_initiate_policy`,
25+
`build_file_claim`, `send_transaction`, `get_events`, `get_latest_ledger`.
26+
27+
`error_type` values: `client_error`, `unavailable`, `unknown`.
28+
29+
### Cardinality notes
30+
31+
- `route` is normalised: numeric path segments → `:id`, UUIDs → `:uuid`,
32+
Stellar addresses → `:address`. Raw URLs are never used as labels.
33+
- `status_code` is the exact HTTP code (200, 400, 404, 500…). The set is
34+
bounded so cardinality is safe.
35+
- Never add wallet addresses, policy IDs, or claim IDs as metric labels.
36+
37+
---
38+
39+
## Structured JSON Logs
40+
41+
All log entries are newline-delimited JSON written to stdout.
42+
Ship to your centralised stack (Loki, CloudWatch, Datadog, etc.) via the
43+
container log driver.
44+
45+
### Log field dictionary
46+
47+
| Field | Type | Description |
48+
|---|---|---|
49+
| `timestamp` | ISO 8601 | UTC time of the log entry |
50+
| `level` | string | `error` / `warn` / `info` / `debug` |
51+
| `message` | string | Human-readable summary |
52+
| `service` | string | Always `niffyinsure-api` |
53+
| `requestId` | string | Correlation ID — propagated from `x-request-id` header or generated as a UUID |
54+
| `method` | string | HTTP verb (GET, POST, …) |
55+
| `url` | string | Request path only — query string is omitted to avoid leaking tokens |
56+
| `statusCode` | number | HTTP response status |
57+
| `durationMs` | number | Request duration in milliseconds |
58+
| `ip` | string | Client IP address |
59+
| `userAgent` | string | `User-Agent` header value |
60+
| `context` | string | NestJS class / module name |
61+
| `stack` | string | Error stack trace (error level only) |
62+
| `rpcMethod` | string | Soroban RPC method name (RPC log entries only) |
63+
| `rpcStatus` | string | `success` or `error` (RPC log entries only) |
64+
| `contentLength` | number | Response body size in bytes |
65+
66+
### Fields intentionally omitted
67+
68+
- `Authorization` / `Cookie` / `x-api-key` headers — always `[REDACTED]`
69+
- Request and response bodies — never logged
70+
- IPFS file contents — never logged
71+
- Private keys, seeds, mnemonics, Ed25519 signatures
72+
- Full wallet addresses in log messages (use short prefix for debugging)
73+
74+
### Request ID propagation
75+
76+
Every request receives a `requestId`:
77+
1. If the client sends `x-request-id`, that value is used.
78+
2. Otherwise a UUID v4 is generated.
79+
80+
The ID is echoed back in the `x-request-id` response header and included in
81+
every log entry and error response body for end-to-end correlation.
82+
83+
---
84+
85+
## Grafana Dashboard
86+
87+
Import `docs/grafana-dashboard.json` into Grafana (Dashboards → Import).
88+
Select your Prometheus datasource when prompted.
89+
90+
Panels:
91+
- Request rate by route/method
92+
- HTTP latency p50 / p95 / p99
93+
- 5xx error rate
94+
- RPC call rate by method
95+
- RPC error rate
96+
- RPC latency p95
97+
- Node.js heap usage
98+
- Event loop lag
99+
100+
---
101+
102+
## Alerting
103+
104+
Load `docs/prometheus-alerts.yml` into your Prometheus `rule_files`.
105+
106+
| Alert | Threshold | Severity |
107+
|---|---|---|
108+
| `High5xxRate` | > 0.5 errors/s for 2 min | critical |
109+
| `HighRpcErrorRate` | > 0.2 errors/s for 2 min | warning |
110+
| `HighP99Latency` | p99 > 3 s for 5 min | warning |
111+
| `HighRpcP95Latency` | p95 > 10 s for 5 min | warning |
112+
113+
---
114+
115+
## OpenTelemetry Extension Point
116+
117+
`AppLoggerService.structured()` is the single place to inject OTel trace
118+
context. When you add `@opentelemetry/sdk-node`:
119+
120+
```ts
121+
// In app-logger.service.ts — structured()
122+
import { trace } from '@opentelemetry/api';
123+
const span = trace.getActiveSpan();
124+
const traceId = span?.spanContext().traceId;
125+
const spanId = span?.spanContext().spanId;
126+
this.winston.log(level, message, { ...fields, traceId, spanId });
127+
```
128+
129+
Similarly, `MetricsService.recordHttpRequest` / `recordRpcCall` map directly
130+
to OTel `Meter` histogram/counter calls — swap the prom-client calls for OTel
131+
Meter API calls when you're ready to migrate.

‎backend/docs/prometheus-alerts.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
groups:
2+
- name: niffyinsure_api
3+
rules:
4+
# ── HTTP 5xx alert ────────────────────────────────────────────────────
5+
- alert: High5xxRate
6+
expr: |
7+
sum(rate(http_5xx_errors_total{app="niffyinsure-api"}[5m])) > 0.5
8+
for: 2m
9+
labels:
10+
severity: critical
11+
team: backend
12+
annotations:
13+
summary: "Elevated HTTP 5xx error rate"
14+
description: >
15+
More than 0.5 5xx errors/s over the last 5 minutes.
16+
Check logs with requestId correlation for root cause.
17+
18+
# ── RPC error alert ───────────────────────────────────────────────────
19+
- alert: HighRpcErrorRate
20+
expr: |
21+
sum(rate(rpc_errors_total{app="niffyinsure-api"}[5m])) > 0.2
22+
for: 2m
23+
labels:
24+
severity: warning
25+
team: backend
26+
annotations:
27+
summary: "Elevated Soroban RPC error rate"
28+
description: >
29+
More than 0.2 RPC errors/s. Possible Soroban RPC degradation
30+
or contract deployment issue.
31+
32+
# ── High latency alert ────────────────────────────────────────────────
33+
- alert: HighP99Latency
34+
expr: |
35+
histogram_quantile(0.99,
36+
sum(rate(http_request_duration_seconds_bucket{app="niffyinsure-api"}[5m]))
37+
by (le)
38+
) > 3
39+
for: 5m
40+
labels:
41+
severity: warning
42+
team: backend
43+
annotations:
44+
summary: "p99 HTTP latency above 3 s"
45+
description: >
46+
The 99th-percentile request latency has exceeded 3 seconds
47+
for 5 consecutive minutes.
48+
49+
# ── RPC latency alert ─────────────────────────────────────────────────
50+
- alert: HighRpcP95Latency
51+
expr: |
52+
histogram_quantile(0.95,
53+
sum(rate(rpc_call_duration_seconds_bucket{app="niffyinsure-api"}[5m]))
54+
by (le, rpc_method)
55+
) > 10
56+
for: 5m
57+
labels:
58+
severity: warning
59+
team: backend
60+
annotations:
61+
summary: "Soroban RPC p95 latency above 10 s"
62+
description: >
63+
RPC method {{ $labels.rpc_method }} p95 latency exceeded 10 s.
64+
Soroban network may be congested.

0 commit comments

Comments
 (0)