StellarKraal-/observability/prometheus-rules.yml at main · teslims2/StellarKraal- · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
# Auto-generated by scripts/generate-alert-rules.ts — do not edit manually
groups:
  - name: stellarkraal.rules
    rules:
      - alert: HighP99Latency
        expr: "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 1"
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "p99 HTTP latency exceeds 1s"
          description: "p99 latency is {{ $value | humanizeDuration }} over the last 5 minutes."
          runbook: "http://localhost:3000/d/stellarkraal-backend"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=HighP99Latency"
      - alert: HighErrorRate
        expr: "sum(rate(http_requests_total{status_code=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) > 0.01"
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "HTTP 5xx error rate exceeds 1%"
          description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
          runbook: "http://localhost:3000/d/stellarkraal-backend"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=HighErrorRate"
      - alert: DbPoolExhaustion
        expr: "db_pool_available == 0"
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "DB connection pool exhausted"
          description: "All DB pool connections are in use — new queries will queue or fail."
          runbook: "http://localhost:3000/d/stellarkraal-backend"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=DbPoolExhaustion"
      - alert: RpcFailure
        expr: "stellarkraal_alert_fired{rule=\"rpc-failure\"} > 0"
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "RPC Failure"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/rpc-failure.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=rpc-failure"
      - alert: RpcCircuitOpen
        expr: "stellarkraal_alert_fired{rule=\"rpc-circuit-open\"} > 0"
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "RPC Circuit Breaker Opened"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/rpc-failure.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=rpc-circuit-open"
      - alert: DbError
        expr: "stellarkraal_alert_fired{rule=\"db-error\"} > 0"
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Database Error"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/db-error.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=db-error"
      - alert: LiquidationFailure
        expr: "stellarkraal_alert_fired{rule=\"liquidation-failure\"} > 0"
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Liquidation Engine Failure"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/liquidation-failure.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=liquidation-failure"
      - alert: 5xxSpike
        expr: "stellarkraal_alert_fired{rule=\"5xx-spike\"} > 0"
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "5xx Error Spike"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/5xx-spike.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=5xx-spike"
      - alert: BackupFailure
        expr: "stellarkraal_alert_fired{rule=\"backup-failure\"} > 0"
        for: 60m
        labels:
          severity: critical
        annotations:
          summary: "Database Backup Failed"
          runbook: "https://github.com/teslims2/StellarKraal-/blob/main/docs/runbooks/restore-procedure.md"
          dashboard: "http://localhost:3000/d/stellarkraal-backend?var-alert=backup-failure"