Skip to content

Commit 0575a5f

Browse files
Mystery-CLIMystery-CLI
authored
Merge pull request #365 from popsman01/feature/issue-86-monitoring-alerting
feat(monitoring): comprehensive monitoring and alerting for production
2 parents 44a674b + 0eb2e2f commit 0575a5f

8 files changed

Lines changed: 464 additions & 4 deletions

File tree

‎.env.example‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,15 @@ ADMIN_KEYS=admin-key-123
2020
# ── Optional ─────────────────────────────────────────────────────────────────
2121
REDIS_URL=
2222
LOG_LEVEL=debug
23+
24+
# ── Monitoring & Alerting ─────────────────────────────────────────────────────
25+
GRAFANA_ADMIN_PASSWORD=
26+
GRAFANA_URL=http://localhost:3002
27+
ALERT_WEBHOOK_URL=http://indexer:3000/monitoring/alerts
28+
INCIDENT_WEBHOOK_URL=http://indexer:3000/monitoring/alerts
29+
PAGERDUTY_ROUTING_KEY=
30+
ONCALL_EMAIL=oncall@stellarescrow.app
31+
SMTP_HOST=localhost:587
32+
SMTP_FROM=alerts@stellarescrow.app
33+
SMTP_USER=
34+
SMTP_PASSWORD=

‎Troubleshooting.md‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,4 +525,61 @@ See [Deployment Guide — Section 8](02-DEPLOYMENT.md#8-post-deployment-verifica
525525

526526
---
527527

528+
## 8. Infrastructure Alert Runbooks
529+
530+
### RB-010: High CPU / Memory Usage
531+
532+
**Alert:** `HighCPUUsage` / `CriticalCPUUsage` / `HighMemoryUsage` / `CriticalMemoryUsage`
533+
534+
```bash
535+
# Identify top processes
536+
docker stats --no-stream
537+
top -b -n1 | head -20
538+
539+
# Check indexer memory
540+
docker compose exec indexer cat /proc/self/status | grep VmRSS
541+
542+
# Restart the offending service if memory-leaking
543+
docker compose restart indexer
544+
```
545+
546+
### RB-011: Disk Space Warning
547+
548+
**Alert:** `DiskSpaceWarning` / `DiskSpaceCritical`
549+
550+
```bash
551+
# Find large files
552+
du -sh /var/lib/docker/volumes/* | sort -rh | head -10
553+
554+
# Prune unused Docker data
555+
docker system prune -f
556+
557+
# Truncate old Prometheus data if needed (adjust retention in docker-compose.yml)
558+
# Reduce --storage.tsdb.retention.time=15d and restart prometheus
559+
docker compose restart prometheus
560+
```
561+
562+
### RB-012: PostgreSQL Connection Exhaustion
563+
564+
**Alert:** `PostgresHighConnections`
565+
566+
```bash
567+
# Check active connections
568+
docker compose exec postgres psql -U indexer -d stellar_escrow \
569+
-c "SELECT count(*), state FROM pg_stat_activity GROUP BY state;"
570+
571+
# Kill idle connections older than 10 minutes
572+
docker compose exec postgres psql -U indexer -d stellar_escrow \
573+
-c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity
574+
WHERE state = 'idle' AND query_start < NOW() - INTERVAL '10 minutes';"
575+
```
576+
577+
### RB-013: Redis Down / Low Hit Rate
578+
579+
**Alert:** `RedisDown` / `RedisLowHitRate`
580+
581+
See [RB-005](#rb-005-redis-cache-failure). For low hit rate, check TTL configuration in `indexer/src/cache.rs` and consider increasing cache TTLs for read-heavy endpoints.
582+
583+
---
584+
528585
*© 2026 StellarEscrow*

‎docker-compose.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ services:
205205
- ./monitoring/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro
206206
- ./monitoring/alert_rules_security.yml:/etc/prometheus/alert_rules_security.yml:ro
207207
- ./monitoring/alert_rules_performance.yml:/etc/prometheus/alert_rules_performance.yml:ro
208+
- ./monitoring/alert_rules_infrastructure.yml:/etc/prometheus/alert_rules_infrastructure.yml:ro
208209
- prometheus_data:/prometheus
209210
command:
210211
- "--config.file=/etc/prometheus/prometheus.yml"
@@ -223,11 +224,13 @@ services:
223224
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD must be set}
224225
GF_USERS_ALLOW_SIGN_UP: "false"
225226
GF_SERVER_ROOT_URL: ${GRAFANA_URL:-http://localhost:3002}
227+
GF_FEATURE_TOGGLES_ENABLE: "lokiLive"
226228
volumes:
227229
- grafana_data:/var/lib/grafana
228230
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
229231
depends_on:
230232
- prometheus
233+
- loki
231234
restart: unless-stopped
232235

233236
loki:
@@ -245,6 +248,7 @@ services:
245248
volumes:
246249
- ./monitoring/promtail.yml:/etc/promtail/config.yml:ro
247250
- /var/log:/var/log:ro
251+
- /var/run/docker.sock:/var/run/docker.sock:ro
248252
command: -config.file=/etc/promtail/config.yml
249253
depends_on:
250254
- loki
@@ -260,6 +264,38 @@ services:
260264
- "--config.file=/etc/alertmanager/alertmanager.yml"
261265
restart: unless-stopped
262266

267+
node-exporter:
268+
image: prom/node-exporter:latest
269+
pid: host
270+
network_mode: host
271+
volumes:
272+
- /proc:/host/proc:ro
273+
- /sys:/host/sys:ro
274+
- /:/rootfs:ro
275+
command:
276+
- "--path.procfs=/host/proc"
277+
- "--path.sysfs=/host/sys"
278+
- "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
279+
restart: unless-stopped
280+
281+
postgres-exporter:
282+
image: prometheuscommunity/postgres-exporter:latest
283+
environment:
284+
DATA_SOURCE_NAME: ${DATABASE_URL:?DATABASE_URL must be set}
285+
depends_on:
286+
postgres:
287+
condition: service_healthy
288+
restart: unless-stopped
289+
290+
redis-exporter:
291+
image: oliver006/redis_exporter:latest
292+
environment:
293+
REDIS_ADDR: redis://redis:6379
294+
REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD must be set}
295+
depends_on:
296+
- redis
297+
restart: unless-stopped
298+
263299
volumes:
264300
postgres_data:
265301
redis_data:

‎monitoring/alert_rules_infrastructure.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
groups:
2+
- name: infrastructure
3+
rules:
4+
- alert: ServiceDown
5+
expr: up == 0
6+
for: 1m
7+
labels:
8+
severity: critical
9+
annotations:
10+
summary: "Service {{ $labels.job }} is down"
11+
description: "{{ $labels.job }} has been unreachable for more than 1 minute."
12+
runbook: "https://github.com/Mystery-CLI/StellarEscrow/blob/main/Troubleshooting.md"
13+
14+
- alert: HighCPUUsage
15+
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
16+
for: 5m
17+
labels:
18+
severity: warning
19+
annotations:
20+
summary: "High CPU usage on {{ $labels.instance }}"
21+
description: "CPU usage is {{ $value | humanize }}% (threshold: 85%)."
22+
23+
- alert: CriticalCPUUsage
24+
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
25+
for: 2m
26+
labels:
27+
severity: critical
28+
annotations:
29+
summary: "Critical CPU usage on {{ $labels.instance }}"
30+
description: "CPU usage is {{ $value | humanize }}% (threshold: 95%)."
31+
32+
- alert: HighMemoryUsage
33+
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
34+
for: 5m
35+
labels:
36+
severity: warning
37+
annotations:
38+
summary: "High memory usage on {{ $labels.instance }}"
39+
description: "Memory usage is {{ $value | humanize }}% (threshold: 85%)."
40+
41+
- alert: CriticalMemoryUsage
42+
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 95
43+
for: 2m
44+
labels:
45+
severity: critical
46+
annotations:
47+
summary: "Critical memory usage on {{ $labels.instance }}"
48+
description: "Memory usage is {{ $value | humanize }}% (threshold: 95%)."
49+
50+
- alert: DiskSpaceWarning
51+
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 80
52+
for: 5m
53+
labels:
54+
severity: warning
55+
annotations:
56+
summary: "Disk space warning on {{ $labels.instance }} ({{ $labels.mountpoint }})"
57+
description: "Disk usage is {{ $value | humanize }}% (threshold: 80%)."
58+
59+
- alert: DiskSpaceCritical
60+
expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 90
61+
for: 2m
62+
labels:
63+
severity: critical
64+
annotations:
65+
summary: "Critical disk space on {{ $labels.instance }} ({{ $labels.mountpoint }})"
66+
description: "Disk usage is {{ $value | humanize }}% (threshold: 90%)."
67+
68+
- alert: PostgresDown
69+
expr: pg_up == 0
70+
for: 1m
71+
labels:
72+
severity: critical
73+
annotations:
74+
summary: "PostgreSQL is down"
75+
description: "PostgreSQL exporter reports the database is unreachable."
76+
runbook: "https://github.com/Mystery-CLI/StellarEscrow/blob/main/Troubleshooting.md#rb-001-postgresql-failure"
77+
78+
- alert: PostgresHighConnections
79+
expr: pg_stat_activity_count / pg_settings_max_connections > 0.85
80+
for: 3m
81+
labels:
82+
severity: warning
83+
annotations:
84+
summary: "PostgreSQL connection pool near limit"
85+
description: "{{ $value | humanizePercentage }} of max connections in use."
86+
87+
- alert: RedisDown
88+
expr: redis_up == 0
89+
for: 1m
90+
labels:
91+
severity: high
92+
annotations:
93+
summary: "Redis is down"
94+
description: "Redis exporter reports the cache is unreachable."
95+
runbook: "https://github.com/Mystery-CLI/StellarEscrow/blob/main/Troubleshooting.md#rb-005-redis-cache-failure"
96+
97+
- alert: RedisLowHitRate
98+
expr: rate(redis_keyspace_hits_total[5m]) / (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 0.7
99+
for: 10m
100+
labels:
101+
severity: warning
102+
annotations:
103+
summary: "Redis cache hit rate below 70%"
104+
description: "Cache hit rate is {{ $value | humanizePercentage }}."

‎monitoring/alertmanager.yml‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
global:
22
resolve_timeout: 5m
3+
smtp_smarthost: '${SMTP_HOST:-localhost:587}'
4+
smtp_from: '${SMTP_FROM:-alerts@stellarescrow.app}'
5+
smtp_auth_username: '${SMTP_USER:-}'
6+
smtp_auth_password: '${SMTP_PASSWORD:-}'
37

48
route:
59
group_by: ['alertname', 'severity']
@@ -12,21 +16,47 @@ route:
1216
severity: critical
1317
receiver: 'critical'
1418
repeat_interval: 1h
19+
- match:
20+
severity: high
21+
receiver: 'high'
22+
repeat_interval: 2h
1523

1624
receivers:
1725
- name: 'default'
1826
webhook_configs:
19-
- url: '${ALERT_WEBHOOK_URL:-http://localhost:3000/monitoring/alerts}'
27+
- url: '${ALERT_WEBHOOK_URL:-http://indexer:3000/monitoring/alerts}'
28+
send_resolved: true
29+
30+
- name: 'high'
31+
webhook_configs:
32+
- url: '${ALERT_WEBHOOK_URL:-http://indexer:3000/monitoring/alerts}'
33+
send_resolved: true
34+
email_configs:
35+
- to: '${ONCALL_EMAIL:-oncall@stellarescrow.app}'
2036
send_resolved: true
2137

2238
- name: 'critical'
2339
webhook_configs:
24-
- url: '${INCIDENT_WEBHOOK_URL:-http://localhost:3000/monitoring/alerts}'
40+
- url: '${INCIDENT_WEBHOOK_URL:-http://indexer:3000/monitoring/alerts}'
41+
send_resolved: true
42+
email_configs:
43+
- to: '${ONCALL_EMAIL:-oncall@stellarescrow.app}'
2544
send_resolved: true
45+
pagerduty_configs:
46+
- routing_key: '${PAGERDUTY_ROUTING_KEY:-}'
47+
description: '[CRITICAL] {{ .GroupLabels.alertname }}: {{ .CommonAnnotations.summary }}'
48+
severity: critical
49+
client: 'StellarEscrow Alertmanager'
50+
client_url: '${GRAFANA_URL:-http://localhost:3002}'
2651

2752
inhibit_rules:
2853
- source_match:
2954
severity: 'critical'
3055
target_match:
3156
severity: 'high'
3257
equal: ['alertname']
58+
- source_match:
59+
severity: 'critical'
60+
target_match:
61+
severity: 'warning'
62+
equal: ['alertname']

0 commit comments

Comments
 (0)