Merge pull request #364 from popsman01/feature/issue-81-performance-monitoring

feat(monitoring): implement APM performance monitoring system

Author: Mystery-CLI

PERFORMANCE.md

@@ -94,14 +94,46 @@ Metrics are collected at three layers:
 | Layer | Mechanism | Endpoint |
 |-------|-----------|---------|
 | Infrastructure | `HealthMonitor` (Rust) | `GET /health/metrics` |
+| APM (request-level) | `PerformanceService` (Rust) | `GET /performance/dashboard` |
+| APM alerts | `PerformanceService` alert rules | `GET /performance/alerts` |
+| APM history | Hourly rollup (materialized view) | `GET /performance/history` |
+| Prometheus scrape | `MonitoringService` + APM metrics | `GET /metrics` |
 | Web Vitals | `observeWebVitals()` (JS) | beacons → `POST /api/metrics` |
 | CDN | `observeCdnPerformance()` (JS) | beacons → `POST /api/metrics` |
 | SSL | `monitorTlsConnection()` (JS) | beacons → `POST /api/metrics` |
 
+### APM Endpoints
+
+| Endpoint | Description |
+|----------|-------------|
+| `GET /performance/dashboard` | Full APM snapshot: per-route stats, overall P95/P99, active alerts |
+| `GET /performance/alerts` | Active performance alerts only |
+| `GET /performance/history` | Hourly rollup from `perf_metrics_hourly` (last 24h) |
+| `POST /performance/record` | Ingest a sample `{ route, method, status, duration_ms }` |
+| `GET /metrics` | Prometheus text format (monitoring + APM metrics) |
+
+### Grafana Dashboards
+
+| Dashboard | UID | Description |
+|-----------|-----|-------------|
+| StellarEscrow Platform | `stellar-escrow-main` | Business metrics (trades, compliance, fraud) |
+| APM — Performance | `stellar-escrow-apm` | Latency (avg/P95/P99), error rate, throughput, DB query time |
+| Code Quality | `code-quality` | CI lint/security metrics |
+
+### Alert Rules
+
+| File | Group | Alerts |
+|------|-------|--------|
+| `alert_rules.yml` | `stellar_escrow_platform` | Error rate, disputes, fraud |
+| `alert_rules_security.yml` | `stellar_escrow_compliance` | AML, compliance blocks |
+| `alert_rules_performance.yml` | `stellar_escrow_performance` | Latency (avg/P95), error rate, DB queries, throughput |
+
 ### Key Metrics to Watch
 
 - **TTFB** < 200ms (good), < 800ms (acceptable)
 - **LCP** < 2.5s
+- **Avg response time** < 500ms (warning at 500ms, critical at 2000ms)
+- **P95 response time** < 1000ms (warning at 1000ms, critical at 5000ms)
 - **DB query mean** < 10ms for hot paths (`get_events`, `search_trades`)
 - **Redis hit rate** > 80% under normal load
 - **Indexer memory** < 256MB under normal load

docker-compose.yml

@@ -202,6 +202,9 @@ services:
       - "127.0.0.1:9090:9090"
     volumes:
       - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro
+      - ./monitoring/alert_rules_security.yml:/etc/prometheus/alert_rules_security.yml:ro
+      - ./monitoring/alert_rules_performance.yml:/etc/prometheus/alert_rules_performance.yml:ro
       - prometheus_data:/prometheus
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"

indexer/migrations/20260329000000_apm_views.sql

@@ -0,0 +1,30 @@
+-- APM query helpers: materialized view + index for fast dashboard queries
+
+-- Hourly rollup view for performance metrics
+CREATE MATERIALIZED VIEW IF NOT EXISTS perf_metrics_hourly AS
+SELECT
+    date_trunc('hour', recorded_at)          AS hour,
+    route,
+    method,
+    COUNT(*)                                  AS requests,
+    SUM(CASE WHEN is_error THEN 1 ELSE 0 END) AS errors,
+    AVG(duration_ms)                          AS avg_ms,
+    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY duration_ms) AS p95_ms,
+    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY duration_ms) AS p99_ms
+FROM performance_metrics
+GROUP BY 1, 2, 3
+WITH NO DATA;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_perf_hourly_pk
+    ON perf_metrics_hourly (hour DESC, route, method);
+
+-- Refresh function (called by background job or cron)
+CREATE OR REPLACE FUNCTION refresh_perf_hourly()
+RETURNS void LANGUAGE sql AS $$
+    REFRESH MATERIALIZED VIEW CONCURRENTLY perf_metrics_hourly;
+$$;
+
+-- Index to speed up alert history queries
+CREATE INDEX IF NOT EXISTS idx_perf_alerts_unresolved
+    ON performance_alerts (triggered_at DESC)
+    WHERE resolved_at IS NULL;

indexer/src/database.rs

@@ -1155,7 +1155,43 @@ impl Database {
         .execute(&self.pool)
         .await?;
         Ok(())
-    pub async fn get_integration_deliveries(
+    }
+
+    /// Query the hourly APM rollup materialized view for the last `hours` hours.
+    pub async fn get_perf_hourly_rollup(
+        &self,
+        hours: i64,
+    ) -> Result<Vec<serde_json::Value>, sqlx::Error> {
+        let rows = sqlx::query(
+            r#"
+            SELECT hour, route, method, requests, errors, avg_ms, p95_ms, p99_ms
+            FROM perf_metrics_hourly
+            WHERE hour >= NOW() - ($1 || ' hours')::INTERVAL
+            ORDER BY hour DESC
+            LIMIT 500
+            "#,
+        )
+        .bind(hours)
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows
+            .into_iter()
+            .map(|r| {
+                use sqlx::Row;
+                serde_json::json!({
+                    "hour": r.get::<chrono::DateTime<chrono::Utc>, _>("hour"),
+                    "route": r.get::<String, _>("route"),
+                    "method": r.get::<String, _>("method"),
+                    "requests": r.get::<i64, _>("requests"),
+                    "errors": r.get::<i64, _>("errors"),
+                    "avg_ms": r.get::<f64, _>("avg_ms"),
+                    "p95_ms": r.get::<f64, _>("p95_ms"),
+                    "p99_ms": r.get::<f64, _>("p99_ms"),
+                })
+            })
+            .collect())
+    }
         &self,
         connector_id: Option<&str>,
         limit: i64,

indexer/src/handlers.rs

@@ -569,6 +569,34 @@ pub async fn get_performance_alerts(
     }))
 }
 
+#[derive(serde::Deserialize)]
+pub struct PerfRecordBody {
+    pub route: String,
+    pub method: String,
+    pub status: u16,
+    pub duration_ms: u64,
+}
+
+/// POST /performance/record — ingest a single APM sample (used by middleware / external agents).
+pub async fn record_performance_sample(
+    State(state): State<AppState>,
+    Json(body): Json<PerfRecordBody>,
+) -> StatusCode {
+    state
+        .performance_service
+        .record(&body.route, &body.method, body.status, body.duration_ms)
+        .await;
+    StatusCode::NO_CONTENT
+}
+
+/// GET /performance/history — hourly rollup from the materialized view.
+pub async fn get_performance_history(
+    State(state): State<AppState>,
+) -> Result<Json<serde_json::Value>, AppError> {
+    let rows = state.database.get_perf_hourly_rollup(24).await?;
+    Ok(Json(serde_json::json!({ "hours": rows })))
+}
+
 // =============================================================================
 // Analytics Handlers
 // =============================================================================
@@ -854,11 +882,42 @@ pub async fn get_monitoring_alerts(
     Ok(Json(serde_json::to_value(&alerts).unwrap_or_default()))
 }
 
-/// GET /monitoring/metrics — Prometheus-format metrics.
+/// GET /monitoring/metrics — Prometheus-format metrics (monitoring + APM).
 pub async fn get_prometheus_metrics(
     State(state): State<AppState>,
 ) -> axum::response::Response<String> {
-    let body = state.monitoring_service.prometheus_metrics();
+    let mut body = state.monitoring_service.prometheus_metrics();
+
+    // Append APM metrics from the performance service
+    let dash = state.performance_service.dashboard().await;
+    let apm = format!(
+        "\n# HELP stellar_escrow_api_avg_response_ms Average API response time in ms\n\
+         # TYPE stellar_escrow_api_avg_response_ms gauge\n\
+         stellar_escrow_api_avg_response_ms {avg}\n\
+         # HELP stellar_escrow_api_p95_response_ms P95 API response time in ms\n\
+         # TYPE stellar_escrow_api_p95_response_ms gauge\n\
+         stellar_escrow_api_p95_response_ms {p95}\n\
+         # HELP stellar_escrow_api_p99_response_ms P99 API response time in ms\n\
+         # TYPE stellar_escrow_api_p99_response_ms gauge\n\
+         stellar_escrow_api_p99_response_ms {p99}\n\
+         # HELP stellar_escrow_api_requests_total Total API requests recorded\n\
+         # TYPE stellar_escrow_api_requests_total counter\n\
+         stellar_escrow_api_requests_total {total}\n\
+         # HELP stellar_escrow_api_requests_per_minute API requests per minute\n\
+         # TYPE stellar_escrow_api_requests_per_minute gauge\n\
+         stellar_escrow_api_requests_per_minute {rpm}\n\
+         # HELP stellar_escrow_active_perf_alerts Number of active performance alerts\n\
+         # TYPE stellar_escrow_active_perf_alerts gauge\n\
+         stellar_escrow_active_perf_alerts {alerts}\n",
+        avg = dash.overall.avg_ms,
+        p95 = dash.overall.p95_ms,
+        p99 = dash.overall.p99_ms,
+        total = dash.overall.total_requests,
+        rpm = dash.overall.requests_per_minute,
+        alerts = dash.active_alerts.len(),
+    );
+    body.push_str(&apm);
+
     axum::response::Response::builder()
         .status(200)
         .header("Content-Type", "text/plain; version=0.0.4")

indexer/src/main.rs

@@ -292,6 +292,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         // Performance monitoring
         .route("/performance/dashboard", get(get_performance_dashboard))
         .route("/performance/alerts", get(get_performance_alerts))
+        .route("/performance/record", post(record_performance_sample))
+        .route("/performance/history", get(get_performance_history))
         // Compliance
         .route("/compliance/check", post(run_compliance_check))
         .route("/compliance/status/:address", get(get_compliance_status))

indexer/src/monitoring_service/metrics.rs

@@ -98,3 +98,11 @@ pub const METRIC_API_REQUEST_DURATION_MS: &str = "stellar_escrow_api_request_dur
 pub const METRIC_WEBSOCKET_CONNECTIONS: &str = "stellar_escrow_websocket_connections";
 pub const METRIC_FRAUD_ALERTS: &str = "stellar_escrow_fraud_alerts_total";
 pub const METRIC_ERROR_RATE: &str = "stellar_escrow_error_rate";
+
+// APM-specific metrics exposed to Prometheus
+pub const METRIC_API_AVG_RESPONSE_MS: &str = "stellar_escrow_api_avg_response_ms";
+pub const METRIC_API_P95_RESPONSE_MS: &str = "stellar_escrow_api_p95_response_ms";
+pub const METRIC_API_P99_RESPONSE_MS: &str = "stellar_escrow_api_p99_response_ms";
+pub const METRIC_API_REQUESTS_TOTAL: &str = "stellar_escrow_api_requests_total";
+pub const METRIC_API_REQUESTS_PER_MINUTE: &str = "stellar_escrow_api_requests_per_minute";
+pub const METRIC_ACTIVE_PERF_ALERTS: &str = "stellar_escrow_active_perf_alerts";

monitoring/alert_rules_performance.yml

@@ -0,0 +1,74 @@
+groups:
+  - name: stellar_escrow_performance
+    rules:
+      - alert: HighAvgLatency
+        expr: stellar_escrow_api_avg_response_ms > 500
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High average API latency"
+          description: "Average response time is {{ $value }}ms (threshold: 500ms)."
+
+      - alert: CriticalAvgLatency
+        expr: stellar_escrow_api_avg_response_ms > 2000
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Critical average API latency"
+          description: "Average response time is {{ $value }}ms (threshold: 2000ms)."
+
+      - alert: HighP95Latency
+        expr: stellar_escrow_api_p95_response_ms > 1000
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High P95 API latency"
+          description: "P95 response time is {{ $value }}ms (threshold: 1000ms)."
+
+      - alert: CriticalP95Latency
+        expr: stellar_escrow_api_p95_response_ms > 5000
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Critical P95 API latency"
+          description: "P95 response time is {{ $value }}ms (threshold: 5000ms)."
+
+      - alert: HighAPIErrorRate
+        expr: stellar_escrow_error_rate > 5
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High API error rate"
+          description: "Error rate is {{ $value }}% (threshold: 5%)."
+
+      - alert: CriticalAPIErrorRate
+        expr: stellar_escrow_error_rate > 20
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Critical API error rate"
+          description: "Error rate is {{ $value }}% (threshold: 20%)."
+
+      - alert: SlowDBQueries
+        expr: stellar_escrow_db_query_duration_ms > 500
+        for: 3m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Slow database queries detected"
+          description: "DB query duration is {{ $value }}ms (threshold: 500ms)."
+
+      - alert: LowThroughput
+        expr: stellar_escrow_api_requests_per_minute < 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Very low API throughput"
+          description: "Requests per minute dropped to {{ $value }}."