Merge pull request #359 from olawaleakanbi035-maker/feat/oracle-health-monitoring

feat(oracle): expose health endpoints and lag monitoring

Author: Otaiki1

oracle/README.md

@@ -104,6 +104,61 @@ npm test
 - ✅ Already-finalized raffle handling
 - ✅ Error handling and retry behavior
 
+## Health & Monitoring
+
+### Endpoints
+
+| Endpoint | Description |
+|---|---|
+| `GET /health` | Liveness check — returns `healthy`/`unhealthy` + pending lag count |
+| `GET /oracle/status` | Full status — metrics, lag, RPC health, multi-oracle state, recent errors |
+
+**`GET /health` response:**
+```json
+{
+  "status": "healthy",
+  "timestamp": "2026-04-23T12:00:00.000Z",
+  "pendingLagRequests": 0
+}
+```
+
+**`GET /oracle/status` response (abbreviated):**
+```json
+{
+  "status": "healthy",
+  "metrics": {
+    "queueDepth": 2,
+    "lastProcessedAt": "2026-04-23T11:59:00.000Z",
+    "totalProcessed": 142,
+    "totalFailed": 1,
+    "successRate": "99.30%",
+    "streamStatus": "connected"
+  },
+  "lag": {
+    "pendingCount": 1,
+    "pendingRequests": [
+      { "requestId": "req-abc", "raffleId": 7, "requestedAtLedger": 1234500 }
+    ]
+  },
+  "rpc": [{ "url": "https://soroban-testnet.stellar.org", "healthy": true }],
+  "recentErrors": []
+}
+```
+
+### Lag Alerting
+
+`LagMonitorService` tracks every `RandomnessRequested` event by ledger number. If a request is not fulfilled within **100 ledgers** (~8 minutes on Stellar), an `[ALERT]` log is emitted:
+
+```
+[ALERT] Request req-abc for raffle 7 not fulfilled within 100 ledgers. Lag: 103
+```
+
+### Recommended Monitoring Setup
+
+- **Liveness probe**: `GET /health` — use as Kubernetes `livenessProbe`
+- **Alerting**: Scrape logs for `[ALERT]` pattern or wire a log aggregator
+- **Metrics**: `queueDepth > 10` warns; `queueDepth > 50` marks unhealthy
+- **Heartbeat**: Oracle pings the contract every `HEARTBEAT_INTERVAL_MS` (default: 1 hour)
 ## Queue & Redis
 
 The oracle uses **Bull** (backed by Redis) to reliably process randomness requests.

oracle/src/health/health.controller.ts

@@ -1,5 +1,6 @@
 import { Controller, Get } from '@nestjs/common';
 import { HealthService } from './health.service';
+import { LagMonitorService } from './lag-monitor.service';
 import { OracleRegistryService } from '../multi-oracle/oracle-registry.service';
 import { MultiOracleCoordinatorService } from '../multi-oracle/multi-oracle-coordinator.service';
 import { TxSubmitterService } from '../submitter/tx-submitter.service';
@@ -8,6 +9,7 @@ import { TxSubmitterService } from '../submitter/tx-submitter.service';
 export class HealthController {
   constructor(
     private readonly healthService: HealthService,
+    private readonly lagMonitor: LagMonitorService,
     private readonly oracleRegistry: OracleRegistryService,
     private readonly multiOracleCoordinator: MultiOracleCoordinatorService,
     private readonly txSubmitter: TxSubmitterService,
@@ -19,6 +21,7 @@ export class HealthController {
     return {
       status: isHealthy ? 'healthy' : 'unhealthy',
       timestamp: new Date().toISOString(),
+      pendingLagRequests: this.lagMonitor.getPendingCount(),
     };
   }
 
@@ -28,8 +31,8 @@ export class HealthController {
     const isHealthy = this.healthService.isHealthy();
     const multiOracleConfig = this.oracleRegistry.getConfig();
     const pendingTrackers = this.multiOracleCoordinator.getPendingTrackers();
-    
     const rpcStatus = await this.txSubmitter.getRpcStatus();
+    const pendingLag = this.lagMonitor.getPendingRequests();
 
     return {
       status: isHealthy ? 'healthy' : 'unhealthy',
@@ -50,6 +53,15 @@ export class HealthController {
         streamUptimeMs: metrics.streamUptimeMs,
         lastStreamError: metrics.lastStreamError,
       },
+      lag: {
+        pendingCount: pendingLag.length,
+        pendingRequests: pendingLag.map(r => ({
+          requestId: r.requestId,
+          raffleId: r.raffleId,
+          requestedAtLedger: r.requestedAtLedger,
+          age: new Date().toISOString(),
+        })),
+      },
       multiOracle: {
         enabled: multiOracleConfig.enabled,
         mode: multiOracleConfig.enabled ? 'multi-oracle' : 'single-oracle',

oracle/src/health/health.module.ts

@@ -5,10 +5,11 @@ import { LagMonitorService } from './lag-monitor.service';
 import { HeartbeatService } from './heartbeat.service';
 import { ContractService } from '../contract/contract.service';
 import { TxSubmitterService } from '../submitter/tx-submitter.service';
+import { FeeEstimatorService } from '../submitter/fee-estimator.service';
 
 @Module({
   controllers: [HealthController],
-  providers: [HealthService, LagMonitorService, HeartbeatService, ContractService, TxSubmitterService],
+  providers: [HealthService, LagMonitorService, HeartbeatService, ContractService, TxSubmitterService, FeeEstimatorService],
   exports: [HealthService, LagMonitorService, HeartbeatService],
 })
 export class HealthModule {}

oracle/src/health/lag-monitor.service.ts

@@ -47,4 +47,8 @@ export class LagMonitorService {
   getPendingCount(): number {
     return this.pendingRequests.size;
   }
+
+  getPendingRequests(): PendingRequest[] {
+    return Array.from(this.pendingRequests.values());
+  }
 }