Merge pull request #714 from greatest0fallt1me/feat/circuit-breaker-metrics

Add circuit-breaker state and transition metrics

Author: Jagadeeshftw

docs/starknet/circuit-breaker.md

@@ -73,6 +73,15 @@ Circuit breaker state is exposed in the diagnostics endpoint (`GET /diagnostics/
 - **recentFailureCount**: Number of failures in the current rolling window
 - **openedAt**: Timestamp when circuit opened (milliseconds since epoch), or `null` if closed
 
+### Metrics
+
+The Starknet client exposes two process-local metrics for each RPC endpoint:
+
+- `starknet_circuit_breaker_state`, a gauge where `0=CLOSED`, `1=OPEN`, and `2=HALF_OPEN`.
+- `starknet_circuit_breaker_transitions_total`, a counter labeled with the endpoint and transition name, such as `CLOSED_to_OPEN`.
+
+Alert when an endpoint remains `OPEN` (state `1`) for longer than the normal RPC recovery window, or when `CLOSED_to_OPEN` transitions repeat rapidly. A useful starting threshold is state `1` for more than five minutes, adjusted for the configured circuit-breaker cooldown and the number of healthy fallback endpoints.
+
 ## Example Scenarios
 
 ### Scenario 1: Endpoint Degrades
@@ -112,7 +121,7 @@ Failures are tracked with timestamps in a rolling window. Old failures outside t
 
 ### Half-Open Behavior
 
-In HALF_OPEN state, only one probe call is allowed through at a time. This prevents a "thundering herd" of simultaneous recovery attempts that could overwhelm a recovering endpoint.
+In HALF_OPEN state, only one probe call is allowed through at a time. This prevents a “thundering herd” of simultaneous recovery attempts that could overwhelm a recovering endpoint.
 
 ### Success Threshold Rationale
 

src/starknet/circuit-breaker.test.ts

@@ -5,6 +5,10 @@ import {
   snapshotCircuitBreaker,
   type CircuitBreakerOptions,
 } from "./circuit-breaker.js";
+import {
+  getStarknetMetricsSnapshot,
+  resetStarknetMetrics,
+} from "./client-metrics.js";
 
 const ENDPOINT = "https://rpc.example.com";
 
@@ -40,6 +44,23 @@ describe("CircuitOpenError", () => {
   });
 });
 
+describe("circuit breaker metrics", () => {
+  beforeEach(() => resetStarknetMetrics());
+
+  it("records the current state and every transition with endpoint labels", () => {
+    const breaker = makeBreaker({ failureThreshold: 1 });
+    breaker.recordFailure();
+
+    const { gauges, counters } = getStarknetMetricsSnapshot();
+    expect(gauges['starknet_circuit_breaker_state{endpoint="https://rpc.example.com"}']).toBe(1);
+    expect(
+      counters[
+        'starknet_circuit_breaker_transitions_total{endpoint="https://rpc.example.com",transition="CLOSED_to_OPEN"}'
+      ],
+    ).toBe(1);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Initial state
 // ---------------------------------------------------------------------------

src/starknet/circuit-breaker.ts

@@ -15,8 +15,21 @@
  *             → on failure: resets to OPEN, starting the cooldown again
  */
 
+import {
+  incStarknetMetric,
+  labeledStarknetMetric,
+  setStarknetGauge,
+  STARKNET_METRICS,
+} from "./client-metrics.js";
+
 export type CircuitState = "CLOSED" | "OPEN" | "HALF_OPEN";
 
+const CIRCUIT_STATE_VALUE: Record<CircuitState, number> = {
+  CLOSED: 0,
+  OPEN: 1,
+  HALF_OPEN: 2,
+};
+
 export interface CircuitBreakerOptions {
   /** Number of failures in the rolling window that open the circuit. */
   failureThreshold: number;
@@ -51,6 +64,7 @@ export class EndpointCircuitBreaker {
   constructor(endpointUrl: string, options: CircuitBreakerOptions) {
     this.endpointUrl = endpointUrl;
     this.options = options;
+    this.recordStateMetric("CLOSED");
   }
 
   /** Current circuit state. */
@@ -148,6 +162,13 @@ export class EndpointCircuitBreaker {
   private transitionTo(next: CircuitState): void {
     const previous = this.state;
     this.state = next;
+    this.recordStateMetric(next);
+    incStarknetMetric(
+      labeledStarknetMetric(STARKNET_METRICS.CIRCUIT_BREAKER_TRANSITIONS, {
+        endpoint: this.endpointUrl,
+        transition: `${previous}_to_${next}`,
+      }),
+    );
 
     if (next === "OPEN") {
       this.openedAt = Date.now();
@@ -174,6 +195,15 @@ export class EndpointCircuitBreaker {
     }
   }
 
+  private recordStateMetric(state: CircuitState): void {
+    setStarknetGauge(
+      labeledStarknetMetric(STARKNET_METRICS.CIRCUIT_BREAKER_STATE, {
+        endpoint: this.endpointUrl,
+      }),
+      CIRCUIT_STATE_VALUE[state],
+    );
+  }
+
   private pruneOldFailures(): void {
     const cutoff = Date.now() - this.options.windowMs;
     this.failureTimestamps = this.failureTimestamps.filter((ts) => ts > cutoff);

src/starknet/client-metrics.ts

@@ -95,6 +95,17 @@ export function resetStarknetMetrics(): void {
   for (const k of Object.keys(gauges)) delete gauges[k];
 }
 
+/** Build a Prometheus-style metric key without allowing label syntax injection. */
+export function labeledStarknetMetric(
+  name: string,
+  labels: Record<string, string>,
+): string {
+  const encoded = Object.entries(labels)
+    .map(([key, value]) => `${key}="${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`)
+    .join(",");
+  return `${name}{${encoded}}`;
+}
+
 // ---------------------------------------------------------------------------
 // Structured logging
 // ---------------------------------------------------------------------------
@@ -154,4 +165,6 @@ export const STARKNET_METRICS = {
   NETWORK_INFO_FETCHES: "starknet_network_info_fetches_total",
   NETWORK_INFO_DEDUPED: "starknet_network_info_deduped_total",
   NETWORK_INFO_ERRORS: "starknet_network_info_errors_total",
+  CIRCUIT_BREAKER_STATE: "starknet_circuit_breaker_state",
+  CIRCUIT_BREAKER_TRANSITIONS: "starknet_circuit_breaker_transitions_total",
 } as const;