Merge PR #140 with conflict resolution

Author: laraba9987-cmyk

MULTI_REGION_DR_ARCHITECTURE.md

@@ -0,0 +1,37 @@
+# Multi-Region Replication and Disaster Recovery Architecture
+
+## Objectives
+
+- Keep critical UI/API decision paths under **100 ms P99** by continuously scoring regional latency.
+- Preserve a **99.99% availability** posture with one healthy primary and at least one healthy secondary region.
+- Bound recovery with an operational RPO derived from replication lag and an RTO derived from the failover promotion window.
+- Require security review for replication credentials, cross-region routing, audit logs, and data residency controls before production rollout.
+
+## Runtime design
+
+1. **Active-passive regional topology**: one primary region serves writes while secondary regions continuously receive replicated state. Observer regions can collect telemetry without being eligible for write promotion.
+2. **Health assessment loop**: each region reports P99 latency, replication lag, error rate, and heartbeat freshness. The frontend disaster-recovery planner converts those signals into a deterministic assessment for dashboards and runbooks.
+3. **Failover selection**: if the primary is unhealthy, the planner chooses the healthy secondary with the lowest replication lag, then lowest latency, then lexicographic region name for deterministic tie-breaking.
+4. **Blue-green promotion**: operators promote the selected secondary into a green stack, run smoke and canary analysis, then shift traffic in controlled increments.
+5. **Monitoring and alerting**: emit alerts when the primary is degraded, the minimum healthy-secondary count is not met, or a failover region is recommended.
+
+## Disaster recovery test workflow
+
+1. Capture baseline regional metrics from production dashboards.
+2. Inject primary-region latency or heartbeat failure in a controlled game day.
+3. Verify that the assessment recommends a secondary region and records the expected alert messages.
+4. Promote the recommended region through blue-green deployment.
+5. Run canary analysis against critical paths and confirm P99 remains below 100 ms.
+6. Restore replication in the original primary and document RPO/RTO outcomes in the incident log.
+
+## Runbook checklist
+
+- Confirm at least one secondary is healthy before starting failover.
+- Freeze non-critical deployments during the disaster recovery exercise.
+- Rotate or validate cross-region replication credentials after the exercise.
+- Capture dashboard screenshots for latency, error rate, replication lag, and synthetic availability.
+- File a security-review artifact for routing, credential, and audit-log changes.
+
+## Implementation notes
+
+The `src/lib/disasterRecovery` module contains pure TypeScript helpers for assessing regions and selecting failover targets. Keeping the logic side-effect free makes it safe to reuse in dashboards, API routes, scheduled checks, and unit tests.

package-lock.json

@@ -31,7 +31,7 @@
     "test:bridge": "tsx src/lib/bridge/__tests__/bridgeStore.test.ts",
     "test:staking": "tsx src/lib/staking/__tests__/stakingCalculator.test.ts",
     "test:all": "tsx src/lib/__tests__/offlineQueue.test.ts && tsx src/lib/__tests__/OptimisticTransactionManager.test.ts && tsx src/services/__tests__/localCache.test.ts && tsx src/lib/__tests__/txQueue.test.ts && tsx src/lib/__tests__/slidingWindow.test.ts && tsx src/lib/__tests__/idbClient.test.ts && tsx src/services/__tests__/sharedStateSync.test.ts && tsx src/store/__tests__/workspaceStore.test.ts && tsx src/lib/offline/tests/networkCache.test.ts && tsx src/lib/__tests__/invalidationRegistry.test.ts && tsx src/lib/horizon/__tests__/useHorizonPagination.test.ts && tsx src/lib/cache/__tests__/redisCache.test.ts && tsx src/lib/backup/__tests__/backupRestore.test.ts && tsx src/lib/backup/__tests__/scheduler.test.ts && tsx src/lib/scheduler/__tests__/scheduler.test.ts && tsx src/lib/approvalManager/__tests__/approvalManager.test.ts && tsx src/lib/bridge/__tests__/bridgeStore.test.ts && tsx src/lib/staking/__tests__/stakingCalculator.test.ts",
-    "test:incident": "tsx src/lib/incident/__tests__/runbookAutomation.test.ts"
+    "test:dr": "tsx src/lib/disasterRecovery/__tests__/replicationPlanner.test.ts"
   },
   "dependencies": {
     "@monaco-editor/react": "^4.7.0",

package.json

@@ -0,0 +1,75 @@
+import assert from 'node:assert/strict';
+import {
+  buildDisasterRecoveryAssessment,
+  DEFAULT_DR_POLICY,
+  selectFailoverRegion,
+  type RegionHealth,
+} from '../index';
+
+const now = Date.UTC(2026, 6, 25, 12, 0, 0);
+
+const healthyRegions: RegionHealth[] = [
+  {
+    region: 'us-east-1',
+    role: 'primary',
+    latencyMsP99: 82,
+    replicationLagMs: 900,
+    errorRate: 0.001,
+    lastHeartbeatAt: now - 2_000,
+  },
+  {
+    region: 'us-west-2',
+    role: 'secondary',
+    latencyMsP99: 91,
+    replicationLagMs: 1_200,
+    errorRate: 0.002,
+    lastHeartbeatAt: now - 2_000,
+  },
+];
+
+const healthyAssessment = buildDisasterRecoveryAssessment(healthyRegions, now);
+assert.equal(healthyAssessment.meetsAvailabilityTarget, true);
+assert.equal(healthyAssessment.recommendedFailoverRegion, undefined);
+assert.deepEqual(healthyAssessment.alerts, []);
+
+const degradedRegions: RegionHealth[] = [
+  {
+    ...healthyRegions[0],
+    latencyMsP99: 140,
+    errorRate: 0.03,
+  },
+  {
+    ...healthyRegions[1],
+    replicationLagMs: 750,
+  },
+  {
+    region: 'eu-central-1',
+    role: 'secondary',
+    latencyMsP99: 88,
+    replicationLagMs: 650,
+    errorRate: 0.001,
+    lastHeartbeatAt: now - 1_000,
+  },
+];
+
+const degradedAssessment = buildDisasterRecoveryAssessment(degradedRegions, now);
+assert.equal(degradedAssessment.meetsAvailabilityTarget, false);
+assert.equal(degradedAssessment.recommendedFailoverRegion, 'eu-central-1');
+assert.equal(degradedAssessment.rpoMs, 900);
+assert.ok(degradedAssessment.alerts.includes('primary region us-east-1 is unhealthy'));
+assert.ok(degradedAssessment.alerts.includes('fail over to eu-central-1 using blue-green promotion'));
+
+const staleRegion = buildDisasterRecoveryAssessment(
+  [{ ...healthyRegions[0], lastHeartbeatAt: now - DEFAULT_DR_POLICY.heartbeatTimeoutMs - 1 }],
+  now,
+);
+assert.equal(staleRegion.primary?.healthy, false);
+assert.match(staleRegion.primary?.reasons.join('\n') ?? '', /heartbeat is stale/);
+
+assert.equal(
+  selectFailoverRegion(degradedAssessment.secondaries)?.region,
+  'eu-central-1',
+  'lowest-lag healthy secondary should be selected',
+);
+
+console.log('replicationPlanner tests passed');

src/lib/disasterRecovery/__tests__/replicationPlanner.test.ts

@@ -0,0 +1,2 @@
+export * from './replicationPlanner';
+export * from './types';

src/lib/disasterRecovery/index.ts

@@ -0,0 +1,103 @@
+import type {
+  DisasterRecoveryAssessment,
+  DisasterRecoveryPolicy,
+  RegionAssessment,
+  RegionHealth,
+} from './types';
+
+export const DEFAULT_DR_POLICY: DisasterRecoveryPolicy = {
+  criticalPathP99Ms: 100,
+  maxReplicationLagMs: 5_000,
+  maxErrorRate: 0.01,
+  heartbeatTimeoutMs: 30_000,
+  availabilityTarget: 0.9999,
+  minHealthySecondaries: 1,
+};
+
+export function assessRegion(
+  region: RegionHealth,
+  now: number,
+  policy: DisasterRecoveryPolicy = DEFAULT_DR_POLICY,
+): RegionAssessment {
+  const reasons: string[] = [];
+
+  if (region.latencyMsP99 > policy.criticalPathP99Ms) {
+    reasons.push(`p99 latency ${region.latencyMsP99}ms exceeds ${policy.criticalPathP99Ms}ms target`);
+  }
+
+  if (region.replicationLagMs > policy.maxReplicationLagMs) {
+    reasons.push(`replication lag ${region.replicationLagMs}ms exceeds ${policy.maxReplicationLagMs}ms target`);
+  }
+
+  if (region.errorRate > policy.maxErrorRate) {
+    reasons.push(`error rate ${region.errorRate} exceeds ${policy.maxErrorRate} target`);
+  }
+
+  if (now - region.lastHeartbeatAt > policy.heartbeatTimeoutMs) {
+    reasons.push(`heartbeat is stale by ${now - region.lastHeartbeatAt}ms`);
+  }
+
+  return {
+    ...region,
+    healthy: reasons.length === 0,
+    reasons,
+  };
+}
+
+export function selectFailoverRegion(assessments: RegionAssessment[]): RegionAssessment | undefined {
+  return assessments
+    .filter((region) => region.role === 'secondary' && region.healthy)
+    .sort((left, right) => {
+      if (left.replicationLagMs !== right.replicationLagMs) {
+        return left.replicationLagMs - right.replicationLagMs;
+      }
+
+      if (left.latencyMsP99 !== right.latencyMsP99) {
+        return left.latencyMsP99 - right.latencyMsP99;
+      }
+
+      return left.region.localeCompare(right.region);
+    })[0];
+}
+
+export function buildDisasterRecoveryAssessment(
+  regions: RegionHealth[],
+  now: number = Date.now(),
+  policy: DisasterRecoveryPolicy = DEFAULT_DR_POLICY,
+): DisasterRecoveryAssessment {
+  const assessments = regions.map((region) => assessRegion(region, now, policy));
+  const primary = assessments.find((region) => region.role === 'primary');
+  const secondaries = assessments.filter((region) => region.role === 'secondary');
+  const observers = assessments.filter((region) => region.role === 'observer');
+  const healthySecondaries = secondaries.filter((region) => region.healthy);
+  const recommendedFailoverRegion = primary?.healthy ? undefined : selectFailoverRegion(assessments)?.region;
+  const rpoMs = Math.max(0, ...assessments.map((region) => region.replicationLagMs));
+  const rtoMs = recommendedFailoverRegion ? Math.max(1_000, rpoMs) : 0;
+  const alerts: string[] = [];
+
+  if (!primary) {
+    alerts.push('no primary region is configured');
+  } else if (!primary.healthy) {
+    alerts.push(`primary region ${primary.region} is unhealthy`);
+  }
+
+  if (healthySecondaries.length < policy.minHealthySecondaries) {
+    alerts.push(`healthy secondary count ${healthySecondaries.length} is below required ${policy.minHealthySecondaries}`);
+  }
+
+  if (recommendedFailoverRegion) {
+    alerts.push(`fail over to ${recommendedFailoverRegion} using blue-green promotion`);
+  }
+
+  return {
+    generatedAt: now,
+    meetsAvailabilityTarget: Boolean(primary?.healthy) && healthySecondaries.length >= policy.minHealthySecondaries,
+    primary,
+    secondaries,
+    observers,
+    recommendedFailoverRegion,
+    rpoMs,
+    rtoMs,
+    alerts,
+  };
+}

src/lib/disasterRecovery/replicationPlanner.ts

@@ -0,0 +1,36 @@
+export type RegionRole = 'primary' | 'secondary' | 'observer';
+
+export interface RegionHealth {
+  region: string;
+  role: RegionRole;
+  latencyMsP99: number;
+  replicationLagMs: number;
+  errorRate: number;
+  lastHeartbeatAt: number;
+}
+
+export interface DisasterRecoveryPolicy {
+  criticalPathP99Ms: number;
+  maxReplicationLagMs: number;
+  maxErrorRate: number;
+  heartbeatTimeoutMs: number;
+  availabilityTarget: number;
+  minHealthySecondaries: number;
+}
+
+export interface RegionAssessment extends RegionHealth {
+  healthy: boolean;
+  reasons: string[];
+}
+
+export interface DisasterRecoveryAssessment {
+  generatedAt: number;
+  meetsAvailabilityTarget: boolean;
+  primary?: RegionAssessment;
+  secondaries: RegionAssessment[];
+  observers: RegionAssessment[];
+  recommendedFailoverRegion?: string;
+  rpoMs: number;
+  rtoMs: number;
+  alerts: string[];
+}