feat: DB password rotation drill with pool credential refresh + safety guardrails (Closes #718)

scripts/drill-db-password-rotation.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Database Password Rotation Drill Script
+# Verifies credential rotation at runtime without dropping connections.
+# Runs against staging ONLY (refuses production URLs).
+#
+# Usage: ./scripts/drill-db-password-rotation.sh [--check|--rotate|--all|--help]
+
+: "${STAGING_URL:=http://localhost:3000/health}"
+: "${DB_ROTATION_ENABLED:=false}"
+START_EPOCH=$(date +%s)
+FAILURES=0
+
+log_pass()  { echo "[PASS] $1"; }
+log_fail()  { echo "[FAIL] $1"; FAILURES=$((FAILURES + 1)); }
+log_info()  { echo "[INFO] $1"; }
+log_warn()  { echo "[WARN] $1"; }
+
+safety_check() {
+  log_info "=== Safety Guardrails ==="
+  if [[ "${STAGING_URL}" == *"api.revora.io"* ]] && [[ "${STAGING_URL}" != *"staging"* ]]; then
+    log_fail "STAGING_URL points to production — aborting."
+    exit 1
+  fi
+  if [[ "${DB_ROTATION_ENABLED}" != "true" ]]; then
+    log_warn "DB_ROTATION_ENABLED is not 'true' — server-side rotation is a no-op."
+  fi
+  if [[ -z "${NEW_DB_PASSWORD:-}" ]]; then
+    log_fail "NEW_DB_PASSWORD not set."
+    exit 1
+  fi
+  if [[ -z "${DATABASE_URL:-}" ]]; then
+    log_fail "DATABASE_URL not set."
+    exit 1
+  fi
+  log_pass "Safety checks passed"
+}
+
+check_current_connection() {
+  log_info "=== Pre-Rotation Connection Check ==="
+  if psql "$DATABASE_URL" -c "SELECT 1 AS pre_check" >/dev/null 2>&1; then
+    log_pass "Current credentials accept connections"
+  else
+    log_fail "Current credentials invalid — aborting."
+    exit 1
+  fi
+  local c
+  c=$(psql "$DATABASE_URL" -At -c "SELECT count(*) FROM pg_stat_activity WHERE state='active';" 2>/dev/null || echo "0")
+  log_info "Active connections before rotation: $c"
+}
+
+trigger_rotation() {
+  log_info "=== Triggering Rotation ==="
+  local admin_url="${STAGING_URL%/health}/admin/db/rotate-credentials"
+  local code
+  code=$(curl -s -o /tmp/rot.json -w "%{http_code}" -X POST \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer ${ADMIN_TOKEN:-skip}" \
+    -d "{\"password\":\"${NEW_DB_PASSWORD}\"}" \
+    --max-time 15 "$admin_url" 2>/dev/null || echo "000")
+  if [[ "$code" == "200" ]] || [[ "$code" == "204" ]]; then
+    log_pass "Rotation endpoint returned $code"
+  elif [[ "$code" == "404" ]]; then
+    log_warn "Endpoint not found — testing new password directly."
+    local nu
+    nu=$(echo "$DATABASE_URL" | sed "s/:[^:@]*@/:${NEW_DB_PASSWORD}@/")
+    if psql "$nu" -c "SELECT 1 AS smoke" >/dev/null 2>&1; then
+      log_pass "New credentials valid (smoke test)"
+    else
+      log_fail "New credentials rejected"
+    fi
+  else
+    log_warn "Rotation endpoint returned $code"
+  fi
+}
+
+verify_post() {
+  log_info "=== Post-Rotation Verification ==="
+  sleep 2
+  local code
+  code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$STAGING_URL" 2>/dev/null || echo "000")
+  if [[ "$code" == "200" ]]; then
+    log_pass "Health endpoint OK after rotation"
+  else
+    log_fail "Health returned $code after rotation"
+  fi
+}
+
+run_all() {
+  log_info "=== DB Password Rotation Drill ==="
+  log_info "Start: $(date -u)  Staging: $STAGING_URL  RotationEnabled: $DB_ROTATION_ENABLED"
+  safety_check && echo "" && check_current_connection && echo ""
+  trigger_rotation && echo "" && verify_post && echo ""
+  local e=$(($(date +%s) - START_EPOCH))
+  log_info "Drill completed in ${e}s"
+  if [[ "$FAILURES" -eq 0 ]]; then log_pass "All checks passed."; else log_fail "$FAILURES check(s) failed."; fi
+  echo "=== Drill Complete ==="
+  exit "$FAILURES"
+}
+
+show_help() {
+  cat <<EOF
+DB Password Rotation Drill
+Usage: $0 [--check|--rotate|--all|--help]
+Env: DATABASE_URL, NEW_DB_PASSWORD, DB_ROTATION_ENABLED, STAGING_URL, ADMIN_TOKEN
+EOF
+  exit 0
+}
+
+case "${1:---all}" in
+  --check) safety_check && check_current_connection ;;
+  --rotate) safety_check && trigger_rotation && verify_post ;;
+  --all) run_all ;;
+  --help|-h) show_help ;;
+  *) echo "Unknown: $1"; show_help ;;
+esac

src/db/pool.rotation.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Tests for database credential rotation (pool.ts).
+ *
+ * Covers:
+ *  - rotatePoolCredentials smoke-test (valid & invalid credentials)
+ *  - DB_ROTATION_ENABLED gating
+ *  - Listener notification (onCredentialsRotated)
+ *  - Metric counters
+ */
+import {
+  pool,
+  rotatePoolCredentials,
+  onCredentialsRotated,
+  clearRotationListeners,
+  closeAllPools,
+} from '../pool';
+import { globalMetrics } from '../../lib/metrics';
+import { Pool } from 'pg';
+
+jest.mock('pg', () => {
+  const actualPg = jest.requireActual('pg');
+  return { ...actualPg, Pool: jest.fn() };
+});
+
+const MockedPool = Pool as jest.MockedClass<typeof Pool>;
+
+function mockPoolQuery() {
+  return {
+    query: jest.fn().mockResolvedValue({ rows: [{ '?column?': 1 }] }),
+    end: jest.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe('rotatePoolCredentials', () => {
+  let mockNewPool: ReturnType<typeof mockPoolQuery>;
+  let incrementSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    clearRotationListeners();
+    mockNewPool = mockPoolQuery();
+    MockedPool.mockImplementation(() => mockNewPool as unknown as Pool);
+    incrementSpy = jest.spyOn(globalMetrics, 'incrementCounter').mockImplementation(() => {});
+  });
+
+  afterEach(() => { incrementSpy.mockRestore(); });
+  afterAll(async () => { await closeAllPools(); });
+
+  it('is a no-op when DB_ROTATION_ENABLED is not "true"', async () => {
+    delete process.env.DB_ROTATION_ENABLED;
+    await rotatePoolCredentials({ password: 'newpass' });
+    expect(MockedPool).not.toHaveBeenCalled();
+  });
+
+  it('creates new pool and smoke-tests on success', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    await rotatePoolCredentials({ password: 'newpass123' });
+    expect(MockedPool).toHaveBeenCalledTimes(1);
+    expect(mockNewPool.query).toHaveBeenCalledWith('SELECT 1');
+    expect(incrementSpy).toHaveBeenCalledWith(
+      'db.pool.credential_rotation', undefined, 1, expect.any(String),
+    );
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+
+  it('emits failure counter and throws on bad credentials', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    mockNewPool.query.mockRejectedValueOnce(new Error('password auth failed'));
+    await expect(rotatePoolCredentials({ password: 'bad' })).rejects.toThrow(/Credential rotation failed/);
+    expect(incrementSpy).toHaveBeenCalledWith(
+      'db.pool.credential_rotation_failed', undefined, 1, expect.any(String),
+    );
+    expect(mockNewPool.end).toHaveBeenCalled();
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+
+  it('notifies listeners on success', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    const cb = jest.fn();
+    onCredentialsRotated(cb);
+    await rotatePoolCredentials({ password: 'ok' });
+    expect(cb).toHaveBeenCalledWith('rotated', { timestamp: expect.any(String) });
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+
+  it('notifies listeners on failure', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    const cb = jest.fn();
+    onCredentialsRotated(cb);
+    mockNewPool.query.mockRejectedValueOnce(new Error('bad pw'));
+    await expect(rotatePoolCredentials({ password: 'bad' })).rejects.toThrow();
+    expect(cb).toHaveBeenCalledWith('failed', { timestamp: expect.any(String), error: 'bad pw' });
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+
+  it('swallows listener errors', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    onCredentialsRotated(() => { throw new Error('boom'); });
+    await rotatePoolCredentials({ password: 'ok' });
+    expect(incrementSpy).toHaveBeenCalledWith(
+      'db.pool.credential_rotation', undefined, 1, expect.any(String),
+    );
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+
+  it('uses config values over env vars', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    process.env.DB_HOST = 'env-host';
+    await rotatePoolCredentials({ host: 'cfg-host', password: 'pw' });
+    expect(MockedPool.mock.calls[0]?.[0]).toMatchObject({ host: 'cfg-host' });
+    delete process.env.DB_ROTATION_ENABLED;
+    delete process.env.DB_HOST;
+  });
+
+  it('clearRotationListeners removes all callbacks', async () => {
+    process.env.DB_ROTATION_ENABLED = 'true';
+    const cb = jest.fn();
+    onCredentialsRotated(cb);
+    clearRotationListeners();
+    await rotatePoolCredentials({ password: 'pw' });
+    expect(cb).not.toHaveBeenCalled();
+    delete process.env.DB_ROTATION_ENABLED;
+  });
+});

src/db/pool.ts

@@ -134,6 +134,115 @@ export async function readQuery<T extends QueryResultRow = Record<string, unknow
   return pool.query<T>(sql, params);
 }
 
+// ---------------------------------------------------------------------------
+// Credential rotation
+// ---------------------------------------------------------------------------
+
+/**
+ * Configuration for database credential rotation.
+ */
+export interface PoolCredentialConfig {
+  host?: string;
+  port?: number;
+  database?: string;
+  user?: string;
+  password?: string;
+}
+
+/**
+ * Callback invoked after credential rotation completes (success or failure).
+ */
+export type CredentialsRotatedCallback = (
+  event: 'rotated' | 'failed',
+  details: { timestamp: string; error?: string },
+) => void;
+
+const rotationListeners: CredentialsRotatedCallback[] = [];
+
+/**
+ * Register a callback to be notified of credential rotation events.
+ */
+export function onCredentialsRotated(cb: CredentialsRotatedCallback): void {
+  rotationListeners.push(cb);
+}
+
+/** Remove all credential rotation listeners (useful in test teardown). */
+export function clearRotationListeners(): void {
+  rotationListeners.length = 0;
+}
+
+function notifyListeners(
+  event: 'rotated' | 'failed',
+  details: { error?: string },
+): void {
+  const payload = { timestamp: new Date().toISOString(), ...details };
+  for (const cb of rotationListeners) {
+    try { cb(event, payload); } catch { /* swallow */ }
+  }
+}
+
+/**
+ * Rotate the primary database pool credentials at runtime.
+ *
+ * Creates a fresh pool with the supplied credentials, smoke-tests it,
+ * then swaps the exported `pool` reference. The old pool is drained
+ * gracefully (5 s delay) before closing.
+ *
+ * Guardrails:
+ *  - Gated on `DB_ROTATION_ENABLED=true` (no-op otherwise).
+ *  - Counter `db.pool.credential_rotation` incremented on success.
+ *  - Counter `db.pool.credential_rotation_failed` + listener on failure.
+ *
+ * @example
+ * await rotatePoolCredentials({ password: process.env.NEW_DB_PASSWORD });
+ */
+export async function rotatePoolCredentials(
+  config: PoolCredentialConfig,
+): Promise<void> {
+  if (process.env.DB_ROTATION_ENABLED !== 'true') {
+    return;
+  }
+
+  const newPool = new Pool({
+    host: config.host ?? process.env.DB_HOST ?? 'localhost',
+    port: config.port ?? Number(process.env.DB_PORT ?? 5432),
+    database: config.database ?? process.env.DB_NAME ?? 'revora',
+    user: config.user ?? process.env.DB_USER ?? 'postgres',
+    password: config.password ?? process.env.DB_PASSWORD ?? '',
+    max: 10,
+    idleTimeoutMillis: 30_000,
+    connectionTimeoutMillis: 2_000,
+  });
+
+  // Smoke-test: verify new credentials with a lightweight query
+  try {
+    await newPool.query('SELECT 1');
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    globalMetrics.incrementCounter(
+      'db.pool.credential_rotation_failed', undefined, 1,
+      'Number of failed credential rotation attempts',
+    );
+    notifyListeners('failed', { error: message });
+    await newPool.end().catch(() => {});
+    throw new Error(`Credential rotation failed: ${message}`);
+  }
+
+  // Swap pools — keep old alive for in-flight queries
+  const oldPool = pool;
+  (pool as Pool) = newPool;
+
+  setTimeout(() => { 
+    if (typeof oldPool.end === 'function') oldPool.end().catch(() => {});
+  }, 5_000);
+
+  globalMetrics.incrementCounter(
+    'db.pool.credential_rotation', undefined, 1,
+    'Number of successful credential rotation events',
+  );
+  notifyListeners('rotated', {});
+}
+
 // ---------------------------------------------------------------------------
 // Graceful shutdown helper
 // ---------------------------------------------------------------------------