Skip to content

Commit 8652d15

Browse files
feat: enable automated RDS and Redis credential rotation
1 parent 8ddf1e7 commit 8652d15

40 files changed

Lines changed: 3590 additions & 192 deletions

backend/.env.example

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ PGUSER=
4343
PGPASSWORD=
4444
# PostgreSQL database name | string | # Optional (default: empty)
4545
PGDATABASE=
46+
# AWS Secrets Manager ARN for RDS credentials; overrides static PG* credentials after hydration | ARN | # Optional (default: empty)
47+
DB_SECRET_ARN=
48+
# Alias for DB_SECRET_ARN | ARN | # Optional (default: empty)
49+
DATABASE_SECRET_ARN=
50+
# Database secret refresh TTL | integer milliseconds | # Optional (default: 300000)
51+
DB_SECRET_CACHE_TTL_MS=300000
52+
# AWS region used by the Secrets Manager client in AWS/ECS deployments | AWS region | # Optional (default: SDK chain)
53+
AWS_REGION=
54+
# Alias region variable recognized by the AWS SDK | AWS region | # Optional (default: SDK chain)
55+
AWS_DEFAULT_REGION=
4656
# Maximum number of connections in the PostgreSQL pool | integer | # Optional (default: 10)
4757
DB_POOL_SIZE=10
4858
# CI marker used by tooling and env validation checks | string | # Optional (default: empty)
@@ -91,6 +101,14 @@ RATE_LIMIT_WRITE_BURST_MAX=3
91101
# --- Queue and caching ---
92102
# Redis URL for BullMQ queues/workers | URL | # Optional (default: redis://localhost:6379)
93103
REDIS_URL=redis://localhost:6379
104+
# AWS Secrets Manager ARN for Redis endpoint/AUTH token JSON; dynamically builds REDIS_URL | ARN | # Optional (default: empty)
105+
REDIS_SECRET_ARN=
106+
# Alias for REDIS_SECRET_ARN | ARN | # Optional (default: empty)
107+
REDIS_AUTH_SECRET_ARN=
108+
# Redis secret refresh TTL | integer milliseconds | # Optional (default: 300000)
109+
REDIS_SECRET_CACHE_TTL_MS=300000
110+
# Redis availability probe cache TTL | integer milliseconds | # Optional (default: 30000)
111+
REDIS_AVAILABILITY_CACHE_TTL_MS=30000
94112
# Toggle in-memory portfolio cache for experiments | boolean | # Optional (default: false)
95113
USE_MEMORY_CACHE=false
96114

backend/package-lock.json

Lines changed: 2232 additions & 61 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

backend/package.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
"db:migrate:status": "tsx src/db/migrate.ts -- --status",
1313
"db:setup": "tsx src/db/migrate.ts",
1414
"db:seed:e2e": "tsx src/db/seed.e2e.ts",
15-
1615
"test:watch": "vitest",
1716
"test:coverage": "vitest run --coverage",
1817
"test:ci": "npm run test:coverage",
@@ -34,6 +33,7 @@
3433
"indexer:reindex": "tsx scripts/reindex-events.ts"
3534
},
3635
"dependencies": {
36+
"@aws-sdk/client-secrets-manager": "^3.1098.0",
3737
"@opentelemetry/api": "^1.9.1",
3838
"@opentelemetry/exporter-trace-otlp-http": "^0.219.0",
3939
"@opentelemetry/instrumentation": "^0.219.0",
@@ -45,6 +45,7 @@
4545
"@sentry/node": "^9.14.0",
4646
"@sentry/profiling-node": "^9.14.0",
4747
"@stellar/stellar-sdk": "^12.0.1",
48+
"axios": "^1.7.9",
4849
"better-sqlite3": "^12.11.1",
4950
"bullmq": "^5.69.3",
5051
"cors": "^2.8.5",
@@ -55,10 +56,9 @@
5556
"jsonwebtoken": "^9.0.3",
5657
"newrelic": "^12.24.0",
5758
"node-cron": "^3.0.3",
59+
"node-telegram-bot-api": "^0.67.0",
5860
"nodemailer": "^8.0.1",
59-
"node-telegram-bot-api": "^0.67.0",
60-
"axios": "^1.7.9",
61-
"pdfkit": "^0.17.2",
61+
"pdfkit": "^0.17.2",
6262
"pg": "^8.11.3",
6363
"pino": "^9.6.0",
6464
"prom-client": "^15.1.3",
@@ -74,8 +74,8 @@
7474
"@types/js-yaml": "^4.0.9",
7575
"@types/jsonwebtoken": "^9.0.10",
7676
"@types/node": "^20.19.39",
77-
"@types/node-telegram-bot-api": "^0.64.15",
7877
"@types/node-cron": "^3.0.11",
78+
"@types/node-telegram-bot-api": "^0.64.15",
7979
"@types/nodemailer": "^7.0.10",
8080
"@types/pdfkit": "^0.17.5",
8181
"@types/pg": "^8.10.9",
Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
import { fetchJsonSecret, maskSecretId, stableFingerprint } from './secretsManager.js'
2+
import { logger } from '../utils/logger.js'
3+
4+
const DEFAULT_SECRET_REFRESH_MS = 5 * 60 * 1000
5+
const MIN_SECRET_REFRESH_MS = 30 * 1000
6+
7+
export interface SecretRefreshResult {
8+
configured: boolean
9+
refreshed: boolean
10+
changed: boolean
11+
fingerprint?: string
12+
}
13+
14+
export interface RuntimeSecretRefreshOptions {
15+
onDatabaseCredentialsChanged?: () => Promise<void> | void
16+
onRedisCredentialsChanged?: () => Promise<void> | void
17+
}
18+
19+
let databaseFingerprint: string | null = null
20+
let databaseLastRefreshAt = 0
21+
let redisFingerprint: string | null = null
22+
let redisLastRefreshAt = 0
23+
let databaseRefreshTimer: NodeJS.Timeout | null = null
24+
let redisRefreshTimer: NodeJS.Timeout | null = null
25+
26+
function envSecretId(...names: string[]): string | undefined {
27+
for (const name of names) {
28+
const value = process.env[name]?.trim()
29+
if (value) return value
30+
}
31+
return undefined
32+
}
33+
34+
function refreshIntervalMs(envName: string): number {
35+
const raw = process.env[envName]?.trim()
36+
const parsed = raw ? Number.parseInt(raw, 10) : NaN
37+
if (Number.isFinite(parsed) && parsed >= MIN_SECRET_REFRESH_MS) return parsed
38+
return DEFAULT_SECRET_REFRESH_MS
39+
}
40+
41+
function asString(value: unknown): string | undefined {
42+
if (typeof value === 'string' && value.trim().length > 0) return value.trim()
43+
if (typeof value === 'number' && Number.isFinite(value)) return String(value)
44+
return undefined
45+
}
46+
47+
function asBoolean(value: unknown, defaultValue = false): boolean {
48+
if (typeof value === 'boolean') return value
49+
if (typeof value === 'string') {
50+
const normalized = value.trim().toLowerCase()
51+
if (['true', '1', 'yes', 'y'].includes(normalized)) return true
52+
if (['false', '0', 'no', 'n'].includes(normalized)) return false
53+
}
54+
return defaultValue
55+
}
56+
57+
function setEnvIfDefined(name: string, value: string | undefined): void {
58+
if (value !== undefined) process.env[name] = value
59+
}
60+
61+
function redactUrl(url: string): string {
62+
return url.replace(/:\/\/[^@]*@/, '://***@')
63+
}
64+
65+
function endpointHost(value: string): string {
66+
const trimmed = value.trim()
67+
if (!trimmed.includes('://')) return trimmed.split(':')[0]
68+
69+
try {
70+
return new URL(trimmed).hostname
71+
} catch {
72+
return trimmed.split(':')[0]
73+
}
74+
}
75+
76+
function endpointPort(value: string, defaultPort: string): string {
77+
const trimmed = value.trim()
78+
if (!trimmed.includes('://')) {
79+
const parts = trimmed.split(':')
80+
return parts.length > 1 && parts[1] ? parts[1] : defaultPort
81+
}
82+
83+
try {
84+
return new URL(trimmed).port || defaultPort
85+
} catch {
86+
return defaultPort
87+
}
88+
}
89+
90+
function buildRedisUrl(secret: Record<string, unknown>): string | undefined {
91+
const explicitUrl = asString(secret.url) || asString(secret.redis_url) || asString(secret.REDIS_URL)
92+
if (explicitUrl) return explicitUrl
93+
94+
const hostSource =
95+
asString(secret.primary_endpoint_address) ||
96+
asString(secret.host) ||
97+
asString(secret.hostname) ||
98+
asString(secret.endpoint) ||
99+
asString(secret.address)
100+
if (!hostSource) return undefined
101+
102+
const port = asString(secret.port) || endpointPort(hostSource, '6379')
103+
const host = endpointHost(hostSource)
104+
const authToken =
105+
asString(secret.auth_token) ||
106+
asString(secret.password) ||
107+
asString(secret.token) ||
108+
asString(secret.REDIS_AUTH_TOKEN)
109+
const tls = asBoolean(secret.tls, asBoolean(secret.transit_encryption_enabled, true))
110+
const protocol = tls ? 'rediss' : 'redis'
111+
const auth = authToken ? `:${encodeURIComponent(authToken)}@` : ''
112+
113+
return `${protocol}://${auth}${host}:${port}`
114+
}
115+
116+
export function hasDatabaseSecretConfigured(): boolean {
117+
return Boolean(envSecretId('DB_SECRET_ARN', 'DATABASE_SECRET_ARN', 'PG_SECRET_ARN'))
118+
}
119+
120+
export function hasRedisSecretConfigured(): boolean {
121+
return Boolean(envSecretId('REDIS_SECRET_ARN', 'REDIS_AUTH_SECRET_ARN'))
122+
}
123+
124+
export async function refreshDatabaseSecret(options: { force?: boolean } = {}): Promise<SecretRefreshResult> {
125+
const secretId = envSecretId('DB_SECRET_ARN', 'DATABASE_SECRET_ARN', 'PG_SECRET_ARN')
126+
if (!secretId) return { configured: false, refreshed: false, changed: false }
127+
128+
const now = Date.now()
129+
if (!options.force && databaseLastRefreshAt > 0 && now - databaseLastRefreshAt < refreshIntervalMs('DB_SECRET_CACHE_TTL_MS')) {
130+
return { configured: true, refreshed: false, changed: false, fingerprint: databaseFingerprint ?? undefined }
131+
}
132+
133+
const secret = await fetchJsonSecret(secretId)
134+
const databaseUrl = asString(secret.url) || asString(secret.DATABASE_URL) || asString(secret.connectionString)
135+
const username = asString(secret.username) || asString(secret.user) || asString(secret.PGUSER)
136+
const password = asString(secret.password) || asString(secret.PGPASSWORD)
137+
const host = asString(secret.host) || asString(secret.hostname) || asString(secret.endpoint) || asString(secret.address)
138+
const port = asString(secret.port) || '5432'
139+
const database =
140+
asString(secret.dbname) ||
141+
asString(secret.database) ||
142+
asString(secret.db_name) ||
143+
asString(secret.PGDATABASE)
144+
145+
setEnvIfDefined('DATABASE_URL', databaseUrl)
146+
setEnvIfDefined('PGUSER', username)
147+
setEnvIfDefined('PGPASSWORD', password)
148+
setEnvIfDefined('PGHOST', host)
149+
setEnvIfDefined('PGPORT', port)
150+
setEnvIfDefined('PGDATABASE', database)
151+
152+
// Keep the legacy DB_* names synchronized for containers or scripts that still inspect them.
153+
setEnvIfDefined('DB_USER', username)
154+
setEnvIfDefined('DB_PASSWORD', password)
155+
setEnvIfDefined('DB_HOST', host)
156+
setEnvIfDefined('DB_NAME', database)
157+
158+
const applied = {
159+
databaseUrl: databaseUrl ?? null,
160+
username: username ?? null,
161+
password: password ?? null,
162+
host: host ?? null,
163+
port,
164+
database: database ?? process.env.PGDATABASE ?? null,
165+
}
166+
const fingerprint = stableFingerprint(applied)
167+
const changed = databaseFingerprint !== null && databaseFingerprint !== fingerprint
168+
databaseFingerprint = fingerprint
169+
databaseLastRefreshAt = now
170+
171+
logger.info('[SECRETS] Refreshed database credentials from Secrets Manager', {
172+
secretId: maskSecretId(secretId),
173+
changed,
174+
hostConfigured: Boolean(host),
175+
databaseConfigured: Boolean(database || process.env.PGDATABASE),
176+
})
177+
178+
return { configured: true, refreshed: true, changed, fingerprint }
179+
}
180+
181+
export async function refreshRedisSecret(options: { force?: boolean } = {}): Promise<SecretRefreshResult> {
182+
const secretId = envSecretId('REDIS_SECRET_ARN', 'REDIS_AUTH_SECRET_ARN')
183+
if (!secretId) return { configured: false, refreshed: false, changed: false }
184+
185+
const now = Date.now()
186+
if (!options.force && redisLastRefreshAt > 0 && now - redisLastRefreshAt < refreshIntervalMs('REDIS_SECRET_CACHE_TTL_MS')) {
187+
return { configured: true, refreshed: false, changed: false, fingerprint: redisFingerprint ?? undefined }
188+
}
189+
190+
const secret = await fetchJsonSecret(secretId)
191+
const redisUrl = buildRedisUrl(secret)
192+
const authToken =
193+
asString(secret.auth_token) ||
194+
asString(secret.password) ||
195+
asString(secret.token) ||
196+
asString(secret.REDIS_AUTH_TOKEN)
197+
198+
setEnvIfDefined('REDIS_URL', redisUrl)
199+
setEnvIfDefined('REDIS_AUTH_TOKEN', authToken)
200+
201+
const fingerprint = stableFingerprint({ redisUrl: redisUrl ?? null, authToken: authToken ?? null })
202+
const changed = redisFingerprint !== null && redisFingerprint !== fingerprint
203+
redisFingerprint = fingerprint
204+
redisLastRefreshAt = now
205+
206+
logger.info('[SECRETS] Refreshed Redis credentials from Secrets Manager', {
207+
secretId: maskSecretId(secretId),
208+
changed,
209+
redisUrl: redisUrl ? redactUrl(redisUrl) : '<not-configured>',
210+
})
211+
212+
return { configured: true, refreshed: true, changed, fingerprint }
213+
}
214+
215+
export async function initializeRuntimeSecrets(): Promise<void> {
216+
await Promise.all([
217+
refreshDatabaseSecret({ force: true }).catch((error: unknown) => {
218+
logger.error('[SECRETS] Failed to load database credentials from Secrets Manager', {
219+
error: error instanceof Error ? error.message : String(error),
220+
})
221+
throw error
222+
}),
223+
refreshRedisSecret({ force: true }).catch((error: unknown) => {
224+
logger.error('[SECRETS] Failed to load Redis credentials from Secrets Manager', {
225+
error: error instanceof Error ? error.message : String(error),
226+
})
227+
throw error
228+
}),
229+
])
230+
}
231+
232+
export function startRuntimeSecretRefresh(options: RuntimeSecretRefreshOptions = {}): void {
233+
stopRuntimeSecretRefresh()
234+
235+
if (hasDatabaseSecretConfigured()) {
236+
databaseRefreshTimer = setInterval(() => {
237+
void refreshDatabaseSecret({ force: true })
238+
.then(async (result) => {
239+
if (result.changed) await options.onDatabaseCredentialsChanged?.()
240+
})
241+
.catch((error: unknown) => {
242+
logger.error('[SECRETS] Database credential refresh failed', {
243+
error: error instanceof Error ? error.message : String(error),
244+
})
245+
})
246+
}, refreshIntervalMs('DB_SECRET_CACHE_TTL_MS'))
247+
databaseRefreshTimer.unref?.()
248+
}
249+
250+
if (hasRedisSecretConfigured()) {
251+
redisRefreshTimer = setInterval(() => {
252+
void refreshRedisSecret({ force: true })
253+
.then(async (result) => {
254+
if (result.changed) await options.onRedisCredentialsChanged?.()
255+
})
256+
.catch((error: unknown) => {
257+
logger.error('[SECRETS] Redis credential refresh failed', {
258+
error: error instanceof Error ? error.message : String(error),
259+
})
260+
})
261+
}, refreshIntervalMs('REDIS_SECRET_CACHE_TTL_MS'))
262+
redisRefreshTimer.unref?.()
263+
}
264+
}
265+
266+
export function stopRuntimeSecretRefresh(): void {
267+
if (databaseRefreshTimer) clearInterval(databaseRefreshTimer)
268+
if (redisRefreshTimer) clearInterval(redisRefreshTimer)
269+
databaseRefreshTimer = null
270+
redisRefreshTimer = null
271+
}
272+
273+
export function __resetRuntimeSecretStateForTests(): void {
274+
stopRuntimeSecretRefresh()
275+
databaseFingerprint = null
276+
databaseLastRefreshAt = 0
277+
redisFingerprint = null
278+
redisLastRefreshAt = 0
279+
}

0 commit comments

Comments
 (0)