feat: alerting threshold rules, SSE stream, dashboard badge (#250) (#255)

Implements issue #250 (part of Token Insights #247):
- Alert types, event bus, DB migrations (SQLite + PG)
- Config: 7 new settings with env overrides, clamping, webhook URL validation
- AlertService: per-request threshold evaluation, anomaly detection, cooldown deduplication, webhook delivery
- HTTP endpoints: history, config GET/POST, SSE stream, acknowledge, acknowledge-all
- Dashboard: AlertsView, useAlertStream SSE hook, unacknowledged count nav badge

Author: tombii

apps/server/src/server.ts

@@ -21,7 +21,7 @@ import {
 	DatabaseFactory,
 	initPayloadEncryption,
 } from "@better-ccflare/database";
-import { APIRouter, AuthService } from "@better-ccflare/http-api";
+import { AlertService, APIRouter, AuthService } from "@better-ccflare/http-api";
 import {
 	LeastUsedStrategy,
 	SessionStrategy,
@@ -666,6 +666,10 @@ export default async function startServer(options?: {
 	container.registerInstance(SERVICE_KEYS.PricingLogger, pricingLogger);
 	setPricingLogger(pricingLogger);
 
+	const alertService = new AlertService(db, config);
+	alertService.start();
+	registerDisposable({ dispose: () => alertService.stop() });
+
 	// Strategy is constructed below after RuntimeConfig is built. The router
 	// accepts a getter so it can read the live (post-hot-reload) instance.
 	let currentStrategy: LoadBalancingStrategy | null = null;
@@ -674,6 +678,7 @@ export default async function startServer(options?: {
 		db,
 		config,
 		dbOps,
+		alertService,
 		runtime: {
 			port,
 			tlsEnabled,

packages/config/src/alerts-config.test.ts

@@ -0,0 +1,203 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Config } from "./index";
+
+const ENV_KEYS = [
+	"ALERT_DAILY_SPEND_USD",
+	"ALERT_TOKENS_PER_HOUR",
+	"ALERT_REQUEST_TOKENS",
+	"ALERT_ANOMALY_ENABLED",
+	"ALERT_ANOMALY_INTERVAL_MINUTES",
+	"ALERT_COOLDOWN_MINUTES",
+	"ALERT_WEBHOOK_URL",
+] as const;
+
+const ORIGINAL_ENV: Record<string, string | undefined> = {};
+for (const key of ENV_KEYS) {
+	ORIGINAL_ENV[key] = process.env[key];
+}
+
+function makeConfig(): { config: Config; cleanup: () => void } {
+	const dir = mkdtempSync(join(tmpdir(), "better-ccflare-config-"));
+	return {
+		config: new Config(join(dir, "config.json")),
+		cleanup: () => rmSync(dir, { recursive: true, force: true }),
+	};
+}
+
+describe("alert config settings", () => {
+	afterEach(() => {
+		for (const key of ENV_KEYS) {
+			const original = ORIGINAL_ENV[key];
+			if (original === undefined) {
+				delete process.env[key];
+			} else {
+				process.env[key] = original;
+			}
+		}
+	});
+
+	it("returns defaults when nothing is configured", () => {
+		for (const key of ENV_KEYS) {
+			delete process.env[key];
+		}
+		const { config, cleanup } = makeConfig();
+
+		try {
+			expect(config.getAlertDailySpendUsd()).toBe(0);
+			expect(config.getAlertTokensPerHour()).toBe(0);
+			expect(config.getAlertRequestTokens()).toBe(0);
+			expect(config.getAlertAnomalyEnabled()).toBe(false);
+			expect(config.getAlertAnomalyIntervalMinutes()).toBe(15);
+			expect(config.getAlertCooldownMinutes()).toBe(60);
+			expect(config.getAlertWebhookUrl()).toBe("");
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("honors environment variable overrides", () => {
+		process.env.ALERT_DAILY_SPEND_USD = "25.5";
+		process.env.ALERT_TOKENS_PER_HOUR = "500000";
+		process.env.ALERT_REQUEST_TOKENS = "200000";
+		process.env.ALERT_ANOMALY_ENABLED = "true";
+		process.env.ALERT_ANOMALY_INTERVAL_MINUTES = "30";
+		process.env.ALERT_COOLDOWN_MINUTES = "120";
+		process.env.ALERT_WEBHOOK_URL = "https://example.com/hook";
+		const { config, cleanup } = makeConfig();
+
+		try {
+			expect(config.getAlertDailySpendUsd()).toBe(25.5);
+			expect(config.getAlertTokensPerHour()).toBe(500000);
+			expect(config.getAlertRequestTokens()).toBe(200000);
+			expect(config.getAlertAnomalyEnabled()).toBe(true);
+			expect(config.getAlertAnomalyIntervalMinutes()).toBe(30);
+			expect(config.getAlertCooldownMinutes()).toBe(120);
+			expect(config.getAlertWebhookUrl()).toBe("https://example.com/hook");
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("treats non-true anomaly env values as disabled", () => {
+		process.env.ALERT_ANOMALY_ENABLED = "disabled";
+		const { config, cleanup } = makeConfig();
+
+		try {
+			expect(config.getAlertAnomalyEnabled()).toBe(false);
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("clamps out-of-range environment values", () => {
+		process.env.ALERT_DAILY_SPEND_USD = "-5";
+		process.env.ALERT_TOKENS_PER_HOUR = "9999999999";
+		process.env.ALERT_REQUEST_TOKENS = "-100";
+		process.env.ALERT_ANOMALY_INTERVAL_MINUTES = "2";
+		process.env.ALERT_COOLDOWN_MINUTES = "0";
+		const { config, cleanup } = makeConfig();
+
+		try {
+			expect(config.getAlertDailySpendUsd()).toBe(0);
+			expect(config.getAlertTokensPerHour()).toBe(1_000_000_000);
+			expect(config.getAlertRequestTokens()).toBe(0);
+			expect(config.getAlertAnomalyIntervalMinutes()).toBe(5);
+			expect(config.getAlertCooldownMinutes()).toBe(1);
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("clamps out-of-range config-file values via setters", () => {
+		for (const key of ENV_KEYS) {
+			delete process.env[key];
+		}
+		const { config, cleanup } = makeConfig();
+
+		try {
+			config.setAlertDailySpendUsd(2_000_000);
+			expect(config.getAlertDailySpendUsd()).toBe(1_000_000);
+
+			config.setAlertAnomalyIntervalMinutes(3);
+			expect(config.getAlertAnomalyIntervalMinutes()).toBe(5);
+
+			config.setAlertAnomalyIntervalMinutes(99999);
+			expect(config.getAlertAnomalyIntervalMinutes()).toBe(1440);
+
+			config.setAlertCooldownMinutes(0);
+			expect(config.getAlertCooldownMinutes()).toBe(1);
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("persists setter values readable by getters", () => {
+		for (const key of ENV_KEYS) {
+			delete process.env[key];
+		}
+		const { config, cleanup } = makeConfig();
+
+		try {
+			config.setAlertDailySpendUsd(50);
+			config.setAlertTokensPerHour(1_000_000);
+			config.setAlertRequestTokens(300_000);
+			config.setAlertAnomalyEnabled(true);
+			config.setAlertCooldownMinutes(45);
+			config.setAlertWebhookUrl("https://hooks.example.com/alert");
+
+			expect(config.getAlertDailySpendUsd()).toBe(50);
+			expect(config.getAlertTokensPerHour()).toBe(1_000_000);
+			expect(config.getAlertRequestTokens()).toBe(300_000);
+			expect(config.getAlertAnomalyEnabled()).toBe(true);
+			expect(config.getAlertCooldownMinutes()).toBe(45);
+			expect(config.getAlertWebhookUrl()).toBe(
+				"https://hooks.example.com/alert",
+			);
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("accepts an empty webhook URL (disabled) and rejects invalid ones", () => {
+		for (const key of ENV_KEYS) {
+			delete process.env[key];
+		}
+		const { config, cleanup } = makeConfig();
+
+		try {
+			expect(() => config.setAlertWebhookUrl("")).not.toThrow();
+			expect(config.getAlertWebhookUrl()).toBe("");
+
+			expect(() => config.setAlertWebhookUrl("not-a-url")).toThrow();
+			expect(() => config.setAlertWebhookUrl("ftp://example.com")).toThrow();
+			expect(() =>
+				config.setAlertWebhookUrl("http://example.com/hook"),
+			).not.toThrow();
+		} finally {
+			cleanup();
+		}
+	});
+
+	it("includes alert settings in getAllSettings()", () => {
+		for (const key of ENV_KEYS) {
+			delete process.env[key];
+		}
+		const { config, cleanup } = makeConfig();
+
+		try {
+			const settings = config.getAllSettings();
+			expect(settings.alert_daily_spend_usd).toBe(0);
+			expect(settings.alert_tokens_per_hour).toBe(0);
+			expect(settings.alert_request_tokens).toBe(0);
+			expect(settings.alert_anomaly_enabled).toBe(false);
+			expect(settings.alert_anomaly_interval_minutes).toBe(15);
+			expect(settings.alert_cooldown_minutes).toBe(60);
+			expect(settings.alert_webhook_url).toBe("");
+		} finally {
+			cleanup();
+		}
+	});
+});

packages/config/src/index.ts

@@ -62,6 +62,13 @@ export interface ConfigData {
 	usage_throttling_five_hour_enabled?: boolean;
 	usage_throttling_weekly_enabled?: boolean;
 	health_detail_enabled?: boolean;
+	alert_daily_spend_usd?: number;
+	alert_tokens_per_hour?: number;
+	alert_request_tokens?: number;
+	alert_anomaly_enabled?: boolean;
+	alert_anomaly_interval_minutes?: number;
+	alert_cooldown_minutes?: number;
+	alert_webhook_url?: string;
 	// Database configuration
 	db_wal_mode?: boolean;
 	db_busy_timeout_ms?: number;
@@ -416,6 +423,128 @@ export class Config extends EventEmitter {
 		return false;
 	}
 
+	getAlertDailySpendUsd(): number {
+		const fromEnv = process.env.ALERT_DAILY_SPEND_USD;
+		if (fromEnv) {
+			const n = Number.parseFloat(fromEnv);
+			if (!Number.isNaN(n)) return this.clamp(n, 0, 1_000_000);
+		}
+		const fromFile = this.data.alert_daily_spend_usd;
+		if (typeof fromFile === "number") return this.clamp(fromFile, 0, 1_000_000);
+		return 0;
+	}
+
+	setAlertDailySpendUsd(value: number): void {
+		this.set("alert_daily_spend_usd", this.clamp(value, 0, 1_000_000));
+	}
+
+	getAlertTokensPerHour(): number {
+		const fromEnv = process.env.ALERT_TOKENS_PER_HOUR;
+		if (fromEnv) {
+			const n = Number.parseInt(fromEnv, 10);
+			if (!Number.isNaN(n)) return this.clamp(n, 0, 1_000_000_000);
+		}
+		const fromFile = this.data.alert_tokens_per_hour;
+		if (typeof fromFile === "number") {
+			return this.clamp(fromFile, 0, 1_000_000_000);
+		}
+		return 0;
+	}
+
+	setAlertTokensPerHour(value: number): void {
+		this.set("alert_tokens_per_hour", this.clamp(value, 0, 1_000_000_000));
+	}
+
+	getAlertRequestTokens(): number {
+		const fromEnv = process.env.ALERT_REQUEST_TOKENS;
+		if (fromEnv) {
+			const n = Number.parseInt(fromEnv, 10);
+			if (!Number.isNaN(n)) return this.clamp(n, 0, 1_000_000_000);
+		}
+		const fromFile = this.data.alert_request_tokens;
+		if (typeof fromFile === "number") {
+			return this.clamp(fromFile, 0, 1_000_000_000);
+		}
+		return 0;
+	}
+
+	setAlertRequestTokens(value: number): void {
+		this.set("alert_request_tokens", this.clamp(value, 0, 1_000_000_000));
+	}
+
+	getAlertAnomalyEnabled(): boolean {
+		const fromEnv = parseEnabledEnvFlag(process.env.ALERT_ANOMALY_ENABLED);
+		if (fromEnv !== undefined) {
+			return fromEnv;
+		}
+		const fromFile = this.data.alert_anomaly_enabled;
+		if (typeof fromFile === "boolean") return fromFile;
+		return false;
+	}
+
+	setAlertAnomalyEnabled(value: boolean): void {
+		this.set("alert_anomaly_enabled", value);
+	}
+
+	getAlertAnomalyIntervalMinutes(): number {
+		const fromEnv = process.env.ALERT_ANOMALY_INTERVAL_MINUTES;
+		if (fromEnv) {
+			const n = Number.parseInt(fromEnv, 10);
+			if (!Number.isNaN(n)) return this.clamp(n, 5, 1440);
+		}
+		const fromFile = this.data.alert_anomaly_interval_minutes;
+		if (typeof fromFile === "number") return this.clamp(fromFile, 5, 1440);
+		return 15;
+	}
+
+	setAlertAnomalyIntervalMinutes(value: number): void {
+		this.set("alert_anomaly_interval_minutes", this.clamp(value, 5, 1440));
+	}
+
+	getAlertCooldownMinutes(): number {
+		const fromEnv = process.env.ALERT_COOLDOWN_MINUTES;
+		if (fromEnv) {
+			const n = Number.parseInt(fromEnv, 10);
+			if (!Number.isNaN(n)) return this.clamp(n, 1, 1440);
+		}
+		const fromFile = this.data.alert_cooldown_minutes;
+		if (typeof fromFile === "number") return this.clamp(fromFile, 1, 1440);
+		return 60;
+	}
+
+	setAlertCooldownMinutes(value: number): void {
+		this.set("alert_cooldown_minutes", this.clamp(value, 1, 1440));
+	}
+
+	getAlertWebhookUrl(): string {
+		const fromEnv = process.env.ALERT_WEBHOOK_URL;
+		if (fromEnv !== undefined) return fromEnv;
+		const fromFile = this.data.alert_webhook_url;
+		if (typeof fromFile === "string") return fromFile;
+		return "";
+	}
+
+	setAlertWebhookUrl(value: string): void {
+		if (value !== "") {
+			let parsed: URL;
+			try {
+				parsed = new URL(value);
+			} catch (_error) {
+				throw new ValidationError(
+					"Invalid alert webhook URL",
+					"alert_webhook_url",
+				);
+			}
+			if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+				throw new ValidationError(
+					"Invalid alert webhook URL",
+					"alert_webhook_url",
+				);
+			}
+		}
+		this.set("alert_webhook_url", value);
+	}
+
 	getAllSettings(): Record<string, string | number | boolean | undefined> {
 		// Include current strategy (which might come from env)
 		return {
@@ -432,6 +561,13 @@ export class Config extends EventEmitter {
 				this.getUsageThrottlingFiveHourEnabled(),
 			usage_throttling_weekly_enabled: this.getUsageThrottlingWeeklyEnabled(),
 			health_detail_enabled: this.getHealthDetailEnabled(),
+			alert_daily_spend_usd: this.getAlertDailySpendUsd(),
+			alert_tokens_per_hour: this.getAlertTokensPerHour(),
+			alert_request_tokens: this.getAlertRequestTokens(),
+			alert_anomaly_enabled: this.getAlertAnomalyEnabled(),
+			alert_anomaly_interval_minutes: this.getAlertAnomalyIntervalMinutes(),
+			alert_cooldown_minutes: this.getAlertCooldownMinutes(),
+			alert_webhook_url: this.getAlertWebhookUrl(),
 		};
 	}