fix(slack): dedupe user overage alerts daily

Author: Harbor

backend/README.md

@@ -185,7 +185,7 @@ Common optional settings:
 - provider keys such as `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_VERSION`, `ODDISH_AZURE_OPENAI_DEPLOYMENTS`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `DAYTONA_API_KEY`
 - `ODDISH_OPENAI_PROVIDER=openai` plus `OPENAI_API_KEY` only when intentionally routing OpenAI-family jobs to public OpenAI
 - GitHub notifier settings such as `GITHUB_TOKEN` and `ODDISH_DASHBOARD_URL`
-- `SLACK_ALERT_BOT_TOKEN` (scopes `chat:write`, `im:write`, `users:read.email`) for deterministic cost alerts, which DM an experiment's owner: a milestone for each $1,000 spent in the past 24 hours, and any trial over $200 that finished in that window. The same token delivers the other DM-only alerts -- trial failed, QA failed, experiment failed -- and resolves in-channel mentions by account email. The email delivery channel has been removed entirely. `SLACK_EXPENSE_WEBHOOK_URL` carries what is left in-channel: unpriceable-model alerts from the past 24 hours; an escalation when a running or retrying trial's live cost rises above the configured floor, which `<@...>`-mentions its owner plus the always-ping list; and a `<!channel>` alert when a user's rolling seven-day spend, including live running-trial checkpoints, rises more than the configured dollar delta above their workspace's average spender. The two channel escalation thresholds and the ping list are set by an admin on the Costs tab of `/admin` and stored in `slack_alert_settings`; the constants in `slack_alert_settings.py` are the defaults they override, and no setting here is environment-configurable. Weekly user alerts re-arm after spend drops back below the threshold. The per-user DM cutoffs (the milestone and completed-trial floor) are separate deploy-time constants in `user_alert_prefs.py` that each person tunes in their own notification settings, not admin-editable here. Notifications are on by default for the production app; previews opt in with `ODDISH_ENABLE_SLACK_EXPENSE_NOTIFICATIONS=true` and can attach a preview-only notification secret via `ODDISH_SLACK_EXPENSE_SECRET_NAME` / `ODDISH_SLACK_EXPENSE_SECRET_ENVIRONMENT`.
+- `SLACK_ALERT_BOT_TOKEN` (scopes `chat:write`, `im:write`, `users:read.email`) for deterministic cost alerts, which DM an experiment's owner: a milestone for each $1,000 spent in the past 24 hours, and any trial over $200 that finished in that window. The same token delivers the other DM-only alerts -- trial failed, QA failed, experiment failed -- and resolves in-channel mentions by account email. The email delivery channel has been removed entirely. `SLACK_EXPENSE_WEBHOOK_URL` carries what is left in-channel: unpriceable-model alerts from the past 24 hours; an escalation when a running or retrying trial's live cost rises above the configured floor, which `<@...>`-mentions its owner plus the always-ping list; and a `<!channel>` alert when a user's rolling seven-day spend, including live running-trial checkpoints, rises more than the configured dollar delta above their workspace's average spender. The two channel escalation thresholds and the ping list are set by an admin on the Costs tab of `/admin` and stored in `slack_alert_settings`; the constants in `slack_alert_settings.py` are the defaults they override, and no setting here is environment-configurable. User daily-overage alerts fire at most once per UTC calendar day. The per-user DM cutoffs (the milestone and completed-trial floor) are separate deploy-time constants in `user_alert_prefs.py` that each person tunes in their own notification settings, not admin-editable here. Notifications are on by default for the production app; previews opt in with `ODDISH_ENABLE_SLACK_EXPENSE_NOTIFICATIONS=true` and can attach a preview-only notification secret via `ODDISH_SLACK_EXPENSE_SECRET_NAME` / `ODDISH_SLACK_EXPENSE_SECRET_ENVIRONMENT`.
 - `ODDISH_SLACK_UNFURL_*` for a lean, single-workspace Slack app that unfurls Oddish task, experiment, and public-share links. It requires `links:read` and `links:write`, a `link_shared` event subscription pointed at `/webhooks/slack/events`, a signing secret, bot token, and bound Oddish org. Optional team/channel allowlists add defense in depth. This is separate from the expense notifications above.
 - `ODDISH_CARL_*`, `ODDISH_API_KEY`, and `ODDISH_DATABASE_URL_RO` extend that same Slack app with read-only answers to permitted `app_mention` events. Carl keeps the existing `/webhooks/slack/events` URL and `link_shared` subscription; add `app_mentions:read` and subscribe the installed app to `app_mention`. The SQL DSN must use a dedicated non-superuser role restricted to the analytics table allow-list. See `slackbot/README.md`.
 

backend/slack_notifications.py

@@ -256,6 +256,7 @@ def build_alerts(
     user_prefs: Mapping[str, UserAlertPrefs] | None = None,
 ) -> list[SlackAlert]:
     prefs_by_email = user_prefs or {}
+    alert_date = (recent_cutoff + timedelta(days=1)).astimezone(timezone.utc).date()
 
     def prefs_for(email: str | None) -> UserAlertPrefs:
         return prefs_by_email.get(
@@ -417,7 +418,7 @@ def add_failure_dm(
             continue
         alerts.append(
             SlackAlert(
-                key=f"user-daily-overage:{user.org_id}:{user.user_id}",
+                key=f"user-daily-overage:{user.org_id}:{user.user_id}:{alert_date}",
                 text=(
                     "<!channel>\n"
                     ":moneybag: *User spend above their 7-day daily average*\n"

backend/tests/test_slack_notifications.py

@@ -495,6 +495,7 @@ def test_build_alerts_does_not_channel_escalate_a_finished_trial() -> None:
 
 
 def test_build_alerts_pings_channel_for_user_above_daily_overage_margin() -> None:
+    now = datetime(2026, 8, 18, 12, tzinfo=timezone.utc)
     alerts = build_alerts(
         AlertCandidates(
             user_spend=[
@@ -509,11 +510,13 @@ def test_build_alerts_pings_channel_for_user_above_daily_overage_margin() -> Non
             ]
         ),
         settings=DEFAULT_ALERT_SETTINGS,
-        recent_cutoff=datetime.now(timezone.utc) - timedelta(hours=24),
+        recent_cutoff=now - timedelta(hours=24),
         dashboard_url="https://www.oddish.app",
     )
 
-    assert [alert.key for alert in alerts] == ["user-daily-overage:org-1:user-1"]
+    assert [alert.key for alert in alerts] == [
+        "user-daily-overage:org-1:user-1:2026-08-18"
+    ]
     assert alerts[0].text.splitlines() == [
         "<!channel>",
         ":moneybag: *User spend above their 7-day daily average*",
@@ -530,6 +533,7 @@ def test_build_alerts_pings_channel_for_user_above_daily_overage_margin() -> Non
 def test_build_alerts_user_daily_overage_is_exclusive_and_configurable() -> None:
     # 24h spend $6,000 runs $5,000 above the $1,000 daily average.
     candidate = UserSpend("org-1", "user-1", "Pat", 6_000, 1_000, 0)
+    now = datetime(2026, 8, 18, 12, tzinfo=timezone.utc)
 
     def keys(delta: float) -> list[str]:
         alerts = build_alerts(
@@ -538,13 +542,51 @@ def keys(delta: float) -> list[str]:
                 DEFAULT_ALERT_SETTINGS,
                 user_daily_overage_delta_usd=delta,
             ),
-            recent_cutoff=datetime.now(timezone.utc) - timedelta(hours=24),
+            recent_cutoff=now - timedelta(hours=24),
             dashboard_url="https://www.oddish.app",
         )
         return [alert.key for alert in alerts]
 
     assert keys(5_000) == []
-    assert keys(4_999) == ["user-daily-overage:org-1:user-1"]
+    assert keys(4_999) == ["user-daily-overage:org-1:user-1:2026-08-18"]
+
+
+@pytest.mark.asyncio
+async def test_user_daily_overage_alerts_once_per_utc_day(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    rows: dict[str, dict] = {}
+    posted: list[str] = []
+    candidates = AlertCandidates(
+        user_spend=[UserSpend("org-1", "user-1", "Pat", 6_000, 1_000, 0)]
+    )
+
+    async def post(*_args) -> None:
+        posted.append("")
+
+    def alerts(now: datetime) -> list[SlackAlert]:
+        return build_alerts(
+            candidates,
+            settings=DEFAULT_ALERT_SETTINGS,
+            recent_cutoff=now - timedelta(hours=24),
+            dashboard_url="https://www.oddish.app",
+        )
+
+    _outbox_stubs(monkeypatch, rows)
+    monkeypatch.setattr(notifications, "_post", post)
+
+    for now in (
+        datetime(2026, 8, 18, 1, tzinfo=timezone.utc),
+        datetime(2026, 8, 18, 23, tzinfo=timezone.utc),
+        datetime(2026, 8, 19, 1, tzinfo=timezone.utc),
+    ):
+        await _record_and_deliver(alerts(now), webhook_url="https://hooks.slack.test")
+
+    assert len(posted) == 2
+    assert _sent_keys(rows) == {
+        "user-daily-overage:org-1:user-1:2026-08-18",
+        "user-daily-overage:org-1:user-1:2026-08-19",
+    }
 
 
 def test_build_alerts_reports_unpriceable_models_once_each() -> None: