-
-
Notifications
You must be signed in to change notification settings - Fork 41
plex: add webhook healthcheck against active sessions #1196
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| package plexcron | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/Notifiarr/notifiarr/pkg/apps/apppkg/plex" | ||
| "github.com/Notifiarr/notifiarr/pkg/mnd" | ||
| "github.com/Notifiarr/notifiarr/pkg/website" | ||
| "github.com/Notifiarr/notifiarr/pkg/website/clientinfo" | ||
| ) | ||
|
|
||
| const ( | ||
| defaultWebhookStaleAfter = 20 * time.Minute | ||
| defaultWebhookAlertCD = 6 * time.Hour | ||
| defaultWebhookStartupGrace = 10 * time.Minute | ||
| ) | ||
|
|
||
| type webhookHealthConfig struct { | ||
| Enabled bool | ||
| StaleAfter time.Duration | ||
| AlertCooldown time.Duration | ||
| StartupGrace time.Duration | ||
| } | ||
|
|
||
| type webhookHealthState struct { | ||
| StartAt time.Time | ||
| LastWebhookAt time.Time | ||
| LastAlertAt time.Time | ||
| } | ||
|
|
||
| func defaultWebhookHealthConfig() webhookHealthConfig { | ||
| return webhookHealthConfig{ | ||
| Enabled: true, | ||
| StaleAfter: defaultWebhookStaleAfter, | ||
| AlertCooldown: defaultWebhookAlertCD, | ||
| StartupGrace: defaultWebhookStartupGrace, | ||
| } | ||
| } | ||
|
|
||
| func (c *cmd) currentWebhookHealthConfig() webhookHealthConfig { | ||
| cfg := defaultWebhookHealthConfig() | ||
| ci := clientinfo.Get() | ||
| if ci == nil { | ||
| return cfg | ||
| } | ||
|
|
||
| siteCfg := ci.Actions.Plex | ||
| if siteCfg.WebhookHealthEnabled != nil { | ||
| cfg.Enabled = *siteCfg.WebhookHealthEnabled | ||
| } | ||
|
|
||
| if siteCfg.WebhookStaleAfter.Duration > 0 { | ||
| cfg.StaleAfter = siteCfg.WebhookStaleAfter.Duration | ||
| } | ||
|
|
||
| if siteCfg.WebhookAlertCooldown.Duration > 0 { | ||
| cfg.AlertCooldown = siteCfg.WebhookAlertCooldown.Duration | ||
| } | ||
|
|
||
| if siteCfg.WebhookStartupGrace.Duration > 0 { | ||
| cfg.StartupGrace = siteCfg.WebhookStartupGrace.Duration | ||
| } | ||
|
|
||
| return cfg | ||
| } | ||
|
|
||
| func (c *cmd) currentWebhookHealthState() webhookHealthState { | ||
| c.healthLock.Lock() | ||
| defer c.healthLock.Unlock() | ||
|
|
||
| return webhookHealthState{ | ||
| StartAt: c.startAt, | ||
| LastWebhookAt: c.lastWebhookAt, | ||
| LastAlertAt: c.lastAlertAt, | ||
| } | ||
| } | ||
|
|
||
| func (c *cmd) recordWebhookAt(t time.Time) { | ||
| c.healthLock.Lock() | ||
| defer c.healthLock.Unlock() | ||
|
|
||
| c.lastWebhookAt = t | ||
| } | ||
|
|
||
| func (c *cmd) setWebhookAlertAt(t time.Time) { | ||
| c.healthLock.Lock() | ||
| defer c.healthLock.Unlock() | ||
|
|
||
| c.lastAlertAt = t | ||
| } | ||
|
|
||
| func shouldAlertForStaleWebhook( | ||
| now time.Time, | ||
| activeSessions int, | ||
| cfg webhookHealthConfig, | ||
| state webhookHealthState, | ||
| ) bool { | ||
| switch { | ||
| case !cfg.Enabled: | ||
| return false | ||
| case activeSessions < 1: | ||
| return false | ||
| case cfg.StartupGrace > 0 && now.Sub(state.StartAt) < cfg.StartupGrace: | ||
| return false | ||
| case !state.LastAlertAt.IsZero() && cfg.AlertCooldown > 0 && now.Sub(state.LastAlertAt) < cfg.AlertCooldown: | ||
| return false | ||
| case state.LastWebhookAt.IsZero(): | ||
| return true | ||
| default: | ||
| return now.Sub(state.LastWebhookAt) > cfg.StaleAfter | ||
| } | ||
| } | ||
|
|
||
| func (c *cmd) evaluateWebhookHealth(ctx context.Context, sessions *plex.Sessions) { | ||
| if sessions == nil { | ||
| return | ||
| } | ||
|
|
||
| now := time.Now() | ||
| state := c.currentWebhookHealthState() | ||
| cfg := c.currentWebhookHealthConfig() | ||
| if !shouldAlertForStaleWebhook(now, len(sessions.Sessions), cfg, state) { | ||
| return | ||
| } | ||
|
|
||
| c.setWebhookAlertAt(now) | ||
|
|
||
| hook := &plex.IncomingWebhook{ReqID: mnd.GetID(ctx), Event: "notifiarr.webhook.health"} | ||
| hook.Server.Title = c.Plex.Name() | ||
| hook.Metadata.Type = "healthcheck" | ||
| hook.Metadata.Title = "Plex webhook appears inactive" | ||
| hook.Metadata.Summary = fmt.Sprintf( | ||
| "Detected %d active Plex sessions but no accepted webhook for %s.", | ||
| len(sessions.Sessions), now.Sub(state.LastWebhookAt).Round(time.Second)) | ||
| if state.LastWebhookAt.IsZero() { | ||
| hook.Metadata.Summary = fmt.Sprintf( | ||
| "Detected %d active Plex sessions but no accepted webhook since startup.", | ||
| len(sessions.Sessions)) | ||
| } | ||
|
|
||
| website.SendData(&website.Request{ | ||
| ReqID: mnd.GetID(ctx), | ||
| Route: website.PlexRoute, | ||
| Event: website.EventHook, | ||
| Payload: &website.Payload{ | ||
| Snap: c.getMetaSnap(ctx), | ||
| Plex: sessions, | ||
| Load: hook, | ||
| }, | ||
| LogMsg: fmt.Sprintf( | ||
| "Plex Webhook Health Alert: active sessions=%d, lastWebhook=%s", | ||
| len(sessions.Sessions), state.LastWebhookAt.Format(time.RFC3339)), | ||
| LogPayload: true, | ||
| }) | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,102 @@ | ||||||
| package plexcron | ||||||
|
Check failure on line 1 in pkg/triggers/plexcron/webhook_health_test.go
|
||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
See if you can make the tests work like this. It's not mandatory, but if you don't then you need to make the linter shut up. |
||||||
|
|
||||||
| import ( | ||||||
| "testing" | ||||||
| "time" | ||||||
| ) | ||||||
|
|
||||||
| func TestShouldAlertForStaleWebhook(t *testing.T) { | ||||||
|
Check failure on line 8 in pkg/triggers/plexcron/webhook_health_test.go
|
||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can add an override for test files in .golangci-lint.yml to ignore the funlen linter. |
||||||
| now := time.Date(2026, 2, 16, 12, 0, 0, 0, time.UTC) | ||||||
|
|
||||||
| cfg := webhookHealthConfig{ | ||||||
| Enabled: true, | ||||||
| StaleAfter: 20 * time.Minute, | ||||||
| StartupGrace: 10 * time.Minute, | ||||||
| AlertCooldown: 6 * time.Hour, | ||||||
| } | ||||||
|
|
||||||
| tests := []struct { | ||||||
| name string | ||||||
| activeSessions int | ||||||
| state webhookHealthState | ||||||
| want bool | ||||||
| }{ | ||||||
| { | ||||||
| name: "alerts when sessions active and webhook stale", | ||||||
| activeSessions: 2, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-2 * time.Hour), | ||||||
| LastWebhookAt: now.Add(-25 * time.Minute), | ||||||
| }, | ||||||
| want: true, | ||||||
| }, | ||||||
| { | ||||||
| name: "does not alert when no sessions", | ||||||
| activeSessions: 0, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-2 * time.Hour), | ||||||
| LastWebhookAt: now.Add(-25 * time.Minute), | ||||||
| }, | ||||||
| want: false, | ||||||
| }, | ||||||
| { | ||||||
| name: "does not alert when webhook is recent", | ||||||
| activeSessions: 1, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-2 * time.Hour), | ||||||
| LastWebhookAt: now.Add(-5 * time.Minute), | ||||||
| }, | ||||||
| want: false, | ||||||
| }, | ||||||
| { | ||||||
| name: "does not alert during startup grace", | ||||||
| activeSessions: 3, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-5 * time.Minute), | ||||||
| LastWebhookAt: time.Time{}, | ||||||
| }, | ||||||
| want: false, | ||||||
| }, | ||||||
| { | ||||||
| name: "does not alert during cooldown", | ||||||
| activeSessions: 3, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-2 * time.Hour), | ||||||
| LastWebhookAt: now.Add(-25 * time.Minute), | ||||||
| LastAlertAt: now.Add(-2 * time.Hour), | ||||||
| }, | ||||||
| want: false, | ||||||
| }, | ||||||
| { | ||||||
| name: "alerts if no webhook was ever seen and grace has passed", | ||||||
| activeSessions: 1, | ||||||
| state: webhookHealthState{ | ||||||
| StartAt: now.Add(-2 * time.Hour), | ||||||
| LastWebhookAt: time.Time{}, | ||||||
| }, | ||||||
| want: true, | ||||||
| }, | ||||||
| } | ||||||
|
|
||||||
| for _, tc := range tests { | ||||||
|
Check failure on line 81 in pkg/triggers/plexcron/webhook_health_test.go
|
||||||
| t.Run(tc.name, func(t *testing.T) { | ||||||
| got := shouldAlertForStaleWebhook(now, tc.activeSessions, cfg, tc.state) | ||||||
| if got != tc.want { | ||||||
| t.Fatalf("shouldAlertForStaleWebhook() = %v, want %v", got, tc.want) | ||||||
| } | ||||||
| }) | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| func TestDefaultWebhookHealthConfig(t *testing.T) { | ||||||
|
Check failure on line 91 in pkg/triggers/plexcron/webhook_health_test.go
|
||||||
| cfg := defaultWebhookHealthConfig() | ||||||
|
|
||||||
| if !cfg.Enabled { | ||||||
| t.Fatal("expected default webhook health config to be enabled") | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Tell your AI to use testify. |
||||||
| } | ||||||
|
|
||||||
| if cfg.StaleAfter <= 0 || cfg.StartupGrace <= 0 || cfg.AlertCooldown <= 0 { | ||||||
| t.Fatalf("expected positive durations, got stale=%s grace=%s cooldown=%s", | ||||||
| cfg.StaleAfter, cfg.StartupGrace, cfg.AlertCooldown) | ||||||
| } | ||||||
| } | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This payload structure you created int's going to work out. I'll figure out what we need...