|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "bytes" |
| 5 | + "context" |
| 6 | + "crypto/tls" |
| 7 | + "fmt" |
| 8 | + "log" |
| 9 | + "net/smtp" |
| 10 | + "strings" |
| 11 | + "sync" |
| 12 | + "time" |
| 13 | +) |
| 14 | + |
| 15 | +// AlertConfig holds SMTP alert configuration |
| 16 | +type AlertConfig struct { |
| 17 | + Enabled bool |
| 18 | + SMTPHost string |
| 19 | + SMTPPort int |
| 20 | + SMTPUser string |
| 21 | + SMTPPassword string |
| 22 | + SMTPFrom string |
| 23 | + SMTPTo []string |
| 24 | + UseTLS bool |
| 25 | + FailureThreshold float64 // Alert when failure rate exceeds this (e.g., 20.0 = 20%) |
| 26 | + CheckInterval time.Duration // How often to check |
| 27 | + Cooldown time.Duration // Minimum time between alerts |
| 28 | +} |
| 29 | + |
| 30 | +// Alerter handles alerting functionality |
| 31 | +type Alerter struct { |
| 32 | + cfg AlertConfig |
| 33 | + lastAlertAt time.Time |
| 34 | + mu sync.Mutex |
| 35 | + pb *PBClient |
| 36 | + lastStats alertStats |
| 37 | + alertHistory []AlertEvent |
| 38 | +} |
| 39 | + |
| 40 | +type alertStats struct { |
| 41 | + successCount int |
| 42 | + failedCount int |
| 43 | + checkedAt time.Time |
| 44 | +} |
| 45 | + |
| 46 | +// AlertEvent records an alert that was sent |
| 47 | +type AlertEvent struct { |
| 48 | + Timestamp time.Time `json:"timestamp"` |
| 49 | + Type string `json:"type"` |
| 50 | + Message string `json:"message"` |
| 51 | + FailureRate float64 `json:"failure_rate,omitempty"` |
| 52 | +} |
| 53 | + |
| 54 | +// NewAlerter creates a new alerter instance |
| 55 | +func NewAlerter(cfg AlertConfig, pb *PBClient) *Alerter { |
| 56 | + return &Alerter{ |
| 57 | + cfg: cfg, |
| 58 | + pb: pb, |
| 59 | + alertHistory: make([]AlertEvent, 0), |
| 60 | + } |
| 61 | +} |
| 62 | + |
| 63 | +// Start begins the alert monitoring loop |
| 64 | +func (a *Alerter) Start() { |
| 65 | + if !a.cfg.Enabled { |
| 66 | + log.Println("INFO: alerting disabled") |
| 67 | + return |
| 68 | + } |
| 69 | + |
| 70 | + if a.cfg.SMTPHost == "" || len(a.cfg.SMTPTo) == 0 { |
| 71 | + log.Println("WARN: alerting enabled but SMTP not configured") |
| 72 | + return |
| 73 | + } |
| 74 | + |
| 75 | + go a.monitorLoop() |
| 76 | + log.Printf("INFO: alert monitoring started (threshold: %.1f%%, interval: %v)", a.cfg.FailureThreshold, a.cfg.CheckInterval) |
| 77 | +} |
| 78 | + |
| 79 | +func (a *Alerter) monitorLoop() { |
| 80 | + ticker := time.NewTicker(a.cfg.CheckInterval) |
| 81 | + defer ticker.Stop() |
| 82 | + |
| 83 | + for range ticker.C { |
| 84 | + a.checkAndAlert() |
| 85 | + } |
| 86 | +} |
| 87 | + |
| 88 | +func (a *Alerter) checkAndAlert() { |
| 89 | + ctx, cancel := newTimeoutContext(10 * time.Second) |
| 90 | + defer cancel() |
| 91 | + |
| 92 | + // Fetch last hour's data |
| 93 | + data, err := a.pb.FetchDashboardData(ctx, 1) |
| 94 | + if err != nil { |
| 95 | + log.Printf("WARN: alert check failed: %v", err) |
| 96 | + return |
| 97 | + } |
| 98 | + |
| 99 | + // Calculate current failure rate |
| 100 | + total := data.SuccessCount + data.FailedCount |
| 101 | + if total < 10 { |
| 102 | + // Not enough data to determine rate |
| 103 | + return |
| 104 | + } |
| 105 | + |
| 106 | + failureRate := float64(data.FailedCount) / float64(total) * 100 |
| 107 | + |
| 108 | + // Check if we should alert |
| 109 | + if failureRate >= a.cfg.FailureThreshold { |
| 110 | + a.maybeSendAlert(failureRate, data.FailedCount, total) |
| 111 | + } |
| 112 | +} |
| 113 | + |
| 114 | +func (a *Alerter) maybeSendAlert(rate float64, failed, total int) { |
| 115 | + a.mu.Lock() |
| 116 | + defer a.mu.Unlock() |
| 117 | + |
| 118 | + // Check cooldown |
| 119 | + if time.Since(a.lastAlertAt) < a.cfg.Cooldown { |
| 120 | + return |
| 121 | + } |
| 122 | + |
| 123 | + // Send alert |
| 124 | + subject := fmt.Sprintf("[ProxmoxVED Alert] High Failure Rate: %.1f%%", rate) |
| 125 | + body := fmt.Sprintf(`ProxmoxVE Helper Scripts - Telemetry Alert |
| 126 | +
|
| 127 | +⚠️ High installation failure rate detected! |
| 128 | +
|
| 129 | +Current Statistics (last 24h): |
| 130 | +- Failure Rate: %.1f%% |
| 131 | +- Failed Installations: %d |
| 132 | +- Total Installations: %d |
| 133 | +- Threshold: %.1f%% |
| 134 | +
|
| 135 | +Time: %s |
| 136 | +
|
| 137 | +Please check the dashboard for more details. |
| 138 | +
|
| 139 | +--- |
| 140 | +This is an automated alert from the telemetry service. |
| 141 | +`, rate, failed, total, a.cfg.FailureThreshold, time.Now().Format(time.RFC1123)) |
| 142 | + |
| 143 | + if err := a.sendEmail(subject, body); err != nil { |
| 144 | + log.Printf("ERROR: failed to send alert email: %v", err) |
| 145 | + return |
| 146 | + } |
| 147 | + |
| 148 | + a.lastAlertAt = time.Now() |
| 149 | + a.alertHistory = append(a.alertHistory, AlertEvent{ |
| 150 | + Timestamp: time.Now(), |
| 151 | + Type: "high_failure_rate", |
| 152 | + Message: fmt.Sprintf("Failure rate %.1f%% exceeded threshold %.1f%%", rate, a.cfg.FailureThreshold), |
| 153 | + FailureRate: rate, |
| 154 | + }) |
| 155 | + |
| 156 | + // Keep only last 100 alerts |
| 157 | + if len(a.alertHistory) > 100 { |
| 158 | + a.alertHistory = a.alertHistory[len(a.alertHistory)-100:] |
| 159 | + } |
| 160 | + |
| 161 | + log.Printf("ALERT: sent high failure rate alert (%.1f%%)", rate) |
| 162 | +} |
| 163 | + |
| 164 | +func (a *Alerter) sendEmail(subject, body string) error { |
| 165 | + // Build message |
| 166 | + var msg bytes.Buffer |
| 167 | + msg.WriteString(fmt.Sprintf("From: %s\r\n", a.cfg.SMTPFrom)) |
| 168 | + msg.WriteString(fmt.Sprintf("To: %s\r\n", strings.Join(a.cfg.SMTPTo, ", "))) |
| 169 | + msg.WriteString(fmt.Sprintf("Subject: %s\r\n", subject)) |
| 170 | + msg.WriteString("MIME-Version: 1.0\r\n") |
| 171 | + msg.WriteString("Content-Type: text/plain; charset=UTF-8\r\n") |
| 172 | + msg.WriteString("\r\n") |
| 173 | + msg.WriteString(body) |
| 174 | + |
| 175 | + addr := fmt.Sprintf("%s:%d", a.cfg.SMTPHost, a.cfg.SMTPPort) |
| 176 | + |
| 177 | + var auth smtp.Auth |
| 178 | + if a.cfg.SMTPUser != "" && a.cfg.SMTPPassword != "" { |
| 179 | + auth = smtp.PlainAuth("", a.cfg.SMTPUser, a.cfg.SMTPPassword, a.cfg.SMTPHost) |
| 180 | + } |
| 181 | + |
| 182 | + if a.cfg.UseTLS { |
| 183 | + // TLS connection |
| 184 | + tlsConfig := &tls.Config{ |
| 185 | + ServerName: a.cfg.SMTPHost, |
| 186 | + } |
| 187 | + |
| 188 | + conn, err := tls.Dial("tcp", addr, tlsConfig) |
| 189 | + if err != nil { |
| 190 | + return fmt.Errorf("TLS dial failed: %w", err) |
| 191 | + } |
| 192 | + defer conn.Close() |
| 193 | + |
| 194 | + client, err := smtp.NewClient(conn, a.cfg.SMTPHost) |
| 195 | + if err != nil { |
| 196 | + return fmt.Errorf("SMTP client failed: %w", err) |
| 197 | + } |
| 198 | + defer client.Close() |
| 199 | + |
| 200 | + if auth != nil { |
| 201 | + if err := client.Auth(auth); err != nil { |
| 202 | + return fmt.Errorf("SMTP auth failed: %w", err) |
| 203 | + } |
| 204 | + } |
| 205 | + |
| 206 | + if err := client.Mail(a.cfg.SMTPFrom); err != nil { |
| 207 | + return fmt.Errorf("SMTP MAIL failed: %w", err) |
| 208 | + } |
| 209 | + |
| 210 | + for _, to := range a.cfg.SMTPTo { |
| 211 | + if err := client.Rcpt(to); err != nil { |
| 212 | + return fmt.Errorf("SMTP RCPT failed: %w", err) |
| 213 | + } |
| 214 | + } |
| 215 | + |
| 216 | + w, err := client.Data() |
| 217 | + if err != nil { |
| 218 | + return fmt.Errorf("SMTP DATA failed: %w", err) |
| 219 | + } |
| 220 | + |
| 221 | + _, err = w.Write(msg.Bytes()) |
| 222 | + if err != nil { |
| 223 | + return fmt.Errorf("SMTP write failed: %w", err) |
| 224 | + } |
| 225 | + |
| 226 | + return w.Close() |
| 227 | + } |
| 228 | + |
| 229 | + // Non-TLS (STARTTLS) |
| 230 | + return smtp.SendMail(addr, auth, a.cfg.SMTPFrom, a.cfg.SMTPTo, msg.Bytes()) |
| 231 | +} |
| 232 | + |
| 233 | +// GetAlertHistory returns recent alert events |
| 234 | +func (a *Alerter) GetAlertHistory() []AlertEvent { |
| 235 | + a.mu.Lock() |
| 236 | + defer a.mu.Unlock() |
| 237 | + result := make([]AlertEvent, len(a.alertHistory)) |
| 238 | + copy(result, a.alertHistory) |
| 239 | + return result |
| 240 | +} |
| 241 | + |
| 242 | +// TestAlert sends a test alert email |
| 243 | +func (a *Alerter) TestAlert() error { |
| 244 | + if !a.cfg.Enabled || a.cfg.SMTPHost == "" { |
| 245 | + return fmt.Errorf("alerting not configured") |
| 246 | + } |
| 247 | + |
| 248 | + subject := "[ProxmoxVED] Test Alert" |
| 249 | + body := fmt.Sprintf(`This is a test alert from ProxmoxVE Helper Scripts telemetry service. |
| 250 | +
|
| 251 | +If you received this email, your alert configuration is working correctly. |
| 252 | +
|
| 253 | +Time: %s |
| 254 | +SMTP Host: %s |
| 255 | +Recipients: %s |
| 256 | +
|
| 257 | +--- |
| 258 | +This is an automated test message. |
| 259 | +`, time.Now().Format(time.RFC1123), a.cfg.SMTPHost, strings.Join(a.cfg.SMTPTo, ", ")) |
| 260 | + |
| 261 | + return a.sendEmail(subject, body) |
| 262 | +} |
| 263 | + |
| 264 | +// Helper for timeout context |
| 265 | +func newTimeoutContext(d time.Duration) (context.Context, context.CancelFunc) { |
| 266 | + return context.WithTimeout(context.Background(), d) |
| 267 | +} |
0 commit comments