monitoring-alerting/alerts.yml at main · Mohamed-Abdulla/monitoring-alerting · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
groups:
  - name: node_exporter_alerts
    rules:
      # --- Node Exporter Down ---
      - alert: NodeExporterDown
        expr: up{job="node-exporter"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Node Exporter down"
          description: "Node Exporter has been down for more than 1 minute."

      # --- Disk Usage Alerts ---
      - alert: DiskSpaceUsageWarning
        expr:
          (node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} - node_filesystem_free_bytes{fstype!~"tmpfs|overlay"})
          / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} > 0.75
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Disk space warning on {{ $labels.instance }}"
          description: "Disk usage is above 75% on {{ $labels.mountpoint }} ({{ $labels.device }})"

      - alert: DiskSpaceUsageCritical
        expr:
          (node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} - node_filesystem_free_bytes{fstype!~"tmpfs|overlay"})
          / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} > 0.85
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Disk space critical on {{ $labels.instance }}"
          description: "Disk usage is above 85% on {{ $labels.mountpoint }} ({{ $labels.device }})"

      # --- CPU Usage Alerts ---
      - alert: HighCPUUsage
        expr: (1 - avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.85
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage has been above 85% for 3 minutes."

      # --- Memory Usage Alerts ---
      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.85
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: "High Memory usage on {{ $labels.instance }}"
          description: "Memory usage is above 85% on {{ $labels.instance }}."

      # --- Instance Uptime Alert ---
      - alert: NodeRebooted
        expr: (time() - node_boot_time_seconds) < 300
        for: 0m
        labels:
          severity: info
        annotations:
          summary: "Node {{ $labels.instance }} rebooted"
          description: "Node has restarted within the last 5 minutes."

      # --- Load Average Alert ---
      - alert: HighSystemLoad
        expr: node_load1 / count by(instance) (node_cpu_seconds_total{mode="system"}) > 1.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High system load on {{ $labels.instance }}"
          description: "System load average (1m) is high compared to CPU count."

      # --- Filesystem Read-only Alert ---
      - alert: FilesystemReadOnly
        expr: node_filesystem_readonly == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Filesystem in read-only mode on {{ $labels.instance }}"
          description: "Filesystem {{ $labels.mountpoint }} is in read-only mode!"