konnosDashboard
host.yml122 lines · main
1# Host + container resource alerts (node-exporter + cAdvisor).
2# Routed via alertmanager → discord bridge: severity=critical|warning →
3# #briven-alerts, info → #briven-deploys. Severity labels MUST match the
4# route matchers in infra/observability/alertmanager/alertmanager.yml.
5#
6# Thresholds tuned for kvm4: 4 vCPU, 16 GB RAM, 4 GB swap, on an
7# oversubscribed Hostinger VPS (CPU steal is a real signal here).
8#
9# NOTE: Dokploy builds can run as a HOST process (no container cgroup), so
10# a runaway build is caught by the host memory/swap alerts below, not by
11# the per-container cAdvisor alert. Both layers are intentional.
12
13groups:
14 - name: briven-host
15 interval: 30s
16 rules:
17 - alert: HostMemoryCritical
18 expr: |
19 node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 5
20 for: 5m
21 labels:
22 severity: critical
23 annotations:
24 summary: 'host {{ $labels.instance }} memory critically low'
25 description: 'Available memory <5% for 5m ({{ $value | printf "%.1f" }}%). Builds/exec will start failing with OCI "unable to start container process".'
26
27 - alert: HostMemoryHigh
28 expr: |
29 node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
30 for: 10m
31 labels:
32 severity: warning
33 annotations:
34 summary: 'host {{ $labels.instance }} memory low'
35 description: 'Available memory <10% for 10m ({{ $value | printf "%.1f" }}%).'
36
37 - alert: HostSwapHigh
38 expr: |
39 (1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 50
40 for: 10m
41 labels:
42 severity: warning
43 annotations:
44 summary: 'host {{ $labels.instance }} swapping heavily'
45 description: 'Swap usage >50% for 10m ({{ $value | printf "%.0f" }}%). Sustained swap means a process (often a build) is exhausting RAM.'
46
47 - alert: HostDiskHigh
48 # rootfs only. With --path.rootfs=/host, node-exporter reports the
49 # host root as mountpoint "/". If your build reports "/host", widen
50 # the matcher to mountpoint=~"/|/host".
51 expr: |
52 (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
53 / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 85
54 for: 10m
55 labels:
56 severity: warning
57 annotations:
58 summary: 'host {{ $labels.instance }} disk >85%'
59 description: 'Root filesystem {{ $value | printf "%.0f" }}% full. Run docker builder/image prune.'
60
61 - alert: HostDiskCritical
62 expr: |
63 (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
64 / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 92
65 for: 5m
66 labels:
67 severity: critical
68 annotations:
69 summary: 'host {{ $labels.instance }} disk >92%'
70 description: 'Root filesystem {{ $value | printf "%.0f" }}% full — deploys and Docker will fail imminently.'
71
72 - alert: HostLoadHigh
73 # 5-min load above 2x core count. count(idle cpu series) = vCPU count.
74 expr: |
75 node_load5 > 2 * count by (instance) (node_cpu_seconds_total{mode="idle"})
76 for: 10m
77 labels:
78 severity: warning
79 annotations:
80 summary: 'host {{ $labels.instance }} load high'
81 description: '5-min load {{ $value | printf "%.1f" }} is >2x vCPU count for 10m.'
82
83 - alert: HostCPUStealHigh
84 # Hostinger oversubscription signal — the hypervisor is taking CPU.
85 expr: |
86 avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 25
87 for: 15m
88 labels:
89 severity: warning
90 annotations:
91 summary: 'host {{ $labels.instance }} CPU steal >25%'
92 description: 'Hypervisor stealing {{ $value | printf "%.0f" }}% CPU for 15m. VPS is oversold — consider resizing or moving load.'
93
94 - alert: ContainerMemoryHigh
95 # Any single container >6 GB RSS. Catches containerized heavy builds
96 # and memory-leaking apps. Host-level builds are caught by HostMemory*.
97 expr: |
98 container_memory_rss{name!=""} > 6e9
99 for: 10m
100 labels:
101 severity: warning
102 annotations:
103 summary: 'container {{ $labels.name }} using >6 GB RAM'
104 description: '{{ $labels.name }} RSS {{ $value | humanize1024 }}B for 10m on a 16 GB host.'
105
106 - alert: ContainerOOMKilled
107 expr: |
108 increase(container_oom_events_total{name!=""}[10m]) > 0
109 labels:
110 severity: warning
111 annotations:
112 summary: 'container {{ $labels.name }} OOM-killed'
113 description: '{{ $labels.name }} hit an OOM event in the last 10m.'
114
115 - alert: ContainerRestarting
116 expr: |
117 changes(container_start_time_seconds{name!=""}[15m]) > 2
118 labels:
119 severity: warning
120 annotations:
121 summary: 'container {{ $labels.name }} restart-looping'
122 description: '{{ $labels.name }} restarted >2 times in 15m.'