konnosDashboard
platform-health.ts324 lines · main
1/**
2 * Platform health summary — the single source of truth for the four
3 * upstream readiness checks. Both /ready (apps/api) and the superadmin
4 * Overview consume this so the two can never disagree about what
5 * "healthy" means.
6 *
7 * `host` (CPU / RAM / disk) comes from the Phase 4 observability stack:
8 * node-exporter scraped by Prometheus, queried here via the instant API.
9 * HARD honesty rule — if Prometheus is unset/unreachable, or an
10 * individual query returns no data, that field is null and the UI shows
11 * "—". We NEVER fabricate a number so a dead exporter can't masquerade
12 * as a healthy 0%.
13 */
14import { pingDb } from '../db/client.js';
15import { deepPingDataPlane } from '../db/data-plane.js';
16import { env } from '../env.js';
17import { log } from '../lib/logger.js';
18import { pingRedis } from '../lib/redis.js';
19
20export type HealthCheck = 'ok' | 'unreachable' | 'not_configured';
21
22export interface HealthChecks {
23 control_postgres: HealthCheck;
24 data_plane_postgres: HealthCheck;
25 runtime: HealthCheck;
26 realtime: HealthCheck;
27 redis: HealthCheck;
28}
29
30/**
31 * Real host metrics for the primary server, sourced from node-exporter
32 * via Prometheus. Each numeric field is independently nullable: a single
33 * failed/empty query degrades only that field, not the whole object.
34 *
35 * Multi-host: the prometheus.yml `node` job scrapes one node-exporter
36 * target today, so this is a single-host summary. If the deploy grows to
37 * multiple hosts, the non-aggregated queries (memory) return one series
38 * per host and we report the FIRST (primary) — see parsePromSample. The
39 * `instance` label surfaces which host the numbers belong to.
40 */
41export interface HostMetrics {
42 cpuPercent: number | null;
43 memUsedBytes: number | null;
44 memTotalBytes: number | null;
45 diskPercent: number | null;
46 stealPercent: number | null;
47 instance?: string;
48}
49
50export interface HealthSummary {
51 checks: HealthChecks;
52 /** Real host metrics, or null when Prometheus is unset/unreachable. */
53 host: HostMetrics | null;
54}
55
56/**
57 * The PromQL behind each host metric. Kept as data (not inlined) so the
58 * queries are auditable in one place and reusable in tests. Mountpoint
59 * "/" matches node-exporter's default root filesystem label.
60 */
61export const PROM_QUERIES = {
62 cpuPercent: '100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
63 memTotalBytes: 'node_memory_MemTotal_bytes',
64 memAvailableBytes: 'node_memory_MemAvailable_bytes',
65 diskPercent:
66 '100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100)',
67 stealPercent: 'avg(rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100',
68} as const;
69
70/** One sample from a Prometheus instant-query vector result. */
71export interface PromSample {
72 value: number | null;
73 instance?: string;
74}
75
76/**
77 * Pure parser for a Prometheus instant-query (`/api/v1/query`) response.
78 * Returns the FIRST series' value as a finite number, plus its `instance`
79 * label when present. Returns `{ value: null }` for any non-success
80 * status, empty result, or unparseable value — never throws, never
81 * fabricates. Exported so the parse logic is unit-tested in isolation.
82 */
83export function parsePromSample(json: unknown): PromSample {
84 if (!json || typeof json !== 'object') return { value: null };
85 const root = json as { status?: unknown; data?: unknown };
86 if (root.status !== 'success' || !root.data || typeof root.data !== 'object') {
87 return { value: null };
88 }
89 const result = (root.data as { result?: unknown }).result;
90 if (!Array.isArray(result) || result.length === 0) return { value: null };
91 const first = result[0] as { value?: unknown; metric?: unknown };
92 if (!Array.isArray(first.value) || first.value.length < 2) return { value: null };
93 const raw = first.value[1];
94 const num = typeof raw === 'number' ? raw : Number.parseFloat(String(raw));
95 if (!Number.isFinite(num)) return { value: null };
96 const instance =
97 first.metric && typeof first.metric === 'object'
98 ? (first.metric as Record<string, unknown>).instance
99 : undefined;
100 return {
101 value: num,
102 ...(typeof instance === 'string' ? { instance } : {}),
103 };
104}
105
106/** Round a nullable percent to one decimal; null stays null. */
107function round1(n: number | null): number | null {
108 return n === null ? null : Math.round(n * 10) / 10;
109}
110
111// In-memory cache so the cockpit (and /ready, which shares getHealthSummary)
112// can't hammer Prometheus on every page load. 15s is well under the
113// dashboard's natural refresh cadence yet fresh enough to be useful.
114const HOST_METRICS_TTL_MS = 15_000;
115let hostCache: { at: number; value: HostMetrics | null } | null = null;
116
117/**
118 * Query a single PromQL expression against the instant API. Any
119 * failure (network, timeout, non-2xx, bad JSON, empty result) collapses
120 * to `{ value: null }` so one dead metric never poisons the others.
121 */
122async function queryProm(base: string, query: string): Promise<PromSample> {
123 try {
124 const url = `${base}/api/v1/query?query=${encodeURIComponent(query)}`;
125 const res = await fetch(url, { signal: AbortSignal.timeout(2500) });
126 if (!res.ok) return { value: null };
127 return parsePromSample(await res.json());
128 } catch {
129 return { value: null };
130 }
131}
132
133/**
134 * Real host metrics from Prometheus, or null when monitoring isn't
135 * connected. Guarded so it's a no-op (and the UI honest) until an
136 * operator sets BRIVEN_PROMETHEUS_URL — which keeps /ready + the health
137 * route tests green with zero observability stack in dev/CI.
138 */
139export async function getHostMetrics(): Promise<HostMetrics | null> {
140 if (!env.BRIVEN_PROMETHEUS_URL) return null;
141
142 const now = Date.now();
143 if (hostCache && now - hostCache.at < HOST_METRICS_TTL_MS) {
144 return hostCache.value;
145 }
146
147 try {
148 const base = env.BRIVEN_PROMETHEUS_URL;
149 const [cpu, memTotal, memAvail, disk, steal] = await Promise.all([
150 queryProm(base, PROM_QUERIES.cpuPercent),
151 queryProm(base, PROM_QUERIES.memTotalBytes),
152 queryProm(base, PROM_QUERIES.memAvailableBytes),
153 queryProm(base, PROM_QUERIES.diskPercent),
154 queryProm(base, PROM_QUERIES.stealPercent),
155 ]);
156
157 const memUsedBytes =
158 memTotal.value !== null && memAvail.value !== null
159 ? memTotal.value - memAvail.value
160 : null;
161
162 // If every query came back empty, Prometheus is reachable but has no
163 // host data (e.g. node-exporter down) — report null, not a row of "—".
164 const allNull =
165 cpu.value === null &&
166 memTotal.value === null &&
167 memAvail.value === null &&
168 disk.value === null &&
169 steal.value === null;
170
171 const value: HostMetrics | null = allNull
172 ? null
173 : {
174 cpuPercent: round1(cpu.value),
175 memUsedBytes,
176 memTotalBytes: memTotal.value,
177 diskPercent: round1(disk.value),
178 stealPercent: round1(steal.value),
179 // memTotal is a non-aggregated series so it carries the host's
180 // instance label; the aggregated cpu/steal queries don't.
181 ...(memTotal.instance ? { instance: memTotal.instance } : {}),
182 };
183
184 hostCache = { at: now, value };
185 return value;
186 } catch {
187 hostCache = { at: now, value: null };
188 return null;
189 }
190}
191
192async function probeRuntime(): Promise<boolean> {
193 if (!env.BRIVEN_RUNTIME_URL) return false;
194 try {
195 const res = await fetch(`${env.BRIVEN_RUNTIME_URL}/health`, {
196 signal: AbortSignal.timeout(2000),
197 });
198 return res.ok;
199 } catch {
200 return false;
201 }
202}
203
204// Mirrors probeRuntime exactly: a 2s liveness fetch to realtime's
205// /health, collapsing any failure to false so a realtime outage degrades
206// /ready the same way a runtime outage does.
207async function probeRealtime(): Promise<boolean> {
208 if (!env.BRIVEN_REALTIME_URL) return false;
209 try {
210 const res = await fetch(`${env.BRIVEN_REALTIME_URL}/health`, {
211 signal: AbortSignal.timeout(2000),
212 });
213 return res.ok;
214 } catch {
215 return false;
216 }
217}
218
219/**
220 * Deep vault gate with a 3-strike SEATBELT.
221 *
222 * The data-plane readiness verdict that feeds /ready comes from the REAL-login
223 * probe (deepPingDataPlane), NOT a warm-pool SELECT 1 — so it catches the
224 * broken-auth outage that a pooled connection masks. But a single blip must
225 * NEVER pull the live site: the gate only flips to unhealthy after
226 * DEEP_FAIL_THRESHOLD *consecutive* failures. One success resets the streak.
227 *
228 * Probe results are cached for DEEP_PROBE_TTL_MS so /ready + the admin overview
229 * share one probe (no hammering the vault) — which also naturally spaces the
230 * strikes a few seconds apart, so 3 strikes ≈ ~15s of *sustained* failure
231 * before the site is ever gated. Fail-SAFE at boot: starts healthy so a cold
232 * start is never blocked before the first probe.
233 */
234const DEEP_PROBE_TTL_MS = 5_000;
235const DEEP_FAIL_THRESHOLD = 3;
236const deepGate = { at: 0, streak: 0, gatingOk: true, lastOk: true };
237
238/** Test-only: reset the seatbelt state between cases. */
239export function _resetDataPlaneGate(): void {
240 deepGate.at = 0;
241 deepGate.streak = 0;
242 deepGate.gatingOk = true;
243 deepGate.lastOk = true;
244}
245
246async function evaluateDataPlaneGate(): Promise<boolean> {
247 const now = Date.now();
248 if (deepGate.at !== 0 && now - deepGate.at < DEEP_PROBE_TTL_MS) {
249 return deepGate.gatingOk; // reuse the cached verdict within the window
250 }
251 const ok = await deepPingDataPlane();
252 deepGate.at = now;
253 deepGate.lastOk = ok;
254 if (ok) {
255 if (!deepGate.gatingOk) {
256 log.warn('data_plane_gate_recovered', { afterStreak: deepGate.streak });
257 }
258 deepGate.streak = 0;
259 deepGate.gatingOk = true;
260 } else {
261 deepGate.streak += 1;
262 if (deepGate.streak >= DEEP_FAIL_THRESHOLD && deepGate.gatingOk) {
263 deepGate.gatingOk = false;
264 // Site is being gated OFF — the loudest signal, so alerting catches it.
265 log.error('data_plane_gate_tripped', {
266 streak: deepGate.streak,
267 threshold: DEEP_FAIL_THRESHOLD,
268 });
269 } else if (deepGate.gatingOk) {
270 // Failing, but still inside the seatbelt — early warning, no gating yet.
271 log.warn('data_plane_gate_strike', {
272 streak: deepGate.streak,
273 threshold: DEEP_FAIL_THRESHOLD,
274 });
275 }
276 }
277 return deepGate.gatingOk;
278}
279
280export async function getHealthSummary(): Promise<HealthSummary> {
281 const [controlOk, dataOk, runtimeOk, realtimeOk, redisOk, host] = await Promise.all([
282 env.BRIVEN_DATABASE_URL ? pingDb() : Promise.resolve(false),
283 env.BRIVEN_DATA_PLANE_URL ? evaluateDataPlaneGate() : Promise.resolve(false),
284 probeRuntime(),
285 probeRealtime(),
286 env.BRIVEN_REDIS_URL ? pingRedis() : Promise.resolve(false),
287 // null fast when Prometheus is unset; cached ~15s otherwise. /ready
288 // ignores host, so this never affects the readiness verdict.
289 getHostMetrics(),
290 ]);
291
292 const checks: HealthChecks = {
293 control_postgres: env.BRIVEN_DATABASE_URL
294 ? controlOk
295 ? 'ok'
296 : 'unreachable'
297 : 'not_configured',
298 data_plane_postgres: env.BRIVEN_DATA_PLANE_URL
299 ? dataOk
300 ? 'ok'
301 : 'unreachable'
302 : 'not_configured',
303 runtime: runtimeOk ? 'ok' : 'unreachable',
304 realtime: realtimeOk ? 'ok' : 'unreachable',
305 redis: env.BRIVEN_REDIS_URL ? (redisOk ? 'ok' : 'unreachable') : 'not_configured',
306 };
307
308 return { checks, host };
309}
310
311/**
312 * Derives the /ready boolean from the checks. Redis powers logs streaming
313 * + rate limits — required only when configured; unconfigured = dev mode
314 * where logs/limits silently no-op.
315 */
316export function isReady(checks: HealthChecks): boolean {
317 return (
318 checks.control_postgres === 'ok' &&
319 checks.data_plane_postgres === 'ok' &&
320 checks.runtime === 'ok' &&
321 checks.realtime === 'ok' &&
322 (checks.redis === 'ok' || checks.redis === 'not_configured')
323 );
324}