konnosDashboard
auto-snapshot.ts165 lines · main
1import { env } from '../env.js';
2import { audit } from '../services/audit.js';
3import {
4 claimDueAutoSnapshots,
5 nextAutoRunAfter,
6 recordAutoSnapshotResult,
7 type DueAutoSnapshot,
8} from '../services/auto-snapshots.js';
9import { log } from '../lib/logger.js';
10import { createSnapshot, pruneAutoSnapshots } from '../services/snapshots.js';
11
12/**
13 * Automatic scheduled snapshots worker. Every TICK_MS:
14 *
15 * 1. claims auto-snapshot settings rows that are due (enabled and
16 * next_run_at <= now) — optimistic SELECT, no row lock.
17 * 2. for each, takes an `auto`-flagged snapshot (createSnapshot), then
18 * prunes auto snapshots beyond retentionCount (pruneAutoSnapshots).
19 * Manual snapshots are never counted or deleted.
20 * 3. records the outcome with an UPDATE guarded by the pre-claim
21 * next_run_at, advancing it to the next slot. Two workers can't
22 * double-fire the same project — the loser skips silently.
23 *
24 * Idempotent: a crash mid-run just means the project's next_run_at hasn't
25 * advanced yet, so the next tick retries it. A re-run never prunes manual
26 * snapshots or removes more than the retention math dictates.
27 *
28 * This is the same trigger mechanism Briven already uses for scheduled
29 * function invocations (workers/schedule-dispatcher.ts): an in-process
30 * interval armed at boot. The run logic is also exposed to an internal
31 * HTTP endpoint (POST /v1/internal/auto-snapshots/run) so an external cron
32 * can drive it in deployments that prefer that over the in-process timer.
33 */
34
35const TICK_MS = 5 * 60 * 1000; // 5 min — checks are cheap; cadence is hours
36const BATCH_SIZE = 50;
37const MAX_ERROR_LEN = 500;
38
39let timer: ReturnType<typeof setInterval> | null = null;
40let inflight = false;
41
42/**
43 * Run every due project's auto-snapshot once. Returns a small summary the
44 * internal endpoint can echo back. Safe to call concurrently with the
45 * in-process timer — the per-row claim guard prevents double-firing.
46 */
47export async function runDueAutoSnapshots(
48 now: Date = new Date(),
49 limit: number = BATCH_SIZE,
50): Promise<{ due: number; succeeded: number; failed: number }> {
51 const due = await claimDueAutoSnapshots(now, limit);
52 if (due.length === 0) return { due: 0, succeeded: 0, failed: 0 };
53 log.info('auto_snapshot_tick', { dueCount: due.length });
54
55 let succeeded = 0;
56 let failed = 0;
57 // Run projects in parallel; each is an independent data-plane operation.
58 const outcomes = await Promise.all(due.map((row) => runOne(row, now)));
59 for (const ok of outcomes) {
60 if (ok) succeeded += 1;
61 else failed += 1;
62 }
63 return { due: due.length, succeeded, failed };
64}
65
66async function runOne(row: DueAutoSnapshot, now: Date): Promise<boolean> {
67 const claimedNextRunAt = row.nextRunAt;
68 const newNextRunAt = nextAutoRunAfter(row.frequency, now);
69 // Clear, dated label so auto snapshots are obvious in the list. The
70 // `auto: true` flag is what pruning keys off — the name is just for humans.
71 const name = `auto-${now.toISOString().slice(0, 10)}`;
72
73 let status: 'ok' | 'error' = 'ok';
74 let errorMessage: string | undefined;
75 let snapshotId: string | undefined;
76 let pruned = 0;
77
78 try {
79 const snap = await createSnapshot(row.projectId, name, { auto: true });
80 snapshotId = snap.id;
81 const result = await pruneAutoSnapshots(row.projectId, row.retentionCount);
82 pruned = result.pruned.length;
83 } catch (err) {
84 status = 'error';
85 errorMessage = (err instanceof Error ? err.message : String(err)).slice(0, MAX_ERROR_LEN);
86 log.error('auto_snapshot_failed', { projectId: row.projectId, message: errorMessage });
87 }
88
89 const applied = await recordAutoSnapshotResult({
90 settingsId: row.id,
91 claimedNextRunAt,
92 newNextRunAt,
93 ranAt: now,
94 status,
95 errorMessage,
96 });
97 if (!applied) {
98 // Lost the race to a concurrent run; the other owns the outcome record.
99 log.info('auto_snapshot_record_lost_race', { projectId: row.projectId });
100 return status === 'ok';
101 }
102
103 await audit({
104 actorId: null,
105 projectId: row.projectId,
106 action: status === 'ok' ? 'studio.snapshot.auto.create' : 'studio.snapshot.auto.error',
107 ipHash: null,
108 userAgent: null,
109 metadata: {
110 ...(snapshotId ? { snapshotId } : {}),
111 pruned,
112 frequency: row.frequency,
113 retentionCount: row.retentionCount,
114 ...(errorMessage ? { error: errorMessage } : {}),
115 },
116 });
117 return status === 'ok';
118}
119
120async function tick(): Promise<void> {
121 if (inflight) {
122 log.warn('auto_snapshot_tick_skipped_inflight');
123 return;
124 }
125 inflight = true;
126 try {
127 await runDueAutoSnapshots(new Date());
128 } catch (err) {
129 log.error('auto_snapshot_tick_failed', {
130 message: err instanceof Error ? err.message : String(err),
131 });
132 } finally {
133 inflight = false;
134 }
135}
136
137/**
138 * Arm the auto-snapshot worker. Idempotent — a second call is a no-op.
139 * Skipped when the control-plane DB isn't configured (dev / pre-migration).
140 */
141export function startAutoSnapshotWorker(): void {
142 if (timer) return;
143 if (!env.BRIVEN_DATABASE_URL) {
144 log.warn('auto_snapshot_worker_skipped_no_db');
145 return;
146 }
147 // 150s after boot — last in the worker startup cascade (storage janitor
148 // is 120s), keeping the boot-time connection-pool burst smooth.
149 setTimeout(() => {
150 void tick();
151 timer = setInterval(() => {
152 void tick();
153 }, TICK_MS);
154 }, 150_000).unref?.();
155 log.info('auto_snapshot_worker_armed', { tickMs: TICK_MS, batch: BATCH_SIZE });
156}
157
158export function stopAutoSnapshotWorker(): void {
159 if (timer) {
160 clearInterval(timer);
161 timer = null;
162 }
163}
164
165export const _internals = { tick, runOne };