konnosDashboard
storage.ts607 lines · main
1import { newId, NotFoundError, ValidationError } from '@briven/shared';
2import { and, asc, desc, eq, gte, isNotNull, isNull, sql as drizzleSql } from 'drizzle-orm';
3
4import { getDb } from '../db/client.js';
5import { projectFiles, type ProjectFile } from '../db/schema.js';
6import { env } from '../env.js';
7import { log } from '../lib/logger.js';
8import { presignS3Url } from '../lib/s3-presign.js';
9import {
10 bucketNameFor,
11 ensureBucket,
12 isMinioAdminConfigured,
13 restoreObject,
14 setRecoveryLifecycle,
15 type RestoreResult,
16} from './minio-admin.js';
17import { getProjectTier, TIERS, TierLimitExceeded } from './tiers.js';
18
19/**
20 * Per-project object storage. Each project's files live under the prefix
21 * `projects/<projectId>/` inside the shared `briven` bucket. The API is
22 * the only path that mints presigned PUT/GET URLs — clients never get
23 * permanent S3 credentials.
24 *
25 * Two endpoints participate:
26 * - BRIVEN_MINIO_ENDPOINT — server-side traffic. `http://minio:9000`
27 * inside the Dokploy compose, or any reachable internal address.
28 * - BRIVEN_MINIO_PUBLIC_ENDPOINT — what the browser sees in presigned
29 * URLs. Must be HTTPS in production (`https://s3.briven.tech`).
30 *
31 * If only the internal endpoint is set, we fall back to it for both — fine
32 * for dev where the browser can reach the same host as the server.
33 */
34
35const MAX_NAME_LEN = 200;
36const MAX_CT_LEN = 100;
37const MAX_BYTES = 1024 * 1024 * 1024; // 1 GiB per file (alpha cap)
38const UPLOAD_URL_TTL = 600; // 10 min
39const DOWNLOAD_URL_TTL = 300; // 5 min
40const CT_RE = /^[a-zA-Z][a-zA-Z0-9!#$&^_+.\-/]{0,99}$/;
41
42/**
43 * Filename validator. Allows any printable unicode; rejects empty, any
44 * byte < 0x20 (control chars — `no-control-regex` lint rule trips on a
45 * regex spelling of this), and the forward slash (path-separator
46 * confusion in URL routing).
47 */
48function isValidFilename(name: string): boolean {
49 if (name.length === 0 || name.length > MAX_NAME_LEN) return false;
50 for (let i = 0; i < name.length; i += 1) {
51 const c = name.charCodeAt(i);
52 if (c < 0x20) return false;
53 if (c === 0x2f) return false;
54 }
55 return true;
56}
57
58interface StorageEnv {
59 endpoint: string;
60 publicEndpoint: string;
61 region: string;
62 bucket: string;
63 accessKey: string;
64 secretKey: string;
65}
66
67function requireStorageEnv(): StorageEnv {
68 const endpoint = env.BRIVEN_MINIO_ENDPOINT;
69 const accessKey = env.BRIVEN_MINIO_ACCESS_KEY;
70 const secretKey = env.BRIVEN_MINIO_SECRET_KEY;
71 if (!endpoint || !accessKey || !secretKey) {
72 throw new ValidationError(
73 'object storage is not configured on this api (BRIVEN_MINIO_* env vars missing)',
74 );
75 }
76 return {
77 endpoint,
78 publicEndpoint: env.BRIVEN_MINIO_PUBLIC_ENDPOINT ?? endpoint,
79 region: env.BRIVEN_MINIO_REGION ?? 'us-east-1',
80 bucket: env.BRIVEN_MINIO_BUCKET ?? 'briven',
81 accessKey,
82 secretKey,
83 };
84}
85
86// Per-project bucket unify (M2). The `bucket` column on project_files is managed
87// by raw SQL (NOT in the drizzle schema) so it can never break existing selects.
88// Everything here fails safe to the shared bucket.
89let bucketColState: 'unknown' | 'ready' | 'unavailable' = 'unknown';
90const provisionedBuckets = new Set<string>();
91
92async function ensureFileBucketColumn(db: ReturnType<typeof getDb>): Promise<boolean> {
93 if (bucketColState !== 'unknown') return bucketColState === 'ready';
94 try {
95 const res = (await db.execute(
96 drizzleSql.raw(
97 "select 1 as present from information_schema.columns where table_name = 'project_files' and column_name = 'bucket' limit 1",
98 ),
99 )) as unknown;
100 const rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
101 if (rows.length === 0) {
102 await db.execute(drizzleSql.raw('alter table project_files add column bucket text'));
103 }
104 bucketColState = 'ready';
105 return true;
106 } catch (err) {
107 log.warn('storage_bucket_column_unavailable', { err: String(err).slice(0, 200) });
108 bucketColState = 'unavailable';
109 return false;
110 }
111}
112
113/** Read a file's bucket (raw), falling back to the shared bucket on any miss/error. */
114async function bucketForFile(
115 db: ReturnType<typeof getDb>,
116 fileId: string,
117 projectId: string,
118 fallback: string,
119): Promise<string> {
120 if (bucketColState === 'unavailable') return fallback;
121 try {
122 const res = (await db.execute(
123 drizzleSql`select bucket from project_files where id = ${fileId} and project_id = ${projectId} limit 1`,
124 )) as unknown;
125 const rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
126 const b = (rows[0] as { bucket?: unknown } | undefined)?.bucket;
127 return typeof b === 'string' && b.length > 0 ? b : fallback;
128 } catch {
129 return fallback;
130 }
131}
132
133// Public-serving flag (M3). Like `bucket`, the `is_public` column on project_files
134// is managed by raw SQL (NOT in the drizzle schema) so it can never break existing
135// selects. Everything here fails safe to "not public".
136let publicColState: 'unknown' | 'ready' | 'unavailable' = 'unknown';
137
138async function ensureFilePublicColumn(db: ReturnType<typeof getDb>): Promise<boolean> {
139 if (publicColState !== 'unknown') return publicColState === 'ready';
140 try {
141 const res = (await db.execute(
142 drizzleSql.raw(
143 "select 1 as present from information_schema.columns where table_name = 'project_files' and column_name = 'is_public' limit 1",
144 ),
145 )) as unknown;
146 const rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
147 if (rows.length === 0) {
148 await db.execute(
149 drizzleSql.raw('alter table project_files add column is_public boolean default false'),
150 );
151 }
152 publicColState = 'ready';
153 return true;
154 } catch (err) {
155 log.warn('storage_public_column_unavailable', { err: String(err).slice(0, 200) });
156 publicColState = 'unavailable';
157 return false;
158 }
159}
160
161/**
162 * Mark a file public (or private) for `/media` serving. Verifies the file exists
163 * (via getFile, which throws NotFoundError if missing/deleted), then flips the raw
164 * is_public flag. Throws ValidationError if the public column can't be provisioned.
165 */
166export async function setFilePublic(
167 fileId: string,
168 projectId: string,
169 isPublic: boolean,
170): Promise<{ fileId: string; public: boolean }> {
171 const db = getDb();
172 await getFile(fileId, projectId); // throws NotFoundError if missing/deleted
173 const ready = await ensureFilePublicColumn(db);
174 if (!ready) {
175 throw new ValidationError('public serving is temporarily unavailable');
176 }
177 await db.execute(
178 drizzleSql`update project_files set is_public = ${isPublic} where id = ${fileId} and project_id = ${projectId}`,
179 );
180 return { fileId, public: isPublic };
181}
182
183/** IDs of a project's currently-public, live files. Fail-safe → [] on any error. */
184export async function listPublicFileIds(projectId: string): Promise<string[]> {
185 if (publicColState === 'unavailable') return [];
186 try {
187 const db = getDb();
188 const res = (await db.execute(
189 drizzleSql`select id from project_files where project_id = ${projectId} and is_public = true and deleted_at is null`,
190 )) as unknown;
191 const rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
192 return rows
193 .map((r) => (r as { id?: unknown }).id)
194 .filter((id): id is string => typeof id === 'string');
195 } catch {
196 return [];
197 }
198}
199
200/**
201 * Open a public object's bytes for the /media route. Returns null (not throws) when
202 * the file isn't public / doesn't exist / storage is unreachable — the route turns
203 * that into a plain 404. Presigns an INTERNAL GET and streams the body straight
204 * through, so the object store credentials never leave the server.
205 */
206export async function openPublicObject(
207 projectId: string,
208 fileId: string,
209): Promise<{
210 body: ReadableStream<Uint8Array>;
211 contentType: string;
212 contentLength: string | null;
213} | null> {
214 if (publicColState === 'unavailable') return null;
215 const db = getDb();
216 let rows: unknown[];
217 try {
218 const res = (await db.execute(
219 drizzleSql`select object_key, content_type, bucket from project_files where id = ${fileId} and project_id = ${projectId} and is_public = true and deleted_at is null limit 1`,
220 )) as unknown;
221 rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
222 } catch {
223 return null;
224 }
225 const row = rows[0] as
226 | { object_key?: unknown; content_type?: unknown; bucket?: unknown }
227 | undefined;
228 if (!row || typeof row.object_key !== 'string') return null;
229
230 const cfg = requireStorageEnv();
231 const bucket = (typeof row.bucket === 'string' && row.bucket) || cfg.bucket;
232 const url = presignS3Url({
233 endpoint: cfg.endpoint,
234 region: cfg.region,
235 bucket,
236 key: row.object_key,
237 method: 'GET',
238 accessKey: cfg.accessKey,
239 secretKey: cfg.secretKey,
240 expiresIn: 120,
241 });
242 const res = await fetch(url);
243 if (!res.ok || !res.body) return null;
244 return {
245 body: res.body,
246 contentType: (typeof row.content_type === 'string' ? row.content_type : null) ?? 'application/octet-stream',
247 contentLength: res.headers.get('content-length'),
248 };
249}
250
251/**
252 * Open an OWNED object's bytes for the tokenized share-link route. Unlike
253 * `openPublicObject`, this does NOT require `is_public` — a share-link is its own
254 * authorization (the caller has already resolved a valid, unexpired, unrevoked
255 * token to this exact `projectId` + `fileId`). It only requires the file exists,
256 * belongs to the project, and is not soft-deleted. Returns null (not throws) on
257 * any miss / storage-unreachable so the route turns that into a plain 404.
258 * Presigns an INTERNAL GET and streams the body straight through, so the object
259 * store credentials never leave the server.
260 */
261export async function openOwnedObject(
262 projectId: string,
263 fileId: string,
264): Promise<{
265 body: ReadableStream<Uint8Array>;
266 contentType: string;
267 contentLength: string | null;
268} | null> {
269 const db = getDb();
270 let rows: unknown[];
271 try {
272 const res = (await db.execute(
273 drizzleSql`select object_key, content_type, bucket from project_files where id = ${fileId} and project_id = ${projectId} and deleted_at is null limit 1`,
274 )) as unknown;
275 rows = Array.isArray(res) ? res : ((res as { rows?: unknown[] })?.rows ?? []);
276 } catch {
277 return null;
278 }
279 const row = rows[0] as
280 | { object_key?: unknown; content_type?: unknown; bucket?: unknown }
281 | undefined;
282 if (!row || typeof row.object_key !== 'string') return null;
283
284 const cfg = requireStorageEnv();
285 const bucket = (typeof row.bucket === 'string' && row.bucket) || cfg.bucket;
286 const url = presignS3Url({
287 endpoint: cfg.endpoint,
288 region: cfg.region,
289 bucket,
290 key: row.object_key,
291 method: 'GET',
292 accessKey: cfg.accessKey,
293 secretKey: cfg.secretKey,
294 expiresIn: 120,
295 });
296 const res = await fetch(url);
297 if (!res.ok || !res.body) return null;
298 return {
299 body: res.body,
300 contentType:
301 (typeof row.content_type === 'string' ? row.content_type : null) ??
302 'application/octet-stream',
303 contentLength: res.headers.get('content-length'),
304 };
305}
306
307/**
308 * Probe at boot — returns null silently if storage isn't configured, so
309 * the rest of the API stays up. Callers (the routes) translate this into
310 * a 503 / "not configured" error response.
311 */
312export function isStorageConfigured(): boolean {
313 return Boolean(
314 env.BRIVEN_MINIO_ENDPOINT && env.BRIVEN_MINIO_ACCESS_KEY && env.BRIVEN_MINIO_SECRET_KEY,
315 );
316}
317
318export async function listFiles(projectId: string): Promise<ProjectFile[]> {
319 const db = getDb();
320 return db
321 .select()
322 .from(projectFiles)
323 .where(and(eq(projectFiles.projectId, projectId), isNull(projectFiles.deletedAt)))
324 .orderBy(asc(projectFiles.name));
325}
326
327export async function getFile(fileId: string, projectId: string): Promise<ProjectFile> {
328 const db = getDb();
329 const rows = await db
330 .select()
331 .from(projectFiles)
332 .where(
333 and(
334 eq(projectFiles.id, fileId),
335 eq(projectFiles.projectId, projectId),
336 isNull(projectFiles.deletedAt),
337 ),
338 )
339 .limit(1);
340 const row = rows[0];
341 if (!row) throw new NotFoundError('file', fileId);
342 return row;
343}
344
345export interface PresignUploadInput {
346 projectId: string;
347 name: string;
348 contentType: string;
349 sizeBytes: number;
350 uploadedBy: string | null;
351}
352
353export interface PresignUploadResult {
354 file: ProjectFile;
355 uploadUrl: string;
356 // Echo back so the browser can include the same header it signed for.
357 requiredHeaders: Record<string, string>;
358 expiresInSec: number;
359}
360
361export async function presignUpload(input: PresignUploadInput): Promise<PresignUploadResult> {
362 if (!isValidFilename(input.name)) {
363 throw new ValidationError('filename contains invalid characters or is empty');
364 }
365 if (!CT_RE.test(input.contentType)) {
366 throw new ValidationError('invalid content-type');
367 }
368 if (input.contentType.length > MAX_CT_LEN) {
369 throw new ValidationError(`content-type exceeds ${MAX_CT_LEN} chars`);
370 }
371 if (!Number.isFinite(input.sizeBytes) || input.sizeBytes < 0) {
372 throw new ValidationError('sizeBytes must be a non-negative integer');
373 }
374 if (input.sizeBytes > MAX_BYTES) {
375 throw new ValidationError(`file exceeds ${MAX_BYTES} byte upload cap`);
376 }
377
378 const cfg = requireStorageEnv();
379 const fileId = newId('f');
380 const objectKey = `projects/${input.projectId}/${fileId}`;
381
382 const db = getDb();
383
384 // Tier storage cap — hard-block over-quota uploads (M2). Sums this project's
385 // live file bytes; refuses the presign if the new file would bust the tier cap.
386 // Existing files stay readable — only the new upload is blocked. (Kept recovery
387 // versions aren't counted yet; that arrives with the per-project bucket meter.)
388 const tier = (await getProjectTier(input.projectId)) ?? 'free';
389 const cap = TIERS[tier].storageBytes;
390 const [usageRow] = await db
391 .select({
392 used: drizzleSql<number>`coalesce(sum(cast(${projectFiles.sizeBytes} as bigint)), 0)::bigint`,
393 })
394 .from(projectFiles)
395 .where(and(eq(projectFiles.projectId, input.projectId), isNull(projectFiles.deletedAt)));
396 const used = Number(usageRow?.used ?? 0);
397 if (used + input.sizeBytes > cap) {
398 throw new TierLimitExceeded(
399 `storage full: ${used} + ${input.sizeBytes} bytes exceeds the '${tier}' tier cap of ${cap}`,
400 { projectId: input.projectId, tier, used, incoming: input.sizeBytes, cap },
401 );
402 }
403
404 // Decide the destination bucket. Default = legacy shared bucket. Only switch to
405 // the project's own bucket if the bucket column is usable AND minio admin is
406 // configured AND provisioning succeeds — otherwise fall back to shared.
407 const colReady = await ensureFileBucketColumn(db);
408 let targetBucket = cfg.bucket;
409 if (colReady && isMinioAdminConfigured()) {
410 const perProject = bucketNameFor(input.projectId);
411 try {
412 if (!provisionedBuckets.has(perProject)) {
413 await ensureBucket(perProject); // creates bucket + enables versioning (idempotent)
414 await setRecoveryLifecycle(perProject, TIERS[tier].storageRecoveryDays);
415 provisionedBuckets.add(perProject);
416 }
417 targetBucket = perProject;
418 } catch (err) {
419 log.warn('storage_per_project_provision_failed', {
420 projectId: input.projectId,
421 err: String(err).slice(0, 200),
422 });
423 targetBucket = cfg.bucket; // fail safe
424 }
425 }
426
427 const inserted = await db
428 .insert(projectFiles)
429 .values({
430 id: fileId,
431 projectId: input.projectId,
432 name: input.name,
433 objectKey,
434 contentType: input.contentType,
435 sizeBytes: String(input.sizeBytes),
436 uploadedBy: input.uploadedBy,
437 })
438 .returning();
439 const file = inserted[0];
440 if (!file) throw new Error('file row insert returned nothing');
441
442 // Record which bucket this object lives in (raw — the column isn't in the
443 // drizzle schema). If we can't record it, upload to the shared bucket instead
444 // so download/delete (which fall back to shared) can always find it.
445 if (colReady && targetBucket !== cfg.bucket) {
446 try {
447 await db.execute(
448 drizzleSql`update project_files set bucket = ${targetBucket} where id = ${fileId}`,
449 );
450 } catch (err) {
451 log.warn('storage_bucket_record_failed', { fileId, err: String(err).slice(0, 200) });
452 targetBucket = cfg.bucket;
453 }
454 }
455
456 const uploadUrl = presignS3Url({
457 endpoint: cfg.publicEndpoint,
458 region: cfg.region,
459 bucket: targetBucket,
460 key: objectKey,
461 method: 'PUT',
462 accessKey: cfg.accessKey,
463 secretKey: cfg.secretKey,
464 expiresIn: UPLOAD_URL_TTL,
465 contentType: input.contentType,
466 });
467
468 return {
469 file,
470 uploadUrl,
471 requiredHeaders: { 'content-type': input.contentType },
472 expiresInSec: UPLOAD_URL_TTL,
473 };
474}
475
476export interface PresignDownloadResult {
477 file: ProjectFile;
478 downloadUrl: string;
479 expiresInSec: number;
480}
481
482export async function presignDownload(
483 fileId: string,
484 projectId: string,
485): Promise<PresignDownloadResult> {
486 const file = await getFile(fileId, projectId);
487 const cfg = requireStorageEnv();
488 const db = getDb();
489 const bucket = await bucketForFile(db, fileId, projectId, cfg.bucket);
490 const downloadUrl = presignS3Url({
491 endpoint: cfg.publicEndpoint,
492 region: cfg.region,
493 bucket,
494 key: file.objectKey,
495 method: 'GET',
496 accessKey: cfg.accessKey,
497 secretKey: cfg.secretKey,
498 expiresIn: DOWNLOAD_URL_TTL,
499 });
500 return { file, downloadUrl, expiresInSec: DOWNLOAD_URL_TTL };
501}
502
503export async function deleteFile(fileId: string, projectId: string): Promise<ProjectFile> {
504 const file = await getFile(fileId, projectId);
505 const cfg = requireStorageEnv();
506
507 // Soft-delete the metadata row first. If the MinIO DELETE call below
508 // fails, the file is no longer listable + the object key is uniquely
509 // tied to this row so it'll never be reissued — a future janitor can
510 // sweep orphaned objects safely.
511 const db = getDb();
512 const bucket = await bucketForFile(db, file.id, projectId, cfg.bucket);
513 await db
514 .update(projectFiles)
515 .set({ deletedAt: new Date(), updatedAt: new Date() })
516 .where(and(eq(projectFiles.id, fileId), isNull(projectFiles.deletedAt)));
517
518 // Server-side DELETE uses the INTERNAL endpoint (faster, doesn't bounce
519 // through traefik) and the same sigv4 algorithm — just method=DELETE.
520 const internalDeleteUrl = presignS3Url({
521 endpoint: cfg.endpoint,
522 region: cfg.region,
523 bucket,
524 key: file.objectKey,
525 method: 'DELETE',
526 accessKey: cfg.accessKey,
527 secretKey: cfg.secretKey,
528 expiresIn: 60,
529 });
530 const res = await fetch(internalDeleteUrl, { method: 'DELETE' });
531 if (!res.ok && res.status !== 404) {
532 // 404 means already gone — fine. Anything else: leave the row
533 // soft-deleted and surface the error so the operator notices.
534 const body = await res.text().catch(() => '');
535 throw new Error(`minio delete failed: ${res.status} ${body.slice(0, 200)}`);
536 }
537 return { ...file, deletedAt: new Date() };
538}
539
540/** Soft-deleted files still inside the tier recovery window (newest first). */
541export async function listDeletedFiles(projectId: string): Promise<ProjectFile[]> {
542 const db = getDb();
543 const tier = (await getProjectTier(projectId)) ?? 'free';
544 const windowDays = TIERS[tier].storageRecoveryDays;
545 const cutoff = new Date(Date.now() - windowDays * 86_400_000);
546 return db
547 .select()
548 .from(projectFiles)
549 .where(
550 and(
551 eq(projectFiles.projectId, projectId),
552 isNotNull(projectFiles.deletedAt),
553 gte(projectFiles.deletedAt, cutoff),
554 ),
555 )
556 .orderBy(desc(projectFiles.deletedAt));
557}
558
559/** Restore a soft-deleted file within its recovery window (undo delete). */
560export async function restoreFile(
561 fileId: string,
562 projectId: string,
563): Promise<{ file: ProjectFile; status: RestoreResult }> {
564 const db = getDb();
565 const [row] = await db
566 .select()
567 .from(projectFiles)
568 .where(
569 and(
570 eq(projectFiles.id, fileId),
571 eq(projectFiles.projectId, projectId),
572 isNotNull(projectFiles.deletedAt),
573 ),
574 )
575 .limit(1);
576 if (!row) throw new NotFoundError('deleted file', fileId);
577
578 const tier = (await getProjectTier(projectId)) ?? 'free';
579 const windowDays = TIERS[tier].storageRecoveryDays;
580 const deletedAtMs = row.deletedAt ? new Date(row.deletedAt).getTime() : 0;
581 if (deletedAtMs < Date.now() - windowDays * 86_400_000) {
582 throw new ValidationError(
583 `recovery window of ${windowDays} days has passed — the kept copy has expired and can't be restored`,
584 );
585 }
586
587 const cfg = requireStorageEnv();
588 const bucket = await bucketForFile(db, fileId, projectId, cfg.bucket);
589 let status: RestoreResult = 'nothing-to-restore';
590 if (isMinioAdminConfigured()) {
591 status = await restoreObject(bucket, row.objectKey);
592 if (status === 'expired') {
593 throw new ValidationError('the kept copy has already expired — cannot recover this file');
594 }
595 if (status === 'error') {
596 throw new Error('failed to restore the object in storage');
597 }
598 }
599 await db
600 .update(projectFiles)
601 .set({ deletedAt: null, updatedAt: new Date() })
602 .where(eq(projectFiles.id, fileId));
603 return { file: { ...row, deletedAt: null }, status };
604}
605
606// Re-export for callers that want raw access (rare).
607export { drizzleSql };