Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,257 @@
|
||||
// AK-P4a §4 — file-preview service (BullMQ `toolctrl.file-preview`).
|
||||
//
|
||||
// Mirrors scanService.ts: a dedicated AWS S3 client built from STORAGE_* env
|
||||
// GETs the original, renders the preview and PUTs preview/thumb into the same
|
||||
// private bucket (AD-23). previewStatus is written back onto the File row
|
||||
// (pending -> generating -> ready|failed|not_supported).
|
||||
//
|
||||
// Format matrix v1 (designer brief 2026-06-12):
|
||||
// image/heic|heif -> heic-convert -> JPG -> preview.jpg + thumb.webp
|
||||
// image/jpeg|png|gif|webp -> sharp -> WebP -> preview.webp + thumb.webp
|
||||
// application/pdf -> pdf-to-png -> PNG -> preview.png + thumb.webp
|
||||
// text/plain | markdown -> ready (inline render from bytes, no file)
|
||||
// anything else -> not_supported
|
||||
//
|
||||
// The native render libs (sharp, heic-convert, pdf-to-png-converter) are loaded
|
||||
// via dynamic string import (same pattern as qdrant/sentry in index.ts) so
|
||||
// typecheck/build stay green before they are installed.
|
||||
// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @toolctrl/worker`.
|
||||
// If a lib is missing at runtime the service falls back cleanly to
|
||||
// previewStatus=failed (no hard crash) and the user can regenerate from detail.
|
||||
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { FilePreviewJob } from "@toolctrl/queue";
|
||||
import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@toolctrl/storage";
|
||||
|
||||
import { withTenantRLS } from "../db/tenantRls.js";
|
||||
import type { FilePreviewServiceResult } from "../handlers.js";
|
||||
|
||||
export interface FilePreviewServiceEnv {
|
||||
STORAGE_ENDPOINT?: string;
|
||||
STORAGE_REGION?: string;
|
||||
STORAGE_ACCESS_KEY_ID?: string;
|
||||
STORAGE_SECRET_ACCESS_KEY?: string;
|
||||
STORAGE_SESSION_TOKEN?: string;
|
||||
STORAGE_FORCE_PATH_STYLE?: string;
|
||||
}
|
||||
|
||||
type PreviewLogger = {
|
||||
info: (fields: Record<string, unknown>, msg: string) => void;
|
||||
warn: (fields: Record<string, unknown>, msg: string) => void;
|
||||
error: (fields: Record<string, unknown>, msg: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The File write surface the preview status write-back uses. `$executeRaw` is
|
||||
* required so the helper can set `app.tenant_id` on the transaction (AK-P4a RLS
|
||||
* prep). `data` stays string-typed to avoid coupling to the generated Prisma
|
||||
* enum types (keeps the dynamic-import build green before native libs install).
|
||||
*/
|
||||
interface FilePreviewTx {
|
||||
$executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
|
||||
file: {
|
||||
update(args: {
|
||||
where: { id: string };
|
||||
data: {
|
||||
previewStatus?: string;
|
||||
previewStorageKey?: string | null;
|
||||
thumbnailStorageKey?: string | null;
|
||||
};
|
||||
}): Promise<unknown>;
|
||||
};
|
||||
}
|
||||
|
||||
/** Minimal Prisma surface the service needs (tenant transaction). */
|
||||
export interface FilePreviewPrisma {
|
||||
$transaction<T>(fn: (tx: FilePreviewTx) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
export interface FilePreviewService {
|
||||
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
|
||||
}
|
||||
|
||||
const THUMBNAIL_MAX_DIMENSION = 480;
|
||||
const PREVIEW_MAX_DIMENSION = 2048;
|
||||
|
||||
const RASTER_IMAGE_TYPES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
|
||||
const HEIC_TYPES = new Set(["image/heic", "image/heif"]);
|
||||
const TEXT_TYPES = new Set(["text/plain", "text/markdown", "text/csv"]);
|
||||
|
||||
function normalizeMime(mimeType: string): string {
|
||||
return mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the preview service from STORAGE_* env. Returns undefined when the S3
|
||||
* credentials are missing (the bootstrap wiring then skips it and the handler
|
||||
* refuses jobs with a config error instead of running blind).
|
||||
*/
|
||||
export function createWorkerFilePreviewService(
|
||||
env: EnvRecord & FilePreviewServiceEnv,
|
||||
options: { prisma: FilePreviewPrisma; logger: PreviewLogger },
|
||||
): FilePreviewService | undefined {
|
||||
if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const { prisma, logger } = options;
|
||||
const s3 = new AwsS3Client({
|
||||
endpoint: env.STORAGE_ENDPOINT,
|
||||
region: env.STORAGE_REGION,
|
||||
forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
|
||||
credentials: {
|
||||
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
|
||||
sessionToken: env.STORAGE_SESSION_TOKEN,
|
||||
},
|
||||
});
|
||||
|
||||
async function loadOriginal(bucket: string, key: string): Promise<Buffer> {
|
||||
const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const body = result.Body;
|
||||
if (!body) {
|
||||
throw new Error(`S3 GET ${bucket}/${key} returned an empty body`);
|
||||
}
|
||||
// The AWS SDK exposes transformToByteArray() on the Node stream body.
|
||||
const bytes = await (body as { transformToByteArray(): Promise<Uint8Array> }).transformToByteArray();
|
||||
return Buffer.from(bytes);
|
||||
}
|
||||
|
||||
async function putRendered(bucket: string, key: string, body: Buffer, contentType: string): Promise<void> {
|
||||
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }));
|
||||
}
|
||||
|
||||
async function writeStatus(
|
||||
tenantId: string,
|
||||
fileId: string,
|
||||
data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null },
|
||||
): Promise<void> {
|
||||
// Set the tenant GUC so the write satisfies the File RLS policy once it is
|
||||
// activated (P5 cutover). Preview jobs always carry a tenantId.
|
||||
await withTenantRLS(prisma, tenantId, (tx) => tx.file.update({ where: { id: fileId }, data }));
|
||||
}
|
||||
|
||||
return {
|
||||
async generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult> {
|
||||
const mime = normalizeMime(payload.mimeType);
|
||||
const keys = buildFileStorageKeys(payload.tenantId, payload.fileId);
|
||||
|
||||
// Plain text / markdown: detail view renders inline from bytes, no
|
||||
// rendered preview file needed.
|
||||
if (TEXT_TYPES.has(mime)) {
|
||||
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "ready", previewStorageKey: null, thumbnailStorageKey: null });
|
||||
return { previewStatus: "ready" };
|
||||
}
|
||||
|
||||
const isHeic = HEIC_TYPES.has(mime);
|
||||
const isRaster = RASTER_IMAGE_TYPES.has(mime);
|
||||
const isPdf = mime === "application/pdf";
|
||||
|
||||
if (!isHeic && !isRaster && !isPdf) {
|
||||
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "not_supported", previewStorageKey: null, thumbnailStorageKey: null });
|
||||
return { previewStatus: "not_supported" };
|
||||
}
|
||||
|
||||
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "generating" });
|
||||
|
||||
try {
|
||||
const original = await loadOriginal(payload.bucket, payload.storageKey);
|
||||
let previewBuffer: Buffer;
|
||||
let previewExt: FilePreviewExtension;
|
||||
let previewContentType: string;
|
||||
|
||||
if (isHeic) {
|
||||
// HEIC -> JPG (mandatory for iPhone field-worker photos).
|
||||
const jpg = await heicToJpeg(original);
|
||||
previewBuffer = await rasterToFormat(jpg, "jpeg", PREVIEW_MAX_DIMENSION);
|
||||
previewExt = "jpg";
|
||||
previewContentType = "image/jpeg";
|
||||
} else if (isPdf) {
|
||||
const png = await pdfFirstPageToPng(original);
|
||||
previewBuffer = await rasterToFormat(png, "png", PREVIEW_MAX_DIMENSION);
|
||||
previewExt = "png";
|
||||
previewContentType = "image/png";
|
||||
} else {
|
||||
previewBuffer = await rasterToFormat(original, "webp", PREVIEW_MAX_DIMENSION);
|
||||
previewExt = "webp";
|
||||
previewContentType = "image/webp";
|
||||
}
|
||||
|
||||
const thumbnail = await rasterToFormat(previewBuffer, "webp", THUMBNAIL_MAX_DIMENSION);
|
||||
|
||||
const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, previewExt);
|
||||
const thumbnailKey = keys.thumbnail;
|
||||
|
||||
await putRendered(payload.bucket, previewKey, previewBuffer, previewContentType);
|
||||
await putRendered(payload.bucket, thumbnailKey, thumbnail, "image/webp");
|
||||
|
||||
await writeStatus(payload.tenantId, payload.fileId, {
|
||||
previewStatus: "ready",
|
||||
previewStorageKey: previewKey,
|
||||
thumbnailStorageKey: thumbnailKey,
|
||||
});
|
||||
|
||||
return { previewStatus: "ready", previewStorageKey: previewKey, thumbnailStorageKey: thumbnailKey };
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
logger.warn({ fileId: payload.fileId, tenantId: payload.tenantId, mime }, "file preview generation failed");
|
||||
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "failed" });
|
||||
return { previewStatus: "failed", reason };
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// --- dynamic native-lib loaders (kept untyped so build stays green pre-install) ---
|
||||
|
||||
type SharpFactory = (input: Buffer) => {
|
||||
rotate(): ReturnType<SharpFactory>;
|
||||
resize(opts: { width: number; height: number; fit: "inside"; withoutEnlargement: boolean }): ReturnType<SharpFactory>;
|
||||
jpeg(opts: { quality: number }): ReturnType<SharpFactory>;
|
||||
png(): ReturnType<SharpFactory>;
|
||||
webp(opts: { quality: number }): ReturnType<SharpFactory>;
|
||||
toBuffer(): Promise<Buffer>;
|
||||
};
|
||||
|
||||
let sharpModule: SharpFactory | undefined;
|
||||
async function loadSharp(): Promise<SharpFactory> {
|
||||
if (sharpModule) return sharpModule;
|
||||
const moduleName = "sharp";
|
||||
const loaded = (await import(/* @vite-ignore */ moduleName)) as { default?: SharpFactory } & SharpFactory;
|
||||
sharpModule = (loaded.default ?? loaded) as SharpFactory;
|
||||
return sharpModule;
|
||||
}
|
||||
|
||||
async function rasterToFormat(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
|
||||
const sharp = await loadSharp();
|
||||
const pipeline = sharp(input)
|
||||
.rotate()
|
||||
.resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
|
||||
if (format === "jpeg") return pipeline.jpeg({ quality: 82 }).toBuffer();
|
||||
if (format === "png") return pipeline.png().toBuffer();
|
||||
return pipeline.webp({ quality: 80 }).toBuffer();
|
||||
}
|
||||
|
||||
async function heicToJpeg(input: Buffer): Promise<Buffer> {
|
||||
const moduleName = "heic-convert";
|
||||
const loaded = (await import(/* @vite-ignore */ moduleName)) as
|
||||
| { default: (opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
|
||||
| ((opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
|
||||
const convert = typeof loaded === "function" ? loaded : loaded.default;
|
||||
const out = await convert({ buffer: input, format: "JPEG", quality: 0.85 });
|
||||
return Buffer.from(out as ArrayBuffer);
|
||||
}
|
||||
|
||||
async function pdfFirstPageToPng(input: Buffer): Promise<Buffer> {
|
||||
const moduleName = "pdf-to-png-converter";
|
||||
const loaded = (await import(/* @vite-ignore */ moduleName)) as {
|
||||
pdfToPng: (input: Buffer, opts: Record<string, unknown>) => Promise<Array<{ content: Buffer }>>;
|
||||
};
|
||||
const pages = await loaded.pdfToPng(input, { pagesToProcess: [1], viewportScale: 2.0 });
|
||||
const first = pages[0];
|
||||
if (!first) {
|
||||
throw new Error("PDF rendered no pages");
|
||||
}
|
||||
return Buffer.from(first.content);
|
||||
}
|
||||
Reference in new issue
Block a user