Files
HiveGrid-Wireframes/hivegrid-cloud/apps/web/app/routes/internal/imports/dry-run.tsx
T

356 lines
12 KiB
TypeScript

import type { ActionFunctionArgs } from "react-router";
import {
createImportDryRunPreview,
createRedactedParserDiagnostic,
isSupportedImportFormat,
redactDiagnosticMessage,
type ColumnMapping,
type ImportDryRunFieldDefinition,
type ImportDryRunPreviewInput,
type ImportPreviewInputRow,
type NormalizedImportRow,
type NormalizedRowValue,
type ParserDiagnostic,
} from "@toolctrl/imports";
import { createImportJob, markImportJobFailed, markImportJobRunning, markImportJobSucceeded } from "@toolctrl/db/import-jobs.server";
import { authorizeStorageObjectAccess } from "@toolctrl/storage";
import { requireAuth } from "~/utils/loaders.middleware";
import { verifyUserHasPermission } from "~/utils/helpers/.server/PermissionsService";
import type { DefaultPermission } from "~/application/dtos/shared/DefaultPermissions";
import { validateCSRFToken } from "~/utils/session.server";
import { getTranslations } from "~/locale/i18next.server";
import type { TFunction } from "i18next";
const MAX_PREVIEW_ROWS = 500;
const MAX_MAPPINGS = 200;
const MAX_FIELDS = 200;
const DRY_RUN_SOURCE_BUCKET = "import-dry-run";
type DryRunBody = {
format?: unknown;
importType?: unknown;
source?: unknown;
mappings?: unknown;
rows?: unknown;
fields?: unknown;
};
type DryRunSource = {
tenantId: string;
sourceKey: string;
fileName?: string | null;
contentType?: string | null;
requestId?: string | null;
};
function json(payload: unknown, status = 200) {
return Response.json(payload, { status });
}
function invalidInput(message: string, diagnostics?: readonly ParserDiagnostic[]) {
return json({ error: message, ...(diagnostics && diagnostics.length > 0 ? { diagnostics } : {}) }, 400);
}
function methodNotAllowed(t: TFunction) {
return json({ error: t("models.row.import.errors.methodNotAllowed") }, 405);
}
function storageAccessDenied(t: TFunction, reason: string) {
const status = reason === "invalid_tenant" || reason === "invalid_key" ? 400 : 403;
return json({ error: t("models.row.import.errors.sourceStorageAccessDenied"), reason }, status);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isNormalizedRowValue(value: unknown): value is NormalizedRowValue {
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
return true;
}
return Array.isArray(value) && value.every((item) => item === null || typeof item === "string" || typeof item === "number" || typeof item === "boolean");
}
function parseString(value: unknown, fieldName: string, t: TFunction): string {
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(t("models.row.import.errors.requiredField", { field: fieldName }));
}
return value.trim();
}
function parseOptionalString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function parseSource(value: unknown, t: TFunction): DryRunSource {
if (!isRecord(value)) {
throw new Error(t("models.row.import.errors.sourceRequired"));
}
return {
tenantId: parseString(value.tenantId, "source.tenantId", t),
sourceKey: parseString(value.sourceKey, "source.sourceKey", t),
fileName: parseOptionalString(value.fileName),
contentType: parseOptionalString(value.contentType),
requestId: parseOptionalString(value.requestId),
};
}
function parseMappings(value: unknown, t: TFunction): ColumnMapping[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error(t("models.row.import.errors.mappingsRequired"));
}
if (value.length > MAX_MAPPINGS) {
throw new Error(t("models.row.import.errors.maxMappings", { max: MAX_MAPPINGS }));
}
return value.map((item, index) => {
if (!isRecord(item)) {
throw new Error(t("models.row.import.errors.mappingMustBeObject", { index }));
}
return {
sourceColumn: parseString(item.sourceColumn, `mappings[${index}].sourceColumn`, t),
targetField: parseString(item.targetField, `mappings[${index}].targetField`, t),
required: item.required === true,
label: parseOptionalString(item.label) ?? undefined,
};
});
}
function parseFields(value: unknown, t: TFunction): ImportDryRunFieldDefinition[] {
if (value === undefined) {
return [];
}
if (!Array.isArray(value)) {
throw new Error(t("models.row.import.errors.fieldsMustBeArray"));
}
if (value.length > MAX_FIELDS) {
throw new Error(t("models.row.import.errors.maxFields", { max: MAX_FIELDS }));
}
return value.map((item, index) => {
if (!isRecord(item)) {
throw new Error(t("models.row.import.errors.fieldMustBeObject", { index }));
}
return {
targetField: parseString(item.targetField, `fields[${index}].targetField`, t),
required: item.required === true,
label: parseOptionalString(item.label) ?? undefined,
};
});
}
function parseRows(value: unknown, t: TFunction): ImportPreviewInputRow[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error(t("models.row.import.errors.rowsRequired"));
}
if (value.length > MAX_PREVIEW_ROWS) {
throw new Error(t("models.row.import.errors.maxRows", { max: MAX_PREVIEW_ROWS }));
}
return value.map((item, index) => {
if (!isRecord(item) || !isRecord(item.values)) {
throw new Error(t("models.row.import.errors.rowValuesMustBeObject", { index }));
}
const values: NormalizedImportRow = {};
for (const [key, rowValue] of Object.entries(item.values)) {
if (!isNormalizedRowValue(rowValue)) {
throw new Error(t("models.row.import.errors.rowValueMustBeScalar", { index, field: key }));
}
values[key] = rowValue;
}
const rowNumber = typeof item.rowNumber === "number" && Number.isInteger(item.rowNumber) && item.rowNumber > 0 ? item.rowNumber : undefined;
return { rowNumber, values };
});
}
function validationDiagnostic(message: string): ParserDiagnostic[] {
return [createRedactedParserDiagnostic("invalid_dry_run_request", message)];
}
function buildPreviewInput(body: DryRunBody, t: TFunction): ImportDryRunPreviewInput & { source: DryRunSource; importType: string } {
const format = parseString(body.format, "format", t);
if (!isSupportedImportFormat(format)) {
throw new Error(t("models.row.import.errors.unsupportedFormat"));
}
return {
format,
importType: parseString(body.importType, "importType", t),
source: parseSource(body.source, t),
mappings: parseMappings(body.mappings, t),
fields: parseFields(body.fields, t),
rows: parseRows(body.rows, t),
};
}
function getImportPermission(importType: string, t: TFunction): DefaultPermission {
switch (importType) {
case "members":
return "app.settings.members.create";
default:
throw new Error(t("models.row.import.errors.unsupportedType", { importType }));
}
}
function getCreatedJobId(createdJob: unknown, t: TFunction): string {
if (isRecord(createdJob) && typeof createdJob.id === "string" && createdJob.id.length > 0) {
return createdJob.id;
}
throw new Error(t("models.row.import.errors.importJobCreateFailed"));
}
function toErrorMessage(error: unknown, t: TFunction): string {
return error instanceof Error ? error.message : t("models.row.import.errors.dryRunFailed");
}
function sanitizeDiagnosticMessage(message: string): string {
return redactDiagnosticMessage(message)
.replace(/authorization:\s*\[REDACTED\]\s+[^\s,;]+/gi, "authorization: [REDACTED]")
.replace(/Bearer\s+[^\s,;]+/gi, "Bearer [REDACTED]")
.replace(/rawBody\s*=\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)/gi, "rawBody=[REDACTED]");
}
async function markFailureIfPossible(
jobId: string | undefined,
tenantId: string,
attemptsMade: number,
error: unknown,
t: TFunction,
counters?: { rowCount?: number; acceptedRows?: number; rejectedRows?: number }
) {
if (!jobId) {
return;
}
await markImportJobFailed({
id: jobId,
tenantId,
attemptsMade,
rowCount: counters?.rowCount,
acceptedRows: counters?.acceptedRows,
rejectedRows: counters?.rejectedRows,
errorName: error instanceof Error ? error.name : "Error",
errorMessage: sanitizeDiagnosticMessage(toErrorMessage(error, t)),
});
}
export const action = async ({ request, params }: ActionFunctionArgs) => {
const { t } = await getTranslations(request);
if (request.method !== "POST") {
return methodNotAllowed(t);
}
let userInfo: Awaited<ReturnType<typeof requireAuth>>;
try {
userInfo = await requireAuth({ request, params });
} catch (error) {
throw error;
}
let body: DryRunBody;
try {
await validateCSRFToken(request);
const parsed = await request.json();
if (!isRecord(parsed)) {
return invalidInput(t("models.row.import.errors.jsonBodyMustBeObject"));
}
body = parsed;
} catch {
return invalidInput(t("models.row.import.errors.invalidJsonBody"));
}
let previewInput: ImportDryRunPreviewInput & { source: DryRunSource; importType: string };
try {
previewInput = buildPreviewInput(body, t);
} catch (error) {
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
return invalidInput(message, validationDiagnostic(message));
}
let permission: DefaultPermission;
try {
permission = getImportPermission(previewInput.importType, t);
} catch (error) {
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
return invalidInput(message, validationDiagnostic(message));
}
await verifyUserHasPermission(request, permission, previewInput.source.tenantId);
const accessDecision = authorizeStorageObjectAccess({
tenantId: previewInput.source.tenantId,
key: previewInput.source.sourceKey,
operation: "download",
});
if (!accessDecision.allowed) {
return storageAccessDenied(t, accessDecision.reason);
}
const requestId = previewInput.source.requestId ?? crypto.randomUUID();
let jobId: string | undefined;
const attemptsMade = 1;
try {
const createdJob = await createImportJob({
tenantId: previewInput.source.tenantId,
requestedByUserId: userInfo.userId ?? null,
importType: previewInput.importType,
format: previewInput.format,
requestId,
sourceBucket: DRY_RUN_SOURCE_BUCKET,
sourceKey: accessDecision.key,
fileName: previewInput.source.fileName,
contentType: previewInput.source.contentType,
attemptsMade: 0,
});
jobId = getCreatedJobId(createdJob, t);
await markImportJobRunning({ id: jobId, tenantId: previewInput.source.tenantId, attemptsMade });
const preview = createImportDryRunPreview(previewInput);
await markImportJobSucceeded({
id: jobId,
tenantId: previewInput.source.tenantId,
rowCount: preview.summary.totalRows,
acceptedRows: preview.summary.acceptedRows,
rejectedRows: preview.summary.rejectedRows,
attemptsMade,
});
return json({
jobId,
status: "COMPLETED",
summary: preview.summary,
rows: preview.rows,
diagnostics: preview.summary.diagnostics,
});
} catch (error) {
try {
await markFailureIfPossible(jobId, previewInput.source.tenantId, attemptsMade, error, t);
} catch {
// Preserve the original dry-run failure while avoiding diagnostic leakage from persistence failures.
}
return json(
{
error: t("models.row.import.errors.dryRunFailed"),
diagnostics: [createRedactedParserDiagnostic("dry_run_failed", t("models.row.import.errors.dryRunFailedBeforeCompletion"))],
},
500
);
}
};