356 lines
12 KiB
TypeScript
356 lines
12 KiB
TypeScript
import type { ActionFunctionArgs } from "react-router";
|
|
import {
|
|
createImportDryRunPreview,
|
|
createRedactedParserDiagnostic,
|
|
isSupportedImportFormat,
|
|
redactDiagnosticMessage,
|
|
type ColumnMapping,
|
|
type ImportDryRunFieldDefinition,
|
|
type ImportDryRunPreviewInput,
|
|
type ImportPreviewInputRow,
|
|
type NormalizedImportRow,
|
|
type NormalizedRowValue,
|
|
type ParserDiagnostic,
|
|
} from "@toolctrl/imports";
|
|
import { createImportJob, markImportJobFailed, markImportJobRunning, markImportJobSucceeded } from "@toolctrl/db/import-jobs.server";
|
|
import { authorizeStorageObjectAccess } from "@toolctrl/storage";
|
|
import { requireAuth } from "~/utils/loaders.middleware";
|
|
import { verifyUserHasPermission } from "~/utils/helpers/.server/PermissionsService";
|
|
import type { DefaultPermission } from "~/application/dtos/shared/DefaultPermissions";
|
|
import { validateCSRFToken } from "~/utils/session.server";
|
|
import { getTranslations } from "~/locale/i18next.server";
|
|
import type { TFunction } from "i18next";
|
|
|
|
const MAX_PREVIEW_ROWS = 500;
|
|
const MAX_MAPPINGS = 200;
|
|
const MAX_FIELDS = 200;
|
|
const DRY_RUN_SOURCE_BUCKET = "import-dry-run";
|
|
|
|
type DryRunBody = {
|
|
format?: unknown;
|
|
importType?: unknown;
|
|
source?: unknown;
|
|
mappings?: unknown;
|
|
rows?: unknown;
|
|
fields?: unknown;
|
|
};
|
|
|
|
type DryRunSource = {
|
|
tenantId: string;
|
|
sourceKey: string;
|
|
fileName?: string | null;
|
|
contentType?: string | null;
|
|
requestId?: string | null;
|
|
};
|
|
|
|
function json(payload: unknown, status = 200) {
|
|
return Response.json(payload, { status });
|
|
}
|
|
|
|
function invalidInput(message: string, diagnostics?: readonly ParserDiagnostic[]) {
|
|
return json({ error: message, ...(diagnostics && diagnostics.length > 0 ? { diagnostics } : {}) }, 400);
|
|
}
|
|
|
|
function methodNotAllowed(t: TFunction) {
|
|
return json({ error: t("models.row.import.errors.methodNotAllowed") }, 405);
|
|
}
|
|
|
|
function storageAccessDenied(t: TFunction, reason: string) {
|
|
const status = reason === "invalid_tenant" || reason === "invalid_key" ? 400 : 403;
|
|
return json({ error: t("models.row.import.errors.sourceStorageAccessDenied"), reason }, status);
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function isNormalizedRowValue(value: unknown): value is NormalizedRowValue {
|
|
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
return true;
|
|
}
|
|
|
|
return Array.isArray(value) && value.every((item) => item === null || typeof item === "string" || typeof item === "number" || typeof item === "boolean");
|
|
}
|
|
|
|
function parseString(value: unknown, fieldName: string, t: TFunction): string {
|
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
throw new Error(t("models.row.import.errors.requiredField", { field: fieldName }));
|
|
}
|
|
return value.trim();
|
|
}
|
|
|
|
function parseOptionalString(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function parseSource(value: unknown, t: TFunction): DryRunSource {
|
|
if (!isRecord(value)) {
|
|
throw new Error(t("models.row.import.errors.sourceRequired"));
|
|
}
|
|
|
|
return {
|
|
tenantId: parseString(value.tenantId, "source.tenantId", t),
|
|
sourceKey: parseString(value.sourceKey, "source.sourceKey", t),
|
|
fileName: parseOptionalString(value.fileName),
|
|
contentType: parseOptionalString(value.contentType),
|
|
requestId: parseOptionalString(value.requestId),
|
|
};
|
|
}
|
|
|
|
function parseMappings(value: unknown, t: TFunction): ColumnMapping[] {
|
|
if (!Array.isArray(value) || value.length === 0) {
|
|
throw new Error(t("models.row.import.errors.mappingsRequired"));
|
|
}
|
|
|
|
if (value.length > MAX_MAPPINGS) {
|
|
throw new Error(t("models.row.import.errors.maxMappings", { max: MAX_MAPPINGS }));
|
|
}
|
|
|
|
return value.map((item, index) => {
|
|
if (!isRecord(item)) {
|
|
throw new Error(t("models.row.import.errors.mappingMustBeObject", { index }));
|
|
}
|
|
|
|
return {
|
|
sourceColumn: parseString(item.sourceColumn, `mappings[${index}].sourceColumn`, t),
|
|
targetField: parseString(item.targetField, `mappings[${index}].targetField`, t),
|
|
required: item.required === true,
|
|
label: parseOptionalString(item.label) ?? undefined,
|
|
};
|
|
});
|
|
}
|
|
|
|
function parseFields(value: unknown, t: TFunction): ImportDryRunFieldDefinition[] {
|
|
if (value === undefined) {
|
|
return [];
|
|
}
|
|
|
|
if (!Array.isArray(value)) {
|
|
throw new Error(t("models.row.import.errors.fieldsMustBeArray"));
|
|
}
|
|
|
|
if (value.length > MAX_FIELDS) {
|
|
throw new Error(t("models.row.import.errors.maxFields", { max: MAX_FIELDS }));
|
|
}
|
|
|
|
return value.map((item, index) => {
|
|
if (!isRecord(item)) {
|
|
throw new Error(t("models.row.import.errors.fieldMustBeObject", { index }));
|
|
}
|
|
|
|
return {
|
|
targetField: parseString(item.targetField, `fields[${index}].targetField`, t),
|
|
required: item.required === true,
|
|
label: parseOptionalString(item.label) ?? undefined,
|
|
};
|
|
});
|
|
}
|
|
|
|
function parseRows(value: unknown, t: TFunction): ImportPreviewInputRow[] {
|
|
if (!Array.isArray(value) || value.length === 0) {
|
|
throw new Error(t("models.row.import.errors.rowsRequired"));
|
|
}
|
|
|
|
if (value.length > MAX_PREVIEW_ROWS) {
|
|
throw new Error(t("models.row.import.errors.maxRows", { max: MAX_PREVIEW_ROWS }));
|
|
}
|
|
|
|
return value.map((item, index) => {
|
|
if (!isRecord(item) || !isRecord(item.values)) {
|
|
throw new Error(t("models.row.import.errors.rowValuesMustBeObject", { index }));
|
|
}
|
|
|
|
const values: NormalizedImportRow = {};
|
|
for (const [key, rowValue] of Object.entries(item.values)) {
|
|
if (!isNormalizedRowValue(rowValue)) {
|
|
throw new Error(t("models.row.import.errors.rowValueMustBeScalar", { index, field: key }));
|
|
}
|
|
values[key] = rowValue;
|
|
}
|
|
|
|
const rowNumber = typeof item.rowNumber === "number" && Number.isInteger(item.rowNumber) && item.rowNumber > 0 ? item.rowNumber : undefined;
|
|
return { rowNumber, values };
|
|
});
|
|
}
|
|
|
|
function validationDiagnostic(message: string): ParserDiagnostic[] {
|
|
return [createRedactedParserDiagnostic("invalid_dry_run_request", message)];
|
|
}
|
|
|
|
function buildPreviewInput(body: DryRunBody, t: TFunction): ImportDryRunPreviewInput & { source: DryRunSource; importType: string } {
|
|
const format = parseString(body.format, "format", t);
|
|
if (!isSupportedImportFormat(format)) {
|
|
throw new Error(t("models.row.import.errors.unsupportedFormat"));
|
|
}
|
|
|
|
return {
|
|
format,
|
|
importType: parseString(body.importType, "importType", t),
|
|
source: parseSource(body.source, t),
|
|
mappings: parseMappings(body.mappings, t),
|
|
fields: parseFields(body.fields, t),
|
|
rows: parseRows(body.rows, t),
|
|
};
|
|
}
|
|
|
|
function getImportPermission(importType: string, t: TFunction): DefaultPermission {
|
|
switch (importType) {
|
|
case "members":
|
|
return "app.settings.members.create";
|
|
default:
|
|
throw new Error(t("models.row.import.errors.unsupportedType", { importType }));
|
|
}
|
|
}
|
|
|
|
function getCreatedJobId(createdJob: unknown, t: TFunction): string {
|
|
if (isRecord(createdJob) && typeof createdJob.id === "string" && createdJob.id.length > 0) {
|
|
return createdJob.id;
|
|
}
|
|
|
|
throw new Error(t("models.row.import.errors.importJobCreateFailed"));
|
|
}
|
|
|
|
function toErrorMessage(error: unknown, t: TFunction): string {
|
|
return error instanceof Error ? error.message : t("models.row.import.errors.dryRunFailed");
|
|
}
|
|
|
|
function sanitizeDiagnosticMessage(message: string): string {
|
|
return redactDiagnosticMessage(message)
|
|
.replace(/authorization:\s*\[REDACTED\]\s+[^\s,;]+/gi, "authorization: [REDACTED]")
|
|
.replace(/Bearer\s+[^\s,;]+/gi, "Bearer [REDACTED]")
|
|
.replace(/rawBody\s*=\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)/gi, "rawBody=[REDACTED]");
|
|
}
|
|
|
|
async function markFailureIfPossible(
|
|
jobId: string | undefined,
|
|
tenantId: string,
|
|
attemptsMade: number,
|
|
error: unknown,
|
|
t: TFunction,
|
|
counters?: { rowCount?: number; acceptedRows?: number; rejectedRows?: number }
|
|
) {
|
|
if (!jobId) {
|
|
return;
|
|
}
|
|
|
|
await markImportJobFailed({
|
|
id: jobId,
|
|
tenantId,
|
|
attemptsMade,
|
|
rowCount: counters?.rowCount,
|
|
acceptedRows: counters?.acceptedRows,
|
|
rejectedRows: counters?.rejectedRows,
|
|
errorName: error instanceof Error ? error.name : "Error",
|
|
errorMessage: sanitizeDiagnosticMessage(toErrorMessage(error, t)),
|
|
});
|
|
}
|
|
|
|
export const action = async ({ request, params }: ActionFunctionArgs) => {
|
|
const { t } = await getTranslations(request);
|
|
|
|
if (request.method !== "POST") {
|
|
return methodNotAllowed(t);
|
|
}
|
|
|
|
let userInfo: Awaited<ReturnType<typeof requireAuth>>;
|
|
try {
|
|
userInfo = await requireAuth({ request, params });
|
|
} catch (error) {
|
|
throw error;
|
|
}
|
|
|
|
let body: DryRunBody;
|
|
try {
|
|
await validateCSRFToken(request);
|
|
const parsed = await request.json();
|
|
if (!isRecord(parsed)) {
|
|
return invalidInput(t("models.row.import.errors.jsonBodyMustBeObject"));
|
|
}
|
|
body = parsed;
|
|
} catch {
|
|
return invalidInput(t("models.row.import.errors.invalidJsonBody"));
|
|
}
|
|
|
|
let previewInput: ImportDryRunPreviewInput & { source: DryRunSource; importType: string };
|
|
try {
|
|
previewInput = buildPreviewInput(body, t);
|
|
} catch (error) {
|
|
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
|
|
return invalidInput(message, validationDiagnostic(message));
|
|
}
|
|
|
|
let permission: DefaultPermission;
|
|
try {
|
|
permission = getImportPermission(previewInput.importType, t);
|
|
} catch (error) {
|
|
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
|
|
return invalidInput(message, validationDiagnostic(message));
|
|
}
|
|
|
|
await verifyUserHasPermission(request, permission, previewInput.source.tenantId);
|
|
|
|
const accessDecision = authorizeStorageObjectAccess({
|
|
tenantId: previewInput.source.tenantId,
|
|
key: previewInput.source.sourceKey,
|
|
operation: "download",
|
|
});
|
|
|
|
if (!accessDecision.allowed) {
|
|
return storageAccessDenied(t, accessDecision.reason);
|
|
}
|
|
|
|
const requestId = previewInput.source.requestId ?? crypto.randomUUID();
|
|
let jobId: string | undefined;
|
|
const attemptsMade = 1;
|
|
|
|
try {
|
|
const createdJob = await createImportJob({
|
|
tenantId: previewInput.source.tenantId,
|
|
requestedByUserId: userInfo.userId ?? null,
|
|
importType: previewInput.importType,
|
|
format: previewInput.format,
|
|
requestId,
|
|
sourceBucket: DRY_RUN_SOURCE_BUCKET,
|
|
sourceKey: accessDecision.key,
|
|
fileName: previewInput.source.fileName,
|
|
contentType: previewInput.source.contentType,
|
|
attemptsMade: 0,
|
|
});
|
|
jobId = getCreatedJobId(createdJob, t);
|
|
|
|
await markImportJobRunning({ id: jobId, tenantId: previewInput.source.tenantId, attemptsMade });
|
|
|
|
const preview = createImportDryRunPreview(previewInput);
|
|
await markImportJobSucceeded({
|
|
id: jobId,
|
|
tenantId: previewInput.source.tenantId,
|
|
rowCount: preview.summary.totalRows,
|
|
acceptedRows: preview.summary.acceptedRows,
|
|
rejectedRows: preview.summary.rejectedRows,
|
|
attemptsMade,
|
|
});
|
|
|
|
return json({
|
|
jobId,
|
|
status: "COMPLETED",
|
|
summary: preview.summary,
|
|
rows: preview.rows,
|
|
diagnostics: preview.summary.diagnostics,
|
|
});
|
|
} catch (error) {
|
|
try {
|
|
await markFailureIfPossible(jobId, previewInput.source.tenantId, attemptsMade, error, t);
|
|
} catch {
|
|
// Preserve the original dry-run failure while avoiding diagnostic leakage from persistence failures.
|
|
}
|
|
|
|
return json(
|
|
{
|
|
error: t("models.row.import.errors.dryRunFailed"),
|
|
diagnostics: [createRedactedParserDiagnostic("dry_run_failed", t("models.row.import.errors.dryRunFailedBeforeCompletion"))],
|
|
},
|
|
500
|
|
);
|
|
}
|
|
};
|