Track HiveGrid Cloud planning reference

This commit is contained in:
david committed 2026-06-26 20:48:01 +02:00
1 parent b3d5ea6764
commit e341c5938a
5784 files changed
+1104929 -3

No files matched your search

@@ -0,0 +1,290 @@
import type { ActionFunctionArgs } from "react-router";
import { beforeEach, describe, expect, it, vi } from "vitest";
const globalMockStoreKey = "__importsDryRunRouteTestMocks__";
const globalMockStore = globalThis as typeof globalThis & {
__importsDryRunRouteTestMocks__?: {
requireAuth: ReturnType<typeof vi.fn>;
createImportJob: ReturnType<typeof vi.fn>;
markImportJobRunning: ReturnType<typeof vi.fn>;
markImportJobSucceeded: ReturnType<typeof vi.fn>;
markImportJobFailed: ReturnType<typeof vi.fn>;
createImportDryRunPreview: ReturnType<typeof vi.fn>;
verifyUserHasPermission: ReturnType<typeof vi.fn>;
};
};
globalMockStore.__importsDryRunRouteTestMocks__ ??= {
requireAuth: vi.fn(),
createImportJob: vi.fn(),
markImportJobRunning: vi.fn(),
markImportJobSucceeded: vi.fn(),
markImportJobFailed: vi.fn(),
createImportDryRunPreview: vi.fn(),
verifyUserHasPermission: vi.fn(),
};
const mocks = globalMockStore[globalMockStoreKey];
const translations: Record<string, string> = {
"models.row.import.errors.dryRunFailed": "Import dry-run failed.",
"models.row.import.errors.dryRunFailedBeforeCompletion": "The import dry-run failed before completion.",
"models.row.import.errors.sourceStorageAccessDenied": "Import source storage access denied.",
"models.row.import.errors.sourceRequired": "Source is required.",
"models.row.import.errors.mappingsRequired": "Mappings must include at least one field mapping.",
"models.row.import.errors.methodNotAllowed": "Method not allowed.",
"models.row.import.errors.unsupportedFormat": "Unsupported import format.",
"models.row.import.errors.unsupportedType": "Unsupported import type: {{importType}}",
};
vi.mock("~/utils/loaders.middleware", () => ({
requireAuth: mocks.requireAuth,
}));
vi.mock("~/locale/i18next.server", () => ({
getTranslations: vi.fn(async () => ({
locale: "en",
translations: {},
t: (key: string, options?: Record<string, unknown>) => (translations[key] ?? key).replace("{{importType}}", String(options?.importType ?? "")),
})),
}));
vi.mock("~/utils/helpers/.server/PermissionsService", () => ({
verifyUserHasPermission: mocks.verifyUserHasPermission,
}));
vi.mock("~/utils/session.server", () => ({
validateCSRFToken: vi.fn(),
}));
vi.mock("@toolctrl/db/import-jobs.server", () => ({
createImportJob: mocks.createImportJob,
markImportJobRunning: mocks.markImportJobRunning,
markImportJobSucceeded: mocks.markImportJobSucceeded,
markImportJobFailed: mocks.markImportJobFailed,
}));
vi.mock("@toolctrl/imports", () => ({
createImportDryRunPreview: mocks.createImportDryRunPreview,
createRedactedParserDiagnostic: (code: string, message: string, options: Record<string, unknown> = {}) => ({
severity: "error",
...options,
code,
message: message.replace(
/(?:api[_-]?key|authorization|bearer|password|secret|token|rawBody)\s*[:=]\s*[^\s,;]+/gi,
(match) => `${match.split(/[:=]/, 1)[0]}: [REDACTED]`
),
}),
isSupportedImportFormat: (value: string) => ["csv", "json", "xlsx"].includes(value),
redactDiagnosticMessage: (message: string) =>
message.replace(
/(?:api[_-]?key|authorization|bearer|password|secret|token|rawBody)\s*[:=]\s*[^\s,;]+/gi,
(match) => `${match.split(/[:=]/, 1)[0]}: [REDACTED]`
),
}));
const validBody = {
format: "csv",
importType: "members",
source: {
tenantId: "tenant_1",
sourceKey: "tenants/tenant_1/uploads/import.csv",
fileName: "import.csv",
contentType: "text/csv",
requestId: "request_1",
},
fields: [{ targetField: "email", required: true, label: "Email" }],
mappings: [{ sourceColumn: "Email", targetField: "email", required: true, label: "Email" }],
rows: [{ rowNumber: 1, values: { Email: "ada@example.com" } }],
};
const preview = {
rows: [{ ok: true, rowNumber: 1, normalized: { email: "ada@example.com" }, warnings: undefined }],
mappedRows: [{ rowNumber: 1, normalized: { email: "ada@example.com" }, sourceColumns: ["Email"] }],
summary: { format: "csv", totalRows: 1, acceptedRows: 1, rejectedRows: 0, diagnostics: [] },
};
function makeRequest(method: string, body?: unknown) {
return new Request("https://example.com/internal/imports/dry-run", {
method,
body: body === undefined || method === "GET" || method === "HEAD" ? undefined : JSON.stringify(body),
headers: body === undefined ? undefined : { "Content-Type": "application/json", Authorization: "Bearer secret-token" },
});
}
async function callAction(method = "POST", body: unknown = validBody) {
const { action } = await import("./dry-run");
return action({ request: makeRequest(method, body), params: {}, context: {} } as unknown as ActionFunctionArgs) as Promise<Response>;
}
describe("/internal/imports/dry-run action", () => {
beforeEach(() => {
vi.clearAllMocks();
vi.resetModules();
mocks.requireAuth.mockResolvedValue({ userId: "user_1" });
mocks.createImportJob.mockResolvedValue({ id: "job_1" });
mocks.markImportJobRunning.mockResolvedValue(undefined);
mocks.markImportJobSucceeded.mockResolvedValue(undefined);
mocks.markImportJobFailed.mockResolvedValue(undefined);
mocks.createImportDryRunPreview.mockReturnValue(preview);
mocks.verifyUserHasPermission.mockResolvedValue(true);
});
it("authenticates, builds a dry-run preview, persists scalar counters, and returns the redacted response shape", async () => {
const response = await callAction();
expect(response.status).toBe(200);
await expect(response.json()).resolves.toEqual({
jobId: "job_1",
status: "COMPLETED",
summary: preview.summary,
rows: preview.rows,
diagnostics: [],
});
expect(mocks.requireAuth).toHaveBeenCalledWith({ request: expect.any(Request), params: {} });
expect(mocks.verifyUserHasPermission).toHaveBeenCalledWith(expect.any(Request), "app.settings.members.create", "tenant_1");
expect(mocks.createImportJob).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_1",
requestedByUserId: "user_1",
importType: "members",
format: "csv",
requestId: "request_1",
sourceBucket: "import-dry-run",
sourceKey: "tenants/tenant_1/uploads/import.csv",
attemptsMade: 0,
})
);
expect(mocks.markImportJobRunning).toHaveBeenCalledWith({ id: "job_1", tenantId: "tenant_1", attemptsMade: 1 });
expect(mocks.createImportDryRunPreview).toHaveBeenCalledWith(
expect.objectContaining({
format: "csv",
fields: validBody.fields,
mappings: validBody.mappings,
rows: validBody.rows,
})
);
expect(mocks.markImportJobSucceeded).toHaveBeenCalledWith({
id: "job_1",
tenantId: "tenant_1",
rowCount: 1,
acceptedRows: 1,
rejectedRows: 0,
attemptsMade: 1,
});
expect(mocks.markImportJobFailed).not.toHaveBeenCalled();
});
it("returns localized validation messages for missing source and mappings", async () => {
const missingSource = await callAction("POST", { ...validBody, source: undefined });
await expect(missingSource.json()).resolves.toMatchObject({ error: "Source is required." });
const missingMappings = await callAction("POST", { ...validBody, mappings: [] });
await expect(missingMappings.json()).resolves.toMatchObject({ error: "Mappings must include at least one field mapping." });
});
it("rejects non-POST requests before auth or persistence", async () => {
const response = await callAction("GET", undefined);
expect(response.status).toBe(405);
await expect(response.json()).resolves.toEqual({ error: "Method not allowed." });
expect(mocks.requireAuth).not.toHaveBeenCalled();
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
it("rejects unsupported formats and redacts secret-bearing invalid payload details", async () => {
const response = await callAction("POST", { ...validBody, format: "xml", apiKey: "secret-token" });
expect(response.status).toBe(400);
const payload = await response.json();
expect(payload.error).toBe("Unsupported import format.");
expect(payload.diagnostics[0]).toMatchObject({ code: "invalid_dry_run_request", message: "Unsupported import format." });
expect(JSON.stringify(payload)).not.toContain("secret-token");
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
it("rejects missing mappings or rows without checking permissions or creating a job", async () => {
const response = await callAction("POST", { ...validBody, mappings: [], rows: [] });
expect(response.status).toBe(400);
await expect(response.json()).resolves.toMatchObject({
error: "Mappings must include at least one field mapping.",
diagnostics: [expect.objectContaining({ code: "invalid_dry_run_request" })],
});
expect(mocks.verifyUserHasPermission).not.toHaveBeenCalled();
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
it("propagates permission denials before persistence, preview generation, or authorization leakage", async () => {
const denial = new Response(null, { status: 302, headers: { Location: "/unauthorized/app.settings.members.create/tenant_2" } });
mocks.verifyUserHasPermission.mockRejectedValueOnce(denial);
await expect(callAction("POST", { ...validBody, importType: "members", source: { ...validBody.source, tenantId: "tenant_2" } })).rejects.toBe(denial);
expect(mocks.verifyUserHasPermission).toHaveBeenCalledWith(expect.any(Request), "app.settings.members.create", "tenant_2");
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.markImportJobRunning).not.toHaveBeenCalled();
expect(mocks.createImportDryRunPreview).not.toHaveBeenCalled();
expect(mocks.markImportJobSucceeded).not.toHaveBeenCalled();
expect(mocks.markImportJobFailed).not.toHaveBeenCalled();
expect(JSON.stringify(denial.headers)).not.toContain("secret-token");
});
it("rejects dry-run source keys outside the requested tenant prefix", async () => {
const response = await callAction("POST", {
...validBody,
source: { ...validBody.source, sourceKey: "tenants/tenant_2/uploads/import.csv" },
});
expect(response.status).toBe(403);
await expect(response.json()).resolves.toEqual({
error: "Import source storage access denied.",
reason: "tenant_mismatch",
});
expect(mocks.verifyUserHasPermission).toHaveBeenCalledWith(expect.any(Request), "app.settings.members.create", "tenant_1");
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.createImportDryRunPreview).not.toHaveBeenCalled();
});
it("rejects unsupported import types before permissions or persistence", async () => {
const response = await callAction("POST", { ...validBody, importType: "widgets" });
expect(response.status).toBe(400);
await expect(response.json()).resolves.toMatchObject({ error: "Unsupported import type: widgets" });
expect(mocks.verifyUserHasPermission).not.toHaveBeenCalled();
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
it("redacts helper failures, marks the import job failed, and does not leak raw body or tokens", async () => {
mocks.createImportDryRunPreview.mockImplementationOnce(() => {
throw new Error('Parser failed authorization: Bearer secret-token rawBody={"email":"ada@example.com"}');
});
const response = await callAction("POST", validBody);
expect(response.status).toBe(500);
const payload = await response.json();
expect(payload.error).toBe("Import dry-run failed.");
expect(JSON.stringify(payload)).not.toContain("secret-token");
expect(JSON.stringify(payload)).not.toContain("ada@example.com");
expect(mocks.markImportJobFailed).toHaveBeenCalledWith(
expect.objectContaining({
id: "job_1",
attemptsMade: 1,
errorName: "Error",
errorMessage: expect.not.stringContaining("secret-token"),
})
);
});
it("returns a redacted server error when DB helpers fail", async () => {
mocks.markImportJobRunning.mockRejectedValueOnce(new Error("database token=secret-token is unavailable"));
const response = await callAction();
expect(response.status).toBe(500);
const payload = await response.json();
expect(payload).toMatchObject({ error: "Import dry-run failed.", diagnostics: [expect.objectContaining({ code: "dry_run_failed" })] });
expect(JSON.stringify(payload)).not.toContain("secret-token");
expect(mocks.markImportJobSucceeded).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,355 @@
import type { ActionFunctionArgs } from "react-router";
import {
createImportDryRunPreview,
createRedactedParserDiagnostic,
isSupportedImportFormat,
redactDiagnosticMessage,
type ColumnMapping,
type ImportDryRunFieldDefinition,
type ImportDryRunPreviewInput,
type ImportPreviewInputRow,
type NormalizedImportRow,
type NormalizedRowValue,
type ParserDiagnostic,
} from "@toolctrl/imports";
import { createImportJob, markImportJobFailed, markImportJobRunning, markImportJobSucceeded } from "@toolctrl/db/import-jobs.server";
import { authorizeStorageObjectAccess } from "@toolctrl/storage";
import { requireAuth } from "~/utils/loaders.middleware";
import { verifyUserHasPermission } from "~/utils/helpers/.server/PermissionsService";
import type { DefaultPermission } from "~/application/dtos/shared/DefaultPermissions";
import { validateCSRFToken } from "~/utils/session.server";
import { getTranslations } from "~/locale/i18next.server";
import type { TFunction } from "i18next";
const MAX_PREVIEW_ROWS = 500;
const MAX_MAPPINGS = 200;
const MAX_FIELDS = 200;
const DRY_RUN_SOURCE_BUCKET = "import-dry-run";
type DryRunBody = {
format?: unknown;
importType?: unknown;
source?: unknown;
mappings?: unknown;
rows?: unknown;
fields?: unknown;
};
type DryRunSource = {
tenantId: string;
sourceKey: string;
fileName?: string | null;
contentType?: string | null;
requestId?: string | null;
};
function json(payload: unknown, status = 200) {
return Response.json(payload, { status });
}
function invalidInput(message: string, diagnostics?: readonly ParserDiagnostic[]) {
return json({ error: message, ...(diagnostics && diagnostics.length > 0 ? { diagnostics } : {}) }, 400);
}
function methodNotAllowed(t: TFunction) {
return json({ error: t("models.row.import.errors.methodNotAllowed") }, 405);
}
function storageAccessDenied(t: TFunction, reason: string) {
const status = reason === "invalid_tenant" || reason === "invalid_key" ? 400 : 403;
return json({ error: t("models.row.import.errors.sourceStorageAccessDenied"), reason }, status);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isNormalizedRowValue(value: unknown): value is NormalizedRowValue {
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
return true;
}
return Array.isArray(value) && value.every((item) => item === null || typeof item === "string" || typeof item === "number" || typeof item === "boolean");
}
function parseString(value: unknown, fieldName: string, t: TFunction): string {
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(t("models.row.import.errors.requiredField", { field: fieldName }));
}
return value.trim();
}
function parseOptionalString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function parseSource(value: unknown, t: TFunction): DryRunSource {
if (!isRecord(value)) {
throw new Error(t("models.row.import.errors.sourceRequired"));
}
return {
tenantId: parseString(value.tenantId, "source.tenantId", t),
sourceKey: parseString(value.sourceKey, "source.sourceKey", t),
fileName: parseOptionalString(value.fileName),
contentType: parseOptionalString(value.contentType),
requestId: parseOptionalString(value.requestId),
};
}
function parseMappings(value: unknown, t: TFunction): ColumnMapping[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error(t("models.row.import.errors.mappingsRequired"));
}
if (value.length > MAX_MAPPINGS) {
throw new Error(t("models.row.import.errors.maxMappings", { max: MAX_MAPPINGS }));
}
return value.map((item, index) => {
if (!isRecord(item)) {
throw new Error(t("models.row.import.errors.mappingMustBeObject", { index }));
}
return {
sourceColumn: parseString(item.sourceColumn, `mappings[${index}].sourceColumn`, t),
targetField: parseString(item.targetField, `mappings[${index}].targetField`, t),
required: item.required === true,
label: parseOptionalString(item.label) ?? undefined,
};
});
}
function parseFields(value: unknown, t: TFunction): ImportDryRunFieldDefinition[] {
if (value === undefined) {
return [];
}
if (!Array.isArray(value)) {
throw new Error(t("models.row.import.errors.fieldsMustBeArray"));
}
if (value.length > MAX_FIELDS) {
throw new Error(t("models.row.import.errors.maxFields", { max: MAX_FIELDS }));
}
return value.map((item, index) => {
if (!isRecord(item)) {
throw new Error(t("models.row.import.errors.fieldMustBeObject", { index }));
}
return {
targetField: parseString(item.targetField, `fields[${index}].targetField`, t),
required: item.required === true,
label: parseOptionalString(item.label) ?? undefined,
};
});
}
function parseRows(value: unknown, t: TFunction): ImportPreviewInputRow[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error(t("models.row.import.errors.rowsRequired"));
}
if (value.length > MAX_PREVIEW_ROWS) {
throw new Error(t("models.row.import.errors.maxRows", { max: MAX_PREVIEW_ROWS }));
}
return value.map((item, index) => {
if (!isRecord(item) || !isRecord(item.values)) {
throw new Error(t("models.row.import.errors.rowValuesMustBeObject", { index }));
}
const values: NormalizedImportRow = {};
for (const [key, rowValue] of Object.entries(item.values)) {
if (!isNormalizedRowValue(rowValue)) {
throw new Error(t("models.row.import.errors.rowValueMustBeScalar", { index, field: key }));
}
values[key] = rowValue;
}
const rowNumber = typeof item.rowNumber === "number" && Number.isInteger(item.rowNumber) && item.rowNumber > 0 ? item.rowNumber : undefined;
return { rowNumber, values };
});
}
function validationDiagnostic(message: string): ParserDiagnostic[] {
return [createRedactedParserDiagnostic("invalid_dry_run_request", message)];
}
function buildPreviewInput(body: DryRunBody, t: TFunction): ImportDryRunPreviewInput & { source: DryRunSource; importType: string } {
const format = parseString(body.format, "format", t);
if (!isSupportedImportFormat(format)) {
throw new Error(t("models.row.import.errors.unsupportedFormat"));
}
return {
format,
importType: parseString(body.importType, "importType", t),
source: parseSource(body.source, t),
mappings: parseMappings(body.mappings, t),
fields: parseFields(body.fields, t),
rows: parseRows(body.rows, t),
};
}
function getImportPermission(importType: string, t: TFunction): DefaultPermission {
switch (importType) {
case "members":
return "app.settings.members.create";
default:
throw new Error(t("models.row.import.errors.unsupportedType", { importType }));
}
}
function getCreatedJobId(createdJob: unknown, t: TFunction): string {
if (isRecord(createdJob) && typeof createdJob.id === "string" && createdJob.id.length > 0) {
return createdJob.id;
}
throw new Error(t("models.row.import.errors.importJobCreateFailed"));
}
function toErrorMessage(error: unknown, t: TFunction): string {
return error instanceof Error ? error.message : t("models.row.import.errors.dryRunFailed");
}
function sanitizeDiagnosticMessage(message: string): string {
return redactDiagnosticMessage(message)
.replace(/authorization:\s*\[REDACTED\]\s+[^\s,;]+/gi, "authorization: [REDACTED]")
.replace(/Bearer\s+[^\s,;]+/gi, "Bearer [REDACTED]")
.replace(/rawBody\s*=\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)/gi, "rawBody=[REDACTED]");
}
async function markFailureIfPossible(
jobId: string | undefined,
tenantId: string,
attemptsMade: number,
error: unknown,
t: TFunction,
counters?: { rowCount?: number; acceptedRows?: number; rejectedRows?: number }
) {
if (!jobId) {
return;
}
await markImportJobFailed({
id: jobId,
tenantId,
attemptsMade,
rowCount: counters?.rowCount,
acceptedRows: counters?.acceptedRows,
rejectedRows: counters?.rejectedRows,
errorName: error instanceof Error ? error.name : "Error",
errorMessage: sanitizeDiagnosticMessage(toErrorMessage(error, t)),
});
}
export const action = async ({ request, params }: ActionFunctionArgs) => {
const { t } = await getTranslations(request);
if (request.method !== "POST") {
return methodNotAllowed(t);
}
let userInfo: Awaited<ReturnType<typeof requireAuth>>;
try {
userInfo = await requireAuth({ request, params });
} catch (error) {
throw error;
}
let body: DryRunBody;
try {
await validateCSRFToken(request);
const parsed = await request.json();
if (!isRecord(parsed)) {
return invalidInput(t("models.row.import.errors.jsonBodyMustBeObject"));
}
body = parsed;
} catch {
return invalidInput(t("models.row.import.errors.invalidJsonBody"));
}
let previewInput: ImportDryRunPreviewInput & { source: DryRunSource; importType: string };
try {
previewInput = buildPreviewInput(body, t);
} catch (error) {
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
return invalidInput(message, validationDiagnostic(message));
}
let permission: DefaultPermission;
try {
permission = getImportPermission(previewInput.importType, t);
} catch (error) {
const message = sanitizeDiagnosticMessage(toErrorMessage(error, t));
return invalidInput(message, validationDiagnostic(message));
}
await verifyUserHasPermission(request, permission, previewInput.source.tenantId);
const accessDecision = authorizeStorageObjectAccess({
tenantId: previewInput.source.tenantId,
key: previewInput.source.sourceKey,
operation: "download",
});
if (!accessDecision.allowed) {
return storageAccessDenied(t, accessDecision.reason);
}
const requestId = previewInput.source.requestId ?? crypto.randomUUID();
let jobId: string | undefined;
const attemptsMade = 1;
try {
const createdJob = await createImportJob({
tenantId: previewInput.source.tenantId,
requestedByUserId: userInfo.userId ?? null,
importType: previewInput.importType,
format: previewInput.format,
requestId,
sourceBucket: DRY_RUN_SOURCE_BUCKET,
sourceKey: accessDecision.key,
fileName: previewInput.source.fileName,
contentType: previewInput.source.contentType,
attemptsMade: 0,
});
jobId = getCreatedJobId(createdJob, t);
await markImportJobRunning({ id: jobId, tenantId: previewInput.source.tenantId, attemptsMade });
const preview = createImportDryRunPreview(previewInput);
await markImportJobSucceeded({
id: jobId,
tenantId: previewInput.source.tenantId,
rowCount: preview.summary.totalRows,
acceptedRows: preview.summary.acceptedRows,
rejectedRows: preview.summary.rejectedRows,
attemptsMade,
});
return json({
jobId,
status: "COMPLETED",
summary: preview.summary,
rows: preview.rows,
diagnostics: preview.summary.diagnostics,
});
} catch (error) {
try {
await markFailureIfPossible(jobId, previewInput.source.tenantId, attemptsMade, error, t);
} catch {
// Preserve the original dry-run failure while avoiding diagnostic leakage from persistence failures.
}
return json(
{
error: t("models.row.import.errors.dryRunFailed"),
diagnostics: [createRedactedParserDiagnostic("dry_run_failed", t("models.row.import.errors.dryRunFailedBeforeCompletion"))],
},
500
);
}
};