Track HiveGrid Cloud planning reference

This commit is contained in:
david committed 2026-06-26 20:48:01 +02:00
1 parent b3d5ea6764
commit e341c5938a
5784 files changed
+1104929 -3

No files matched your search

@@ -0,0 +1 @@
export { action } from "~/routes/internal/imports/dry-run";
@@ -0,0 +1,263 @@
import type { ActionFunctionArgs } from "react-router";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { ImportJobInspectionRecord } from "@toolctrl/db/import-jobs.server";
const globalMockStoreKey = "__importsSubmitRouteTestMocks__";
const globalMockStore = globalThis as typeof globalThis & {
__importsSubmitRouteTestMocks__?: {
requireAuth: ReturnType<typeof vi.fn>;
getTenantIdFromUrl: ReturnType<typeof vi.fn>;
createImportJob: ReturnType<typeof vi.fn>;
markImportJobFailed: ReturnType<typeof vi.fn>;
findImportJobForInspection: ReturnType<typeof vi.fn>;
verifyUserHasPermission: ReturnType<typeof vi.fn>;
createQueue: ReturnType<typeof vi.fn>;
};
};
globalMockStore.__importsSubmitRouteTestMocks__ ??= {
requireAuth: vi.fn(),
getTenantIdFromUrl: vi.fn(),
createImportJob: vi.fn(),
markImportJobFailed: vi.fn(),
findImportJobForInspection: vi.fn(),
verifyUserHasPermission: vi.fn(),
createQueue: vi.fn(),
};
const mocks = globalMockStore[globalMockStoreKey];
const originalEnv = { ...process.env };
const translations: Record<string, string> = {
"models.row.import.errors.dryRunJobNotFound": "Import preview job not found.",
"models.row.import.errors.dryRunMustBeCompletedBeforeSubmit": "Import dry-run must be completed with at least one accepted row before submit.",
"models.row.import.errors.queueEnqueueFailed": "Import queue enqueue failed.",
"models.row.import.errors.queueNotConfigured": "Import queue is not configured.",
"models.row.import.errors.sourceStorageAccessDenied": "Import source storage access denied.",
"models.row.import.errors.invalidMappingsShape": "Mappings must be an object or an array of mapping objects when provided.",
"models.row.import.errors.methodNotAllowed": "Method not allowed.",
"models.row.import.errors.unsupportedType": "Unsupported import type: {{importType}}",
};
vi.mock("~/utils/loaders.middleware", () => ({
requireAuth: mocks.requireAuth,
}));
vi.mock("~/locale/i18next.server", () => ({
getTranslations: vi.fn(async () => ({
locale: "en",
translations: {},
t: (key: string, options?: Record<string, unknown>) => (translations[key] ?? key).replace("{{importType}}", String(options?.importType ?? "")),
})),
}));
vi.mock("~/utils/services/.server/urlService", () => ({
getTenantIdFromUrl: mocks.getTenantIdFromUrl,
}));
vi.mock("~/utils/helpers/.server/PermissionsService", () => ({
verifyUserHasPermission: mocks.verifyUserHasPermission,
}));
vi.mock("~/utils/session.server", () => ({
validateCSRFToken: vi.fn(),
}));
vi.mock("@toolctrl/db/import-jobs.server", () => ({
createImportJob: mocks.createImportJob,
markImportJobFailed: mocks.markImportJobFailed,
findImportJobForInspection: mocks.findImportJobForInspection,
}));
vi.mock("@toolctrl/queue/runtime", () => ({
createQueue: mocks.createQueue,
}));
const completedDryRunJob: ImportJobInspectionRecord = {
id: "dry_run_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_original",
importType: "members",
format: "csv",
status: "COMPLETED",
requestId: "req_dry_run",
attemptsMade: 1,
sourceBucket: "import-dry-run",
sourceKey: "tenants/tenant_1/uploads/import.csv",
fileName: "import.csv",
contentType: "text/csv",
rowCount: 2,
acceptedRows: 2,
rejectedRows: 0,
errorName: null,
errorMessage: null,
errorReportBucket: null,
errorReportKey: null,
startedAt: new Date("2026-01-01T00:00:00.000Z"),
finishedAt: new Date("2026-01-01T00:00:01.000Z"),
};
function makeRequest(method: string, body?: unknown) {
return new Request("https://app.example.com/acme/internal/imports/submit", {
method,
body: body === undefined || method === "GET" || method === "HEAD" ? undefined : JSON.stringify(body),
headers: body === undefined ? undefined : { "Content-Type": "application/json", Authorization: "Bearer secret-token" },
});
}
async function callAction(method = "POST", body: unknown = { dryRunJobId: "dry_run_job_1" }) {
const { action } = await import("./submit");
return action({ request: makeRequest(method, body), params: { tenant: "acme" }, context: {} } as unknown as ActionFunctionArgs) as Promise<Response>;
}
describe("tenant import submit action", () => {
beforeEach(() => {
vi.clearAllMocks();
vi.resetModules();
process.env = { ...originalEnv, IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", BULLMQ_REDIS_URL: "redis://localhost:6379/0" };
mocks.requireAuth.mockResolvedValue({ userId: "user_1" });
mocks.getTenantIdFromUrl.mockResolvedValue("tenant_1");
mocks.findImportJobForInspection.mockResolvedValue(completedDryRunJob);
mocks.createImportJob.mockResolvedValue({ id: "import_job_queued" });
mocks.verifyUserHasPermission.mockResolvedValue(true);
mocks.createQueue.mockReturnValue({
add: vi.fn().mockResolvedValue({ id: "bull_job_1" }),
close: vi.fn().mockResolvedValue(undefined),
});
});
it("creates and enqueues a final import job from a completed dry-run", async () => {
const response = await callAction("POST", {
dryRunJobId: "dry_run_job_1",
mappings: [{ sourceColumn: "Email", targetField: "email" }],
});
expect(response.status).toBe(202);
await expect(response.json()).resolves.toEqual({ jobId: "import_job_queued", status: "QUEUED" });
expect(mocks.requireAuth).toHaveBeenCalledWith({ request: expect.any(Request), params: { tenant: "acme" } });
expect(mocks.getTenantIdFromUrl).toHaveBeenCalledWith({ tenant: "acme" });
expect(mocks.findImportJobForInspection).toHaveBeenCalledWith({ id: "dry_run_job_1", tenantId: "tenant_1" });
expect(mocks.verifyUserHasPermission).toHaveBeenCalledWith(expect.any(Request), "app.settings.members.create", "tenant_1");
expect(mocks.createImportJob).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_1",
requestedByUserId: "user_1",
importType: "members",
format: "csv",
sourceBucket: "import-dry-run",
sourceKey: "tenants/tenant_1/uploads/import.csv",
attemptsMade: 0,
})
);
const queue = mocks.createQueue.mock.results[0].value;
expect(queue.add).toHaveBeenCalledWith(
"run-import",
expect.objectContaining({
kind: "import.run",
importJobId: "import_job_queued",
tenantId: "tenant_1",
requestedByUserId: "user_1",
importType: "members",
format: "csv",
mappings: { Email: "email" },
metadata: { dryRunJobId: "dry_run_job_1", attemptsMade: 0 },
}),
expect.objectContaining({ jobId: "import_job_queued" })
);
expect(queue.close).toHaveBeenCalled();
});
it("rejects submit when Redis queue configuration is missing before creating jobs", async () => {
process.env = { ...originalEnv, IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", BULLMQ_REDIS_URL: "", REDIS_URL: "" };
const response = await callAction();
expect(response.status).toBe(503);
await expect(response.json()).resolves.toEqual({ error: "Import queue is not configured." });
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.createQueue).not.toHaveBeenCalled();
});
it("rejects submit when import queue processors are not declared ready", async () => {
process.env = { ...originalEnv, IMPORT_QUEUE_ENABLED: "true", BULLMQ_REDIS_URL: "redis://localhost:6379/0" };
const response = await callAction();
expect(response.status).toBe(503);
await expect(response.json()).resolves.toEqual({ error: "Import queue is not configured." });
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.createQueue).not.toHaveBeenCalled();
});
it("marks the created import job failed when enqueue fails", async () => {
const enqueueError = new Error("redis down");
mocks.createQueue.mockReturnValueOnce({
add: vi.fn().mockRejectedValue(enqueueError),
close: vi.fn().mockResolvedValue(undefined),
});
const response = await callAction();
expect(response.status).toBe(503);
await expect(response.json()).resolves.toEqual({ error: "Import queue enqueue failed." });
expect(mocks.markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_queued",
tenantId: "tenant_1",
attemptsMade: 0,
errorName: "Error",
errorMessage: "Import queue enqueue failed.",
});
expect(mocks.createQueue.mock.results[0].value.close).toHaveBeenCalled();
});
it("rejects dry-runs that are not successfully completed with accepted rows", async () => {
mocks.findImportJobForInspection.mockResolvedValueOnce({ ...completedDryRunJob, status: "FAILED", acceptedRows: 0 });
const response = await callAction();
expect(response.status).toBe(409);
await expect(response.json()).resolves.toEqual({ error: "Import dry-run must be completed with at least one accepted row before submit." });
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.createQueue).not.toHaveBeenCalled();
});
it("rejects completed dry-runs whose source key is outside the tenant prefix", async () => {
mocks.findImportJobForInspection.mockResolvedValueOnce({
...completedDryRunJob,
sourceKey: "tenants/tenant_2/uploads/import.csv",
});
const response = await callAction();
expect(response.status).toBe(403);
await expect(response.json()).resolves.toEqual({
error: "Import source storage access denied.",
reason: "tenant_mismatch",
});
expect(mocks.verifyUserHasPermission).toHaveBeenCalledWith(expect.any(Request), "app.settings.members.create", "tenant_1");
expect(mocks.createImportJob).not.toHaveBeenCalled();
expect(mocks.createQueue).not.toHaveBeenCalled();
});
it("returns not found for missing dry-run jobs without leaking submitted tokens", async () => {
mocks.findImportJobForInspection.mockResolvedValueOnce(null);
const response = await callAction("POST", { dryRunJobId: "missing_dry_run", tenantId: "tenant_other", token: "secret-token" });
expect(response.status).toBe(404);
const payload = await response.json();
expect(payload).toEqual({ error: "Import preview job not found." });
expect(JSON.stringify(payload)).not.toContain("secret-token");
expect(mocks.findImportJobForInspection).toHaveBeenCalledWith({ id: "missing_dry_run", tenantId: "tenant_1" });
expect(mocks.verifyUserHasPermission).not.toHaveBeenCalled();
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
it("rejects non-POST requests before auth or persistence", async () => {
const response = await callAction("GET", undefined);
expect(response.status).toBe(405);
await expect(response.json()).resolves.toEqual({ error: "Method not allowed." });
expect(mocks.requireAuth).not.toHaveBeenCalled();
expect(mocks.createImportJob).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,255 @@
import type { ActionFunctionArgs } from "react-router";
import { createImportJob, findImportJobForInspection, markImportJobFailed, type ImportJobInspectionRecord } from "@toolctrl/db/import-jobs.server";
import { buildImportJob, enqueueJob, queueNames, resolveRedisConnectionOptions, type RedisConnectionEnv, type TypedQueue } from "@toolctrl/queue";
import { authorizeStorageObjectAccess } from "@toolctrl/storage";
import { requireAuth } from "~/utils/loaders.middleware";
import { verifyUserHasPermission } from "~/utils/helpers/.server/PermissionsService";
import { getTenantIdFromUrl } from "~/utils/services/.server/urlService";
import type { DefaultPermission } from "~/application/dtos/shared/DefaultPermissions";
import { validateCSRFToken } from "~/utils/session.server";
import { getTranslations } from "~/locale/i18next.server";
import type { TFunction } from "i18next";
type SubmitBody = {
dryRunJobId?: unknown;
mappings?: unknown;
};
type ImportQueue = TypedQueue<typeof queueNames.imports>;
type ImportSubmitEnv = RedisConnectionEnv & { IMPORT_QUEUE_ENABLED?: string; IMPORT_QUEUE_RUNTIME_READY?: string };
function json(payload: unknown, status = 200) {
return Response.json(payload, { status });
}
function methodNotAllowed(t: TFunction) {
return json({ error: t("models.row.import.errors.methodNotAllowed") }, 405);
}
function invalidInput(message: string) {
return json({ error: message }, 400);
}
function storageAccessDenied(t: TFunction, reason: string) {
const status = reason === "invalid_tenant" || reason === "invalid_key" ? 400 : 403;
return json({ error: t("models.row.import.errors.sourceStorageAccessDenied"), reason }, status);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function parseRequiredString(value: unknown, fieldName: string, t: TFunction): string {
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(t("models.row.import.errors.requiredField", { field: fieldName }));
}
return value.trim();
}
function parseMappings(value: unknown, t: TFunction): Record<string, string> | undefined {
if (value === undefined || value === null) {
return undefined;
}
if (isRecord(value)) {
return Object.fromEntries(
Object.entries(value)
.filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[0].trim().length > 0 && entry[1].trim().length > 0)
.map(([sourceColumn, targetField]) => [sourceColumn.trim(), targetField.trim()])
);
}
if (Array.isArray(value)) {
return Object.fromEntries(
value.map((item, index) => {
if (!isRecord(item)) {
throw new Error(t("models.row.import.errors.mappingMustBeObject", { index }));
}
return [
parseRequiredString(item.sourceColumn, `mappings[${index}].sourceColumn`, t),
parseRequiredString(item.targetField, `mappings[${index}].targetField`, t),
];
})
);
}
throw new Error(t("models.row.import.errors.invalidMappingsShape"));
}
function getImportPermission(importType: string, t: TFunction): DefaultPermission {
switch (importType) {
case "members":
return "app.settings.members.create";
default:
throw new Error(t("models.row.import.errors.unsupportedType", { importType }));
}
}
function getCreatedJobId(createdJob: unknown, t: TFunction): string {
if (isRecord(createdJob) && typeof createdJob.id === "string" && createdJob.id.length > 0) {
return createdJob.id;
}
throw new Error(t("models.row.import.errors.importJobCreateFailed"));
}
function toQueueFormat(format: string): "csv" | "json" | "xlsx" | null {
return format === "csv" || format === "json" || format === "xlsx" ? format : null;
}
function canSubmitDryRun(job: ImportJobInspectionRecord): boolean {
return job.status === "COMPLETED" && job.acceptedRows > 0;
}
async function createInternalQueue(env?: ImportSubmitEnv): Promise<ImportQueue | undefined> {
if (env?.IMPORT_QUEUE_ENABLED !== "true") {
return undefined;
}
if (env.IMPORT_QUEUE_RUNTIME_READY !== "true") {
return undefined;
}
const connection = resolveRedisConnectionOptions({ env });
if (!connection) {
return undefined;
}
const queueRuntimeModule = "@toolctrl/queue/runtime";
const { createQueue } = (await import(/* @vite-ignore */ queueRuntimeModule)) as typeof import("@toolctrl/queue/runtime");
return createQueue(queueNames.imports, { connection });
}
export const action = async ({ request, params }: ActionFunctionArgs) => {
const { t } = await getTranslations(request);
if (request.method !== "POST") {
return methodNotAllowed(t);
}
const userInfo = await requireAuth({ request, params });
let body: SubmitBody;
try {
await validateCSRFToken(request);
const parsed = await request.json();
if (!isRecord(parsed)) {
return invalidInput(t("models.row.import.errors.jsonBodyMustBeObject"));
}
body = parsed;
} catch {
return invalidInput(t("models.row.import.errors.invalidJsonBody"));
}
let dryRunJobId: string;
let mappings: Record<string, string> | undefined;
try {
dryRunJobId = parseRequiredString(body.dryRunJobId, "dryRunJobId", t);
mappings = parseMappings(body.mappings, t);
} catch (error) {
return invalidInput(error instanceof Error ? error.message : t("models.row.import.errors.invalidSubmitRequest"));
}
const tenantId = await getTenantIdFromUrl(params);
const dryRunJob = await findImportJobForInspection({ id: dryRunJobId, tenantId });
if (!dryRunJob) {
return json({ error: t("models.row.import.errors.dryRunJobNotFound") }, 404);
}
let permission: DefaultPermission;
try {
permission = getImportPermission(dryRunJob.importType, t);
} catch (error) {
return invalidInput(
error instanceof Error
? t("models.row.import.errors.unsupportedType", { importType: dryRunJob.importType })
: t("models.row.import.errors.unsupportedTypeGeneric")
);
}
await verifyUserHasPermission(request, permission, dryRunJob.tenantId);
if (!canSubmitDryRun(dryRunJob)) {
return json({ error: t("models.row.import.errors.dryRunMustBeCompletedBeforeSubmit") }, 409);
}
const format = toQueueFormat(dryRunJob.format);
if (!format) {
return invalidInput(t("models.row.import.errors.unsupportedFormat"));
}
const accessDecision = authorizeStorageObjectAccess({
tenantId: dryRunJob.tenantId,
key: dryRunJob.sourceKey,
operation: "download",
});
if (!accessDecision.allowed) {
return storageAccessDenied(t, accessDecision.reason);
}
const queue = await createInternalQueue(process.env as RedisConnectionEnv);
if (!queue) {
return json({ error: t("models.row.import.errors.queueNotConfigured") }, 503);
}
let importJobId: string | undefined;
try {
importJobId = getCreatedJobId(
await createImportJob({
tenantId: dryRunJob.tenantId,
requestedByUserId: userInfo.userId ?? dryRunJob.requestedByUserId,
importType: dryRunJob.importType,
format,
requestId: crypto.randomUUID(),
sourceBucket: dryRunJob.sourceBucket,
sourceKey: accessDecision.key,
fileName: dryRunJob.fileName,
contentType: dryRunJob.contentType,
attemptsMade: 0,
}),
t
);
const descriptor = buildImportJob({
importJobId,
tenantId: dryRunJob.tenantId,
requestedByUserId: userInfo.userId ?? dryRunJob.requestedByUserId ?? "unknown",
importType: dryRunJob.importType,
format,
sourceBucket: dryRunJob.sourceBucket,
sourceKey: accessDecision.key,
fileName: dryRunJob.fileName ?? undefined,
contentType: dryRunJob.contentType ?? undefined,
mappings,
metadata: {
dryRunJobId: dryRunJob.id,
attemptsMade: 0,
},
});
try {
await enqueueJob(queue, descriptor, {
idempotencyKey: importJobId,
tenantId: dryRunJob.tenantId,
});
} catch (error) {
const enqueueFailedMessage = t("models.row.import.errors.queueEnqueueFailed");
await markImportJobFailed({
id: importJobId,
tenantId: dryRunJob.tenantId,
attemptsMade: 0,
errorName: error instanceof Error ? error.name : "QueueError",
errorMessage: enqueueFailedMessage,
});
return json({ error: enqueueFailedMessage }, 503);
}
return json({ jobId: importJobId, status: "QUEUED" }, 202);
} finally {
await queue.close();
}
};