Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -0,0 +1,655 @@
import { createHash, randomUUID } from "node:crypto";
import { logActivity } from "@hivegrid/activity";
import { getDbClient } from "@hivegrid/db/client.server";
import type { Prisma } from "@hivegrid/db/prisma-browser";
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import { TenantStorageCapacityError, withTenantStorageWriteReservation } from "@hivegrid/db/storage-capacity.server";
import { auditEvent } from "@hivegrid/foundation";
import { fetchOutboundHttpWithRedirects, readLimitedResponseText, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http";
import { z } from "zod";
import { KWP_INBOUND_BOUNDS, type KwpInboundArchiveFile } from "./inbound-reader.js";
import { KwpProjectDomainError } from "./project-domain.server.js";
import { executeKwpReadOperation } from "./resource-client.server.js";
import { fingerprintKwpSnapshot } from "./resource-client.js";
const ARCHIVE_FILE_ENTITY_TYPE = "kwp_archive_file";
const EXCHANGE_JSON_LIMIT_BYTES = 64 * 1024;
const exportInfoSchema = z
.object({
dateigroesse: z.number().int(),
exchangeFileUri: z.string().url().nullable().optional(),
dateiname: z.string().nullable().optional(),
})
.strict();
export type KwpArchiveStorage = {
putObject(input: {
key: string;
body: Uint8Array | ReadableStream<Uint8Array>;
contentType?: string;
metadata?: Record<string, string>;
}): Promise<{ bucket: string; key: string }>;
deleteObject(input: { key: string }): Promise<unknown>;
};
export type KwpArchiveScanEnqueuer = (input: {
readonly bucket: string;
readonly key: string;
readonly tenantId: string;
readonly fileId: string;
readonly contentType: string;
}) => Promise<void>;
export type KwpArchiveFileSyncResult = {
readonly runId: string;
readonly fileId: string;
readonly projectId: string;
readonly storageKey: string;
readonly state: "created" | "replayed";
};
export async function syncKwpArchiveFile(input: {
readonly bindingId: string;
readonly archiveFile: KwpInboundArchiveFile;
readonly storage: KwpArchiveStorage;
readonly storageBucket: string;
readonly enqueueScan: KwpArchiveScanEnqueuer;
readonly fetchImpl?: typeof fetch;
readonly addressResolver?: OutboundHttpAddressResolver;
readonly now?: Date;
}): Promise<KwpArchiveFileSyncResult> {
const now = input.now ?? new Date();
const prepared = await prepareArchiveFileSync(normalizeId(input.bindingId), input.archiveFile, normalizeStorageBucket(input.storageBucket), now);
if (prepared.existing) {
if (prepared.existing.needsScan) {
await input.enqueueScan({
bucket: prepared.existing.bucket,
key: prepared.existing.storageKey,
tenantId: prepared.tenantId,
fileId: prepared.existing.fileId,
contentType: prepared.existing.mimeType,
});
}
return {
runId: prepared.runId,
fileId: prepared.existing.fileId,
projectId: prepared.projectId,
storageKey: prepared.existing.storageKey,
state: "replayed",
};
}
try {
const exported = await exportArchiveFile({
connectionId: prepared.connectionId,
descriptor: input.archiveFile,
fetchImpl: input.fetchImpl,
addressResolver: input.addressResolver,
});
const storageKey = buildArchiveStorageKey(prepared.tenantId, prepared.connectionId, input.archiveFile.externalId, exported.displayName);
const targetFileId = `file_${randomUUID()}`;
const outcome = await withTenantStorageWriteReservation({
reservation: {
tenantId: prepared.tenantId,
actorUserId: prepared.actorUserId,
targetFileId,
originalFilename: exported.displayName,
mimeType: exported.mimeType,
sizeBytes: exported.sizeBytes,
},
runTransaction: (operation) => getDbClient().$transaction(operation),
upload: () =>
input.storage.putObject({
key: storageKey,
body: exported.body,
contentType: exported.mimeType,
metadata: {
source: "kwp",
project: createHash("sha256").update(prepared.projectId).digest("hex"),
external: createHash("sha256").update(input.archiveFile.externalId).digest("hex"),
},
}),
commit: async (tx, stored) => {
const created = await persistArchiveFile(
tx,
prepared,
input.archiveFile,
targetFileId,
{
storageKey: stored.key,
bucket: stored.bucket,
displayName: exported.displayName,
mimeType: exported.mimeType,
sizeBytes: exported.sizeBytes,
},
now
);
return { ...created, bucket: stored.bucket, uploadedStorageKey: stored.key };
},
rollbackUpload: (stored) => input.storage.deleteObject({ key: stored?.key ?? storageKey }).then(() => undefined),
});
if (outcome.replayed && outcome.uploadedStorageKey !== outcome.storageKey) {
await input.storage.deleteObject({ key: outcome.uploadedStorageKey });
}
await input.enqueueScan({
bucket: outcome.bucket,
key: outcome.storageKey,
tenantId: prepared.tenantId,
fileId: outcome.fileId,
contentType: exported.mimeType,
});
return {
runId: prepared.runId,
fileId: outcome.fileId,
projectId: prepared.projectId,
storageKey: outcome.storageKey,
state: outcome.replayed ? "replayed" : "created",
};
} catch (error) {
if (error instanceof TenantStorageCapacityError && error.code === "STORAGE_RESERVATION_INVALID") {
const replay = await loadArchiveFileReplay(prepared, input.archiveFile, now).catch(() => null);
if (replay) {
if (replay.needsScan) {
await input.enqueueScan({
bucket: input.storageBucket,
key: replay.storageKey,
tenantId: prepared.tenantId,
fileId: replay.fileId,
contentType: replay.mimeType,
});
}
return {
runId: prepared.runId,
fileId: replay.fileId,
projectId: prepared.projectId,
storageKey: replay.storageKey,
state: "replayed",
};
}
}
await failArchiveRun(prepared, error, now);
throw error;
}
}
type PreparedArchiveFileSync = {
readonly tenantId: string;
readonly runId: string;
readonly connectionId: string;
readonly projectId: string;
readonly externalProjectNumber: string;
readonly actorUserId: string;
readonly existing: {
readonly fileId: string;
readonly storageKey: string;
readonly mimeType: string;
readonly bucket: string;
readonly needsScan: boolean;
} | null;
};
async function prepareArchiveFileSync(
bindingId: string,
archiveFile: KwpInboundArchiveFile,
storageBucket: string,
now: Date
): Promise<PreparedArchiveFileSync> {
const tenantId = requireTenantContext();
return getDbClient().$transaction(async (tx) => {
const binding = await tx.appEntityBinding.findFirst({
where: {
id: bindingId,
tenantId,
externalEntityType: "kwp_project",
localEntityType: "project",
state: "active",
},
select: { dataConnectionId: true, externalId: true, localId: true },
});
if (!binding) throw new KwpProjectDomainError("project_not_found");
const [connection, project, existingBinding] = await Promise.all([
tx.dataConnection.findFirst({
where: {
id: binding.dataConnectionId,
tenantId,
appId: "kwp",
archivedAt: null,
connectionStatus: { in: ["active", "degraded"] },
},
select: { createdByUserId: true },
}),
tx.projekt.findFirst({
where: { id: binding.localId, tenantId, archivedAt: null },
select: { id: true, createdByUserId: true },
}),
tx.appEntityBinding.findUnique({
where: {
tenantId_dataConnectionId_externalEntityType_externalId: {
tenantId,
dataConnectionId: binding.dataConnectionId,
externalEntityType: ARCHIVE_FILE_ENTITY_TYPE,
externalId: archiveFile.externalId,
},
},
select: { localId: true },
}),
]);
if (!connection || !project) throw new KwpProjectDomainError("project_not_found");
const actorUserId = await resolveActorUserId(tx, tenantId, connection.createdByUserId ?? project.createdByUserId);
if (!actorUserId) throw new KwpProjectDomainError("project_not_found");
const existingFile = existingBinding
? await tx.file.findFirst({
where: { id: existingBinding.localId, tenantId, archivedAt: null },
select: { id: true, storageKey: true, mimeType: true, scanStatus: true },
})
: null;
const run = await tx.appSyncRun.create({
data: {
tenantId,
dataConnectionId: binding.dataConnectionId,
kind: "scheduled",
status: existingFile ? "completed" : "running",
itemsTotal: 1,
itemsSucceeded: existingFile ? 1 : 0,
summaryText: existingFile ? "integrations.sync.summary.kwpArchiveAlreadyPresent" : "integrations.sync.summary.kwpArchiveImportStarted",
startedAt: now,
completedAt: existingFile ? now : null,
},
});
return {
tenantId,
runId: run.id,
connectionId: binding.dataConnectionId,
projectId: project.id,
externalProjectNumber: binding.externalId,
actorUserId,
existing: existingFile
? {
fileId: existingFile.id,
storageKey: existingFile.storageKey,
mimeType: existingFile.mimeType,
bucket: storageBucket,
needsScan: existingFile.scanStatus === "pending" || existingFile.scanStatus === "skipped",
}
: null,
};
});
}
async function exportArchiveFile(input: {
readonly connectionId: string;
readonly descriptor: KwpInboundArchiveFile;
readonly fetchImpl?: typeof fetch;
readonly addressResolver?: OutboundHttpAddressResolver;
}): Promise<{
readonly body: Uint8Array;
readonly displayName: string;
readonly mimeType: string;
readonly sizeBytes: number;
}> {
const exportInfo = exportInfoSchema.parse(
await executeKwpReadOperation({
connectionId: input.connectionId,
operationId: "archivExternalExportDatei",
operationInput: {
query: {
guid: randomUUID(),
revisionId: input.descriptor.revisionId,
},
},
})
);
if (!exportInfo.exchangeFileUri || exportInfo.dateigroesse < 0 || exportInfo.dateigroesse > KWP_INBOUND_BOUNDS.maxArchiveFileBytes) {
throw new KwpProjectDomainError("project_not_found");
}
const exchangeResponse = await safeDownload(exportInfo.exchangeFileUri, input);
if (!exchangeResponse.ok) throw new Error("KWP archive exchange request failed.");
const exchange = await readLimitedResponseText(exchangeResponse, EXCHANGE_JSON_LIMIT_BYTES);
if (exchange.truncated) throw new Error("KWP archive exchange response exceeded its limit.");
const downloadUrl = parseExchangeDownloadUrl(exchange.body);
const fileResponse = await safeDownload(downloadUrl, input);
if (!fileResponse.ok || !fileResponse.body) throw new Error("KWP archive download failed.");
const declaredLength = parseContentLength(fileResponse.headers.get("content-length"));
const advertisedSize = declaredLength ?? exportInfo.dateigroesse;
if (advertisedSize < 0 || advertisedSize > KWP_INBOUND_BOUNDS.maxArchiveFileBytes) {
await fileResponse.body.cancel().catch(() => undefined);
throw new Error("KWP archive file exceeded its size limit.");
}
const body = new Uint8Array(await new Response(limitBody(fileResponse.body, KWP_INBOUND_BOUNDS.maxArchiveFileBytes)).arrayBuffer());
if (body.byteLength <= 0 || (declaredLength !== null && body.byteLength !== declaredLength)) {
throw new Error("KWP archive file size did not match its declared size.");
}
return {
body,
displayName: safeFileName(exportInfo.dateiname ?? input.descriptor.displayName),
mimeType: safeMimeType(fileResponse.headers.get("content-type"), input.descriptor.providerFileType),
sizeBytes: body.byteLength,
};
}
async function safeDownload(
url: string,
input: { readonly fetchImpl?: typeof fetch; readonly addressResolver?: OutboundHttpAddressResolver }
): Promise<Response> {
return fetchOutboundHttpWithRedirects(
url,
{
method: "GET",
headers: { accept: "application/json, application/octet-stream;q=0.9, */*;q=0.1" },
},
{
fetchImpl: input.fetchImpl,
addressResolver: input.addressResolver,
timeoutMs: 30_000,
redirectLimit: 3,
urlValidator: requireHttpsUrl,
}
);
}
function parseExchangeDownloadUrl(value: string): string {
let parsed: unknown;
try {
parsed = JSON.parse(value);
} catch {
throw new Error("KWP archive exchange response was not valid JSON.");
}
const candidate =
typeof parsed === "string"
? parsed
: isRecord(parsed)
? ["downloadUrl", "downloadUri", "fileUrl", "fileUri", "url", "uri"]
.map((key) => parsed[key])
.find((entry): entry is string => typeof entry === "string")
: undefined;
if (!candidate) throw new Error("KWP archive exchange response did not contain a download URL.");
return requireHttpsUrl(candidate);
}
async function persistArchiveFile(
tx: Prisma.TransactionClient,
prepared: PreparedArchiveFileSync,
descriptor: KwpInboundArchiveFile,
targetFileId: string,
stored: {
readonly storageKey: string;
readonly bucket: string;
readonly displayName: string;
readonly mimeType: string;
readonly sizeBytes: number;
},
now: Date
): Promise<{ readonly fileId: string; readonly replayed: boolean; readonly storageKey: string }> {
const replay = await tx.appEntityBinding.findUnique({
where: {
tenantId_dataConnectionId_externalEntityType_externalId: {
tenantId: prepared.tenantId,
dataConnectionId: prepared.connectionId,
externalEntityType: ARCHIVE_FILE_ENTITY_TYPE,
externalId: descriptor.externalId,
},
},
select: { localId: true },
});
if (replay) {
const existingFile = await tx.file.findFirstOrThrow({
where: { id: replay.localId, tenantId: prepared.tenantId },
select: { id: true, storageKey: true },
});
await completeArchiveRun(tx, prepared, now);
return { fileId: existingFile.id, storageKey: existingFile.storageKey, replayed: true };
}
const file = await tx.file.create({
data: {
id: targetFileId,
tenantId: prepared.tenantId,
createdByUserId: prepared.actorUserId,
displayName: stored.displayName,
mimeType: stored.mimeType,
sizeBytes: stored.sizeBytes,
storageKey: stored.storageKey,
previewStatus: "pending",
scanStatus: "pending",
source: "api",
tags: ["kwp"],
notes: descriptor.documentType,
},
});
await tx.fileEntityLink.create({
data: {
tenantId: prepared.tenantId,
createdByUserId: prepared.actorUserId,
fileId: file.id,
entityType: "project",
entityId: prepared.projectId,
linkRole: "kwp_archive",
},
});
const normalizedSnapshot = { projectId: prepared.projectId, ...descriptor };
await tx.appEntityBinding.create({
data: {
tenantId: prepared.tenantId,
dataConnectionId: prepared.connectionId,
externalEntityType: ARCHIVE_FILE_ENTITY_TYPE,
localEntityType: "file",
externalId: descriptor.externalId,
localId: file.id,
state: "active",
normalizedSnapshot: normalizedSnapshot as Prisma.InputJsonValue,
fingerprint: fingerprintKwpSnapshot(normalizedSnapshot),
lastSeenAt: now,
lastSyncedAt: now,
},
});
await tx.appSyncItem.create({
data: {
tenantId: prepared.tenantId,
syncRunId: prepared.runId,
entityType: ARCHIVE_FILE_ENTITY_TYPE,
entityId: file.id,
externalId: descriptor.externalId,
operation: "created",
status: "synced",
},
});
await auditEvent(
{
tenantId: prepared.tenantId,
actorType: "system",
action: "KWP_ARCHIVE_FILE_IMPORTED",
targetType: "file",
targetId: file.id,
metadata: {
source: "kwp",
projectId: prepared.projectId,
revision: descriptor.revision,
},
},
tx as never
);
await logActivity(
{
tenantId: prepared.tenantId,
actorKind: "system",
activityType: "project_updated",
subjectType: "project",
subjectId: prepared.projectId,
summaryText: "activity.runtime.kwp.archiveImported",
metadata: { source: "kwp", fileId: file.id },
},
{ client: tx }
);
await completeArchiveRun(tx, prepared, now);
return { fileId: file.id, storageKey: file.storageKey, replayed: false };
}
async function completeArchiveRun(tx: Prisma.TransactionClient, prepared: PreparedArchiveFileSync, now: Date): Promise<void> {
await tx.appSyncRun.updateMany({
where: { id: prepared.runId, tenantId: prepared.tenantId, status: "running" },
data: {
status: "completed",
itemsTotal: 1,
itemsSucceeded: 1,
summaryText: "integrations.sync.summary.kwpArchiveImported",
completedAt: now,
},
});
}
async function loadArchiveFileReplay(
prepared: PreparedArchiveFileSync,
descriptor: KwpInboundArchiveFile,
now: Date
): Promise<{ readonly fileId: string; readonly storageKey: string; readonly mimeType: string; readonly needsScan: boolean } | null> {
return getDbClient().$transaction(async (tx) => {
const binding = await tx.appEntityBinding.findUnique({
where: {
tenantId_dataConnectionId_externalEntityType_externalId: {
tenantId: prepared.tenantId,
dataConnectionId: prepared.connectionId,
externalEntityType: ARCHIVE_FILE_ENTITY_TYPE,
externalId: descriptor.externalId,
},
},
select: { localId: true },
});
if (!binding) return null;
const file = await tx.file.findFirst({
where: { id: binding.localId, tenantId: prepared.tenantId },
select: { id: true, storageKey: true, mimeType: true, scanStatus: true },
});
if (!file) return null;
await completeArchiveRun(tx, prepared, now);
return {
fileId: file.id,
storageKey: file.storageKey,
mimeType: file.mimeType,
needsScan: file.scanStatus === "pending" || file.scanStatus === "skipped",
};
});
}
async function failArchiveRun(prepared: PreparedArchiveFileSync, error: unknown, now: Date): Promise<void> {
await getDbClient().appSyncRun.updateMany({
where: { id: prepared.runId, tenantId: prepared.tenantId, status: "running" },
data: {
status: "failed",
itemsTotal: 1,
itemsFailed: 1,
summaryText: buildArchiveFailureSummary(error),
completedAt: now,
},
});
}
function buildArchiveFailureSummary(error: unknown): string {
const code = error && typeof error === "object" && "code" in error ? (error as { code?: unknown }).code : null;
if (code === "provider_outage" || code === "rate_limit_exceeded" || code === "auth_token_invalid") {
return "integrations.sync.summary.kwpArchiveTemporaryFailure";
}
if (code === "auth_scope_missing" || code === "capability_paused" || code === "contract_drift") {
return "integrations.sync.summary.kwpArchiveUnavailable";
}
return "integrations.sync.summary.kwpArchiveFailed";
}
async function resolveActorUserId(tx: Prisma.TransactionClient, tenantId: string, preferredUserId: string | null | undefined): Promise<string | null> {
if (preferredUserId) {
const preferred = await tx.tenantUser.findFirst({
where: { tenantId, userId: preferredUserId },
select: { userId: true },
});
if (preferred) return preferred.userId;
}
const fallback = await tx.tenantUser.findFirst({
where: { tenantId },
select: { userId: true },
orderBy: { createdAt: "asc" },
});
return fallback?.userId ?? null;
}
function limitBody(body: ReadableStream<Uint8Array>, limit: number): ReadableStream<Uint8Array> {
let bytes = 0;
return body.pipeThrough(
new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
bytes += chunk.byteLength;
if (bytes > limit) {
controller.error(new Error("KWP archive file exceeded its size limit."));
return;
}
controller.enqueue(chunk);
},
})
);
}
function buildArchiveStorageKey(tenantId: string, connectionId: string, externalId: string, fileName: string): string {
const identity = createHash("sha256").update(`${connectionId}:${externalId}`).digest("hex");
return `tenants/${safePathPart(tenantId)}/kwp-archive/${identity}/${safeFileName(fileName)}`;
}
function safeFileName(value: string): string {
const name = value.replaceAll("\\", "/").split("/").at(-1)?.trim() ?? "";
const normalized = name.replace(/[\u0000-\u001f\u007f]/gu, "").slice(0, 255);
return normalized || "kwp-datei.bin";
}
function safePathPart(value: string): string {
return value.replace(/[^A-Za-z0-9_-]/gu, "_").slice(0, 191);
}
function safeMimeType(contentType: string | null, providerType: string | null): string {
const fromHeader = contentType?.split(";")[0]?.trim().toLowerCase();
if (fromHeader && /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/u.test(fromHeader)) {
return fromHeader;
}
if (providerType?.trim().toLowerCase() === "pdf") return "application/pdf";
return "application/octet-stream";
}
function parseContentLength(value: string | null): number | null {
if (!value || !/^\d+$/u.test(value)) return null;
const parsed = Number(value);
return Number.isSafeInteger(parsed) ? parsed : null;
}
function requireHttpsUrl(value: string): string {
const url = new URL(value);
if (url.protocol !== "https:" || url.username || url.password) {
throw new Error("KWP archive URL is not allowed.");
}
return url.toString();
}
function normalizeStorageBucket(value: string): string {
const bucket = value.trim();
if (!bucket || bucket.length > 255) throw new Error("KWP archive storage bucket is invalid.");
return bucket;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function requireTenantContext(): string {
const tenantId = getRlsExecutionContext()?.tenantId?.trim();
if (!tenantId) throw new KwpProjectDomainError("project_not_found");
return tenantId;
}
function normalizeId(value: string): string {
const normalized = value.trim();
if (!/^[A-Za-z0-9_-]{3,191}$/u.test(normalized)) throw new KwpProjectDomainError("project_not_found");
return normalized;
}
export const __kwpArchiveInternals = {
buildArchiveStorageKey,
parseExchangeDownloadUrl,
safeFileName,
safeMimeType,
};