Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -0,0 +1,357 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { DeleteObjectsCommand, GetObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import { loadRequiredWebConfig } from "@hivegrid/config";
|
||||
import { buildFileStorageKeys } from "@hivegrid/storage";
|
||||
|
||||
import {
|
||||
deleteFile,
|
||||
hasStorageCredentials,
|
||||
putBufferAsFile,
|
||||
resolveStorageBinding,
|
||||
resolveStorageBucket,
|
||||
} from "../../apps/web/app/utils/integrations/storageService";
|
||||
|
||||
const DEFAULT_STORAGE_CONCURRENCY = 12;
|
||||
const PDF_CONTENT_TYPE = "application/pdf";
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
export type SeedFileObjectDescriptor = {
|
||||
id: string;
|
||||
displayName: string;
|
||||
notes: string;
|
||||
};
|
||||
|
||||
export type SeedFileObjectMetadata = SeedFileObjectDescriptor & {
|
||||
mimeType: string;
|
||||
sizeBytes: number;
|
||||
checksumSha256: string;
|
||||
};
|
||||
|
||||
type PersistedSeedFileObject = SeedFileObjectDescriptor & {
|
||||
mimeType: string;
|
||||
sizeBytes: number;
|
||||
checksumSha256: string;
|
||||
};
|
||||
|
||||
type PutObject = (input: { bucket: string; key: string; body: Uint8Array; contentType?: string }) => Promise<unknown>;
|
||||
|
||||
type DeleteObject = (input: { bucket: string; key: string }) => Promise<unknown>;
|
||||
|
||||
type SeedS3Client = {
|
||||
send(command: unknown): Promise<unknown>;
|
||||
};
|
||||
|
||||
function printableAscii(value: string): string {
|
||||
return value
|
||||
.normalize("NFKD")
|
||||
.replace(/[\u0300-\u036f]/gu, "")
|
||||
.replace(/[^\x20-\x7E]/gu, "?")
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function escapePdfLiteral(value: string): string {
|
||||
return printableAscii(value).replace(/([\\()])/gu, "\\$1");
|
||||
}
|
||||
|
||||
function wrapText(value: string, maxLength = 72): string[] {
|
||||
const words = printableAscii(value).split(" ").filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
if (line.length === 0) {
|
||||
line = word;
|
||||
continue;
|
||||
}
|
||||
if (line.length + word.length + 1 <= maxLength) {
|
||||
line += ` ${word}`;
|
||||
continue;
|
||||
}
|
||||
lines.push(line);
|
||||
line = word;
|
||||
}
|
||||
if (line.length > 0) lines.push(line);
|
||||
return lines.length > 0 ? lines : ["Projektunterlage"];
|
||||
}
|
||||
|
||||
function byteLength(value: string): number {
|
||||
return encoder.encode(value).byteLength;
|
||||
}
|
||||
|
||||
/** Builds a deterministic, single-page PDF whose xref offsets match its exact bytes. */
|
||||
export function buildSeedPdfBytes(file: SeedFileObjectDescriptor): Uint8Array {
|
||||
const title = escapePdfLiteral(file.displayName);
|
||||
const detailLines = wrapText(file.notes).slice(0, 6).map(escapePdfLiteral);
|
||||
const content = [
|
||||
"BT",
|
||||
"/F1 18 Tf",
|
||||
"72 770 Td",
|
||||
`(${title}) Tj`,
|
||||
"/F1 11 Tf",
|
||||
...detailLines.flatMap((line) => ["0 -24 Td", `(${line}) Tj`]),
|
||||
"0 -36 Td",
|
||||
`(Dokument-ID: ${escapePdfLiteral(file.id)}) Tj`,
|
||||
"ET",
|
||||
"",
|
||||
].join("\n");
|
||||
const objects = [
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
`<< /Length ${byteLength(content)} >>\nstream\n${content}endstream`,
|
||||
];
|
||||
|
||||
let pdf = "%PDF-1.4\n% HiveGrid generated staging document\n";
|
||||
const offsets = [0];
|
||||
for (const [index, object] of objects.entries()) {
|
||||
offsets.push(byteLength(pdf));
|
||||
pdf += `${index + 1} 0 obj\n${object}\nendobj\n`;
|
||||
}
|
||||
const xrefOffset = byteLength(pdf);
|
||||
pdf += `xref\n0 ${objects.length + 1}\n`;
|
||||
pdf += "0000000000 65535 f \n";
|
||||
pdf += offsets
|
||||
.slice(1)
|
||||
.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`)
|
||||
.join("");
|
||||
pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
|
||||
return encoder.encode(pdf);
|
||||
}
|
||||
|
||||
export function buildSeedFileObjectMetadata(file: SeedFileObjectDescriptor): SeedFileObjectMetadata {
|
||||
const bytes = buildSeedPdfBytes(file);
|
||||
return {
|
||||
...file,
|
||||
mimeType: PDF_CONTENT_TYPE,
|
||||
sizeBytes: bytes.byteLength,
|
||||
checksumSha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
};
|
||||
}
|
||||
|
||||
export function seedFileStorageKey(tenantId: string, fileId: string): string {
|
||||
return buildFileStorageKeys(tenantId, fileId).original;
|
||||
}
|
||||
|
||||
export function seedFileStoragePrefix(tenantId: string): string {
|
||||
const probe = seedFileStorageKey(tenantId, "seed-prefix-probe");
|
||||
return probe.slice(0, -"seed-prefix-probe/original".length);
|
||||
}
|
||||
|
||||
function createSeedS3Client(): SeedS3Client {
|
||||
const binding = resolveStorageBinding();
|
||||
const { storage } = loadRequiredWebConfig();
|
||||
if (binding.provider !== "s3" || !storage.endpoint || !storage.region || !storage.accessKeyId || !storage.secretAccessKey) {
|
||||
throw new Error("demo-pro seed files: complete S3 staging storage configuration is required");
|
||||
}
|
||||
return new S3Client({
|
||||
endpoint: storage.endpoint,
|
||||
region: storage.region,
|
||||
credentials: {
|
||||
accessKeyId: storage.accessKeyId,
|
||||
secretAccessKey: storage.secretAccessKey,
|
||||
sessionToken: storage.sessionToken,
|
||||
},
|
||||
forcePathStyle: storage.forcePathStyle?.toLowerCase() === "true",
|
||||
}) as unknown as SeedS3Client;
|
||||
}
|
||||
|
||||
async function storageBodyBytes(body: unknown): Promise<Uint8Array> {
|
||||
if (!body || typeof (body as { transformToByteArray?: unknown }).transformToByteArray !== "function") {
|
||||
throw new Error("demo-pro seed files: storage object body is unreadable");
|
||||
}
|
||||
return (body as { transformToByteArray(): Promise<Uint8Array> }).transformToByteArray();
|
||||
}
|
||||
|
||||
function assertConcurrency(concurrency: number): void {
|
||||
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
|
||||
throw new Error("demo-pro seed files: concurrency must be an integer between 1 and 64");
|
||||
}
|
||||
}
|
||||
|
||||
async function forEachConcurrent<T>(items: readonly T[], concurrency: number, worker: (item: T) => Promise<void>): Promise<void> {
|
||||
assertConcurrency(concurrency);
|
||||
let nextIndex = 0;
|
||||
let failed = false;
|
||||
let failure: unknown;
|
||||
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
||||
while (!failed && nextIndex < items.length) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
try {
|
||||
await worker(items[index]!);
|
||||
} catch (error) {
|
||||
if (!failed) failure = error;
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
if (failed) throw failure;
|
||||
}
|
||||
|
||||
function assertMetadataMatches(file: PersistedSeedFileObject, bytes: Uint8Array): void {
|
||||
const checksumSha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
if (file.mimeType !== PDF_CONTENT_TYPE || file.sizeBytes !== bytes.byteLength || file.checksumSha256 !== checksumSha256) {
|
||||
throw new Error(`demo-pro seed files: generated PDF metadata drifted for ${file.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadDemoProSeedFileObjects({
|
||||
tenantId,
|
||||
files,
|
||||
concurrency = DEFAULT_STORAGE_CONCURRENCY,
|
||||
bucket = resolveStorageBucket(),
|
||||
putObject = putBufferAsFile,
|
||||
}: {
|
||||
tenantId: string;
|
||||
files: readonly PersistedSeedFileObject[];
|
||||
concurrency?: number;
|
||||
bucket?: string;
|
||||
putObject?: PutObject;
|
||||
}): Promise<{ bucket: string; uploaded: number }> {
|
||||
await forEachConcurrent(files, concurrency, async (file) => {
|
||||
const bytes = buildSeedPdfBytes(file);
|
||||
assertMetadataMatches(file, bytes);
|
||||
await putObject({
|
||||
bucket,
|
||||
key: seedFileStorageKey(tenantId, file.id),
|
||||
body: bytes,
|
||||
contentType: PDF_CONTENT_TYPE,
|
||||
});
|
||||
});
|
||||
return { bucket, uploaded: files.length };
|
||||
}
|
||||
|
||||
export async function verifyDemoProSeedFileObjects({
|
||||
tenantId,
|
||||
files,
|
||||
concurrency = DEFAULT_STORAGE_CONCURRENCY,
|
||||
bucket = resolveStorageBucket(),
|
||||
client = createSeedS3Client(),
|
||||
}: {
|
||||
tenantId: string;
|
||||
files: readonly PersistedSeedFileObject[];
|
||||
concurrency?: number;
|
||||
bucket?: string;
|
||||
client?: SeedS3Client;
|
||||
}): Promise<{ bucket: string; verified: number }> {
|
||||
const prefix = seedFileStoragePrefix(tenantId);
|
||||
const expectedKeys = new Set(files.map((file) => seedFileStorageKey(tenantId, file.id)));
|
||||
if (expectedKeys.size !== files.length) {
|
||||
throw new Error("demo-pro seed files: expected storage keys are not unique");
|
||||
}
|
||||
const observedKeys = new Set<string>();
|
||||
let continuationToken: string | undefined;
|
||||
do {
|
||||
const page = (await client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, ContinuationToken: continuationToken, MaxKeys: 1_000 }))) as {
|
||||
Contents?: Array<{ Key?: string }>;
|
||||
IsTruncated?: boolean;
|
||||
NextContinuationToken?: string;
|
||||
};
|
||||
for (const object of page.Contents ?? []) {
|
||||
if (!object.Key) throw new Error("demo-pro seed files: storage listing returned an object without a key");
|
||||
if (!object.Key.startsWith(prefix)) {
|
||||
throw new Error("demo-pro seed files: storage listing returned a key outside the tenant file prefix");
|
||||
}
|
||||
if (!expectedKeys.has(object.Key)) {
|
||||
throw new Error(`demo-pro seed files: unexpected storage object ${object.Key}`);
|
||||
}
|
||||
if (observedKeys.has(object.Key)) {
|
||||
throw new Error(`demo-pro seed files: storage listing returned duplicate object ${object.Key}`);
|
||||
}
|
||||
observedKeys.add(object.Key);
|
||||
}
|
||||
if (page.IsTruncated && !page.NextContinuationToken) {
|
||||
throw new Error("demo-pro seed files: truncated storage listing has no continuation token");
|
||||
}
|
||||
continuationToken = page.IsTruncated ? page.NextContinuationToken : undefined;
|
||||
} while (continuationToken);
|
||||
for (const expectedKey of expectedKeys) {
|
||||
if (!observedKeys.has(expectedKey)) throw new Error(`demo-pro seed files: missing storage object ${expectedKey}`);
|
||||
}
|
||||
|
||||
await forEachConcurrent(files, concurrency, async (file) => {
|
||||
const result = (await client.send(new GetObjectCommand({ Bucket: bucket, Key: seedFileStorageKey(tenantId, file.id) }))) as {
|
||||
Body?: unknown;
|
||||
ContentLength?: number;
|
||||
};
|
||||
const bytes = await storageBodyBytes(result.Body);
|
||||
assertMetadataMatches(file, bytes);
|
||||
if (result.ContentLength !== undefined && result.ContentLength !== bytes.byteLength) {
|
||||
throw new Error(`demo-pro seed files: storage size drifted for ${file.id}`);
|
||||
}
|
||||
});
|
||||
return { bucket, verified: files.length };
|
||||
}
|
||||
|
||||
export async function deleteDemoProTenantFileObjects({
|
||||
tenantId,
|
||||
bucket = resolveStorageBucket(),
|
||||
client = createSeedS3Client(),
|
||||
}: {
|
||||
tenantId: string;
|
||||
bucket?: string;
|
||||
client?: SeedS3Client;
|
||||
}): Promise<{ bucket: string; deleted: number }> {
|
||||
const prefix = seedFileStoragePrefix(tenantId);
|
||||
let deleted = 0;
|
||||
let previousFirstKey: string | undefined;
|
||||
while (true) {
|
||||
const page = (await client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, MaxKeys: 1_000 }))) as {
|
||||
Contents?: Array<{ Key?: string }>;
|
||||
IsTruncated?: boolean;
|
||||
NextContinuationToken?: string;
|
||||
};
|
||||
const pageKeys = (page.Contents ?? []).map(({ Key }) => {
|
||||
if (!Key) throw new Error("demo-pro seed files: storage listing returned an object without a key");
|
||||
return Key;
|
||||
});
|
||||
if (pageKeys.some((key) => !key.startsWith(prefix))) {
|
||||
throw new Error("demo-pro seed files: storage listing returned a key outside the tenant file prefix");
|
||||
}
|
||||
if (pageKeys.length > 1_000 || new Set(pageKeys).size !== pageKeys.length) {
|
||||
throw new Error("demo-pro seed files: storage listing returned an invalid cleanup page");
|
||||
}
|
||||
if (pageKeys.length === 0) {
|
||||
if (page.IsTruncated) throw new Error("demo-pro seed files: truncated storage listing returned no objects");
|
||||
break;
|
||||
}
|
||||
if (pageKeys[0] === previousFirstKey) {
|
||||
throw new Error("demo-pro seed files: storage prefix cleanup made no progress");
|
||||
}
|
||||
previousFirstKey = pageKeys[0];
|
||||
const result = (await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: { Objects: pageKeys.map((Key) => ({ Key })), Quiet: true },
|
||||
})
|
||||
)) as { Errors?: unknown[] };
|
||||
if ((result.Errors?.length ?? 0) > 0) throw new Error("demo-pro seed files: storage prefix cleanup failed");
|
||||
deleted += pageKeys.length;
|
||||
}
|
||||
return { bucket, deleted };
|
||||
}
|
||||
|
||||
export async function deleteDemoProSeedFileObjects({
|
||||
tenantId,
|
||||
files,
|
||||
concurrency = DEFAULT_STORAGE_CONCURRENCY,
|
||||
bucket = resolveStorageBucket(),
|
||||
deleteObject = deleteFile,
|
||||
storageConfigured = hasStorageCredentials,
|
||||
}: {
|
||||
tenantId: string;
|
||||
files: readonly Pick<PersistedSeedFileObject, "id">[];
|
||||
concurrency?: number;
|
||||
bucket?: string;
|
||||
deleteObject?: DeleteObject;
|
||||
storageConfigured?: () => boolean;
|
||||
}): Promise<{ bucket: string; deleted: number }> {
|
||||
if (!storageConfigured()) throw new Error("demo-pro seed files: storage provider is not configured");
|
||||
await forEachConcurrent(files, concurrency, async (file) => {
|
||||
await deleteObject({ bucket, key: seedFileStorageKey(tenantId, file.id) });
|
||||
});
|
||||
return { bucket, deleted: files.length };
|
||||
}
|
||||
Reference in new issue
Block a user