Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -5,7 +5,10 @@ import type { DefaultMetrics } from "@hivegrid/metrics";
|
||||
import { parseRedisUrl, queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue";
|
||||
import { createWorker } from "@hivegrid/queue/runtime";
|
||||
import { createRegisteredProcessor, type HandlerDependencies } from "./handlers.js";
|
||||
import { defaultWorkerQueueNames, getWorkerQueueNames } from "./queues.js";
|
||||
import { wrapProcessorWithRlsContext } from "./rls-context.js";
|
||||
import { redactCompletedWorkspaceOwnerInvitationJob, redactTerminalFailedWorkspaceOwnerInvitationJob } from "./security/workspace-owner-invitation-secret.js";
|
||||
import { redactCompletedPortalSetupInvitationJob, redactTerminalFailedPortalSetupInvitationJob } from "./security/portal-setup-invitation-secret.js";
|
||||
import { defaultWorkerQueueNames, getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
|
||||
import { createWorkerStorageDeleteService } from "./storage.js";
|
||||
import { createWorkerScanService } from "./scanService.js";
|
||||
|
||||
@@ -37,6 +40,7 @@ type WorkerJob = {
|
||||
backoff?: number | WorkerBackoffOptions;
|
||||
};
|
||||
queueName?: string;
|
||||
updateData?: (data: never) => Promise<void>;
|
||||
};
|
||||
|
||||
type WorkerLogBindings = {
|
||||
@@ -189,28 +193,52 @@ export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): Wor
|
||||
: undefined;
|
||||
const handlerDependencies: HandlerDependencies = {
|
||||
env,
|
||||
// Handlers emit the provider-stage diagnostics that are required to
|
||||
// distinguish a recoverable KWP outage from a reconnect condition. Keep
|
||||
// the narrow handler logger on the same structured runtime sink instead
|
||||
// of silently falling back to the no-op logger.
|
||||
logger: {
|
||||
info: (fields, message) => logInfo(logger, fields, message),
|
||||
error: (fields, message) => logError(logger, fields, message),
|
||||
},
|
||||
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? { storageDeleteService: createWorkerStorageDeleteService(env) } : {}),
|
||||
...(scanService ? { scanService } : {}),
|
||||
...(mailMetricsHook ? { mailMetricsHook } : {}),
|
||||
...(webhookDeliveryMetricsHook ? { webhookDeliveryMetricsHook } : {}),
|
||||
...(options.extraHandlerDependencies ?? {}),
|
||||
};
|
||||
assertQueueServicesWired(queues, handlerDependencies);
|
||||
assertQueueServicesWired(queues, handlerDependencies, env);
|
||||
const now = options.now ?? (() => performance.now());
|
||||
const workers = queues.map((queueName) => {
|
||||
const baseProcessor = createProcessor(queueName, handlerDependencies);
|
||||
const instrumented = metrics ? wrapProcessorWithMetrics(queueName, baseProcessor, metrics, now) : baseProcessor;
|
||||
const tenantScopedProcessor = wrapProcessorWithRlsContext(baseProcessor);
|
||||
const instrumented = metrics ? wrapProcessorWithMetrics(queueName, tenantScopedProcessor, metrics, now) : tenantScopedProcessor;
|
||||
const processor = createLoggedProcessor(queueName, instrumented, logger, reporter, sentry);
|
||||
const worker = createWorkerImpl(queueName, processor, { connection, concurrency: config.concurrency });
|
||||
|
||||
worker.on("completed", (job: WorkerJob) => {
|
||||
logInfo(createJobLogger(logger, queueName, job), {}, "Worker job completed.");
|
||||
const jobLogger = createJobLogger(logger, queueName, job);
|
||||
logInfo(jobLogger, {}, "Worker job completed.");
|
||||
void Promise.all([redactCompletedWorkspaceOwnerInvitationJob(job), redactCompletedPortalSetupInvitationJob(job)]).catch((error) => {
|
||||
const redactedError = redactWorkerError(error);
|
||||
logError(jobLogger, { error: redactedError }, "Completed capability-bearing mail job payload redaction failed.");
|
||||
reportWorkerError(reporter, "job.secret_redaction_failed", "error", queueName, job, redactedError);
|
||||
});
|
||||
});
|
||||
worker.on("failed", (job: WorkerJob | undefined, error: Error) => {
|
||||
const jobLogger = createJobLogger(logger, queueName, job);
|
||||
const redactedError = redactWorkerError(error);
|
||||
logError(jobLogger, { error: redactedError }, "Worker job failed.");
|
||||
reportWorkerError(reporter, "job.failed", "error", queueName, job, redactedError);
|
||||
if (isTerminalFailure(job)) {
|
||||
void Promise.all([redactTerminalFailedWorkspaceOwnerInvitationJob(job ?? {}), redactTerminalFailedPortalSetupInvitationJob(job ?? {})]).catch(
|
||||
(redactionError) => {
|
||||
const safeRedactionError = redactWorkerError(redactionError);
|
||||
logError(jobLogger, { error: safeRedactionError }, "Terminal capability-bearing mail job payload redaction failed.");
|
||||
reportWorkerError(reporter, "job.secret_redaction_failed", "error", queueName, job, safeRedactionError);
|
||||
}
|
||||
);
|
||||
}
|
||||
void persistTerminalFailedJob({ queueName, job, error: redactedError, logger: jobLogger, recordFailedJob, reporter });
|
||||
});
|
||||
|
||||
@@ -227,24 +255,76 @@ export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): Wor
|
||||
};
|
||||
}
|
||||
|
||||
function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies): void {
|
||||
function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies, env: EnvRecord): void {
|
||||
const ownsImports = queues.includes(queueNames.imports);
|
||||
const ownsExports = queues.includes(queueNames.exports);
|
||||
const ownsKwpImports = queues.includes(queueNames.kwpImports);
|
||||
const importMode = resolveImportQueueRuntimeMode(env);
|
||||
const missing: string[] = [];
|
||||
|
||||
if (importMode === "portal-gdpr-export" && (!ownsKwpImports || !ownsExports || ownsImports || queues.length !== 2)) {
|
||||
throw new Error("Portal GDPR export queue mode must own hivegrid.kwp-imports and hivegrid.exports.");
|
||||
}
|
||||
|
||||
if (ownsKwpImports) {
|
||||
if (!dependencies.tenantMaintenanceLeaseStore) missing.push("tenantMaintenanceLeaseStore");
|
||||
if (!dependencies.syncKwpProjectBinding) missing.push("syncKwpProjectBinding");
|
||||
if (!dependencies.syncKwpProjectCatalog) missing.push("syncKwpProjectCatalog");
|
||||
if (!dependencies.enqueueKwpProjectCatalogContinuation) missing.push("enqueueKwpProjectCatalogContinuation");
|
||||
if (!dependencies.syncKwpProjectInbound) missing.push("syncKwpProjectInbound");
|
||||
if (!dependencies.enqueueKwpArchiveFileJobs) missing.push("enqueueKwpArchiveFileJobs");
|
||||
if (!dependencies.syncKwpArchiveFile) missing.push("syncKwpArchiveFile");
|
||||
if (!dependencies.runKwpContractSentinel) missing.push("runKwpContractSentinel");
|
||||
}
|
||||
|
||||
if (ownsImports) {
|
||||
if (!dependencies.importService) missing.push("importService");
|
||||
if (!dependencies.scanService) missing.push("scanService");
|
||||
if (!dependencies.adminSalesDataService)
|
||||
missing.push("adminSalesDataService");
|
||||
}
|
||||
if (ownsExports && !dependencies.exportService) {
|
||||
missing.push("exportService");
|
||||
if (ownsExports) {
|
||||
const privacyServiceWired =
|
||||
typeof dependencies.privacyGdprExportService?.resolveReady === "function" &&
|
||||
typeof dependencies.privacyGdprExportService?.resolveReclaimableProcessing === "function" &&
|
||||
typeof dependencies.privacyGdprExportService?.createAttempt === "function" &&
|
||||
typeof dependencies.privacyGdprExportService?.runExport === "function" &&
|
||||
typeof dependencies.privacyGdprExportService?.deleteArtifact === "function";
|
||||
if (importMode === "portal-gdpr-export") {
|
||||
if (!dependencies.portalGdprExportService) missing.push("portalGdprExportService");
|
||||
if (!privacyServiceWired) missing.push("privacyGdprExportService");
|
||||
} else {
|
||||
if (!dependencies.exportService) missing.push("exportService");
|
||||
if (!dependencies.portalGdprExportService) missing.push("portalGdprExportService");
|
||||
if (!privacyServiceWired) missing.push("privacyGdprExportService");
|
||||
}
|
||||
}
|
||||
if (isRequiredTenantHardDeleteRuntime(env)) {
|
||||
if (!env.MEILI_URL?.trim()) missing.push("MEILI_URL");
|
||||
if (!env.MEILI_API_KEY?.trim()) missing.push("MEILI_API_KEY");
|
||||
if (!dependencies.privacyTenantDeletionService) missing.push("privacyTenantDeletionService");
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Worker import/export queues require wired production services: missing ${missing.join(", ")}.`);
|
||||
const runtimeLabel = importMode === "portal-gdpr-export"
|
||||
? "KWP and portal/privacy GDPR queues"
|
||||
: importMode === "idle"
|
||||
? "KWP provider queue"
|
||||
: "Worker KWP/import/export queues";
|
||||
throw new Error(`${runtimeLabel} requires wired production services: missing ${missing.join(", ")}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function isRequiredTenantHardDeleteRuntime(env: EnvRecord): boolean {
|
||||
return (
|
||||
env.WORKER_QUEUE_GROUP === "general" &&
|
||||
(env.NODE_ENV === "production" ||
|
||||
env.RLS_RUNTIME_MODE === "required" ||
|
||||
env.SENTRY_ENVIRONMENT === "staging" ||
|
||||
env.SENTRY_ENVIRONMENT === "production")
|
||||
);
|
||||
}
|
||||
|
||||
function createJobLogger(logger: WorkerRuntimeLogger, fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerRuntimeLogger {
|
||||
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
|
||||
return typeof logger.child === "function" ? logger.child(bindings) : createBoundFallbackLogger(logger, bindings);
|
||||
@@ -622,7 +702,10 @@ function redactWorkerDiagnosticText(value: string): string {
|
||||
return value
|
||||
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
|
||||
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
||||
.replace(/(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
||||
.replace(
|
||||
/(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi,
|
||||
"$1$2[redacted]"
|
||||
)
|
||||
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
|
||||
.slice(0, 1_000);
|
||||
}
|
||||
Reference in new issue
Block a user