Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,461 @@
|
||||
import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@toolctrl/config";
|
||||
import { getDbClient } from "@toolctrl/db/client.server";
|
||||
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@toolctrl/queue";
|
||||
import { Queue } from "bullmq";
|
||||
import { getWorkerQueueNames } from "./queues.js";
|
||||
import { getMissingWorkerStorageEnv } from "./storage.js";
|
||||
|
||||
export type WorkerHealthEnv = EnvRecord;
|
||||
|
||||
export type WorkerHealthStatus = "disabled" | "ready" | "misconfigured";
|
||||
export type WorkerHealthRedisSource = "BULLMQ_REDIS_URL" | "REDIS_URL" | "none";
|
||||
|
||||
export type WorkerHealthDatabaseStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthRedisStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthStorageStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthImportStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthMailStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthPluginRuntimeStatus = "skipped" | "ready" | "misconfigured";
|
||||
export type WorkerHealthRequiredModeStatus = "skipped" | "ready" | "misconfigured";
|
||||
|
||||
export type WorkerHealthReport = {
|
||||
status: WorkerHealthStatus;
|
||||
revision: string;
|
||||
workerEnabled: boolean;
|
||||
workerGroup: WorkerQueueGroup | null;
|
||||
redisConfigured: boolean;
|
||||
redisSource: WorkerHealthRedisSource;
|
||||
redisParseStatus: "not_configured" | "valid" | "invalid";
|
||||
redisStatus: WorkerHealthRedisStatus;
|
||||
redisErrorCode?: string;
|
||||
databaseConfigured: boolean;
|
||||
databaseStatus: WorkerHealthDatabaseStatus;
|
||||
databaseErrorCode?: string;
|
||||
storageConfigured: boolean;
|
||||
storageStatus: WorkerHealthStorageStatus;
|
||||
mailjetConfigured: boolean;
|
||||
mailjetStatus: WorkerHealthMailStatus;
|
||||
importConfigured: boolean;
|
||||
importStatus: WorkerHealthImportStatus;
|
||||
pluginRuntimeConfigured: boolean;
|
||||
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus;
|
||||
queues: readonly QueueName[];
|
||||
errors: readonly string[];
|
||||
};
|
||||
|
||||
export type WorkerHealthCliIo = {
|
||||
stderr?: Pick<NodeJS.WriteStream, "write">;
|
||||
stdout: Pick<NodeJS.WriteStream, "write">;
|
||||
};
|
||||
|
||||
export type WorkerDatabaseProbe = () => Promise<void> | void;
|
||||
export type WorkerRedisProbe = (redisUrl: string) => Promise<void> | void;
|
||||
|
||||
export type WorkerHealthDependencies = {
|
||||
databaseProbe?: WorkerDatabaseProbe;
|
||||
redisProbe?: WorkerRedisProbe;
|
||||
};
|
||||
|
||||
export async function buildWorkerHealthReport(
|
||||
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
|
||||
dependencies: WorkerHealthDependencies = {}
|
||||
): Promise<WorkerHealthReport> {
|
||||
const result = loadWorkerConfig(env);
|
||||
const { config } = result;
|
||||
const shouldProbeDatabase = config.enabled && result.ok;
|
||||
const shouldProbeRedis = config.enabled && result.ok;
|
||||
const databaseConfigured = Boolean(env.DATABASE_URL);
|
||||
const [redisReadiness, databaseReadiness] = await Promise.all([
|
||||
checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
|
||||
checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
|
||||
]);
|
||||
const storageReadiness = checkStorageReadiness(config.enabled && env.EXPORT_RETENTION_QUEUE_ENABLED === "true", env);
|
||||
const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
|
||||
const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
|
||||
const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
|
||||
const requiredModeReadiness = checkRequiredGeneralWorkerReadiness(config.enabled && result.ok && getWorkerGroup(env) === "general", env);
|
||||
const status = toWorkerHealthStatus(
|
||||
config.enabled,
|
||||
result.ok,
|
||||
redisReadiness.status,
|
||||
databaseReadiness.status,
|
||||
storageReadiness.status,
|
||||
mailReadiness.status,
|
||||
importReadiness.status,
|
||||
pluginRuntimeReadiness.status,
|
||||
requiredModeReadiness.status
|
||||
);
|
||||
|
||||
return {
|
||||
status,
|
||||
revision: env.APP_REVISION || "unknown",
|
||||
workerEnabled: config.enabled,
|
||||
workerGroup: getWorkerGroup(env),
|
||||
redisConfigured: config.redis.configured,
|
||||
redisSource: config.redis.source,
|
||||
redisParseStatus: toHealthRedisParseStatus(config.redis.parseStatus),
|
||||
redisStatus: redisReadiness.status,
|
||||
...(redisReadiness.errorCode ? { redisErrorCode: redisReadiness.errorCode } : {}),
|
||||
databaseConfigured,
|
||||
databaseStatus: databaseReadiness.status,
|
||||
...(databaseReadiness.errorCode ? { databaseErrorCode: databaseReadiness.errorCode } : {}),
|
||||
storageConfigured: storageReadiness.configured,
|
||||
storageStatus: storageReadiness.status,
|
||||
mailjetConfigured: mailReadiness.configured,
|
||||
mailjetStatus: mailReadiness.status,
|
||||
importConfigured: importReadiness.configured,
|
||||
importStatus: importReadiness.status,
|
||||
pluginRuntimeConfigured: pluginRuntimeReadiness.configured,
|
||||
pluginRuntimeStatus: pluginRuntimeReadiness.status,
|
||||
queues: getWorkerQueueNames(env),
|
||||
errors: [
|
||||
...result.diagnostics.map(formatWorkerConfigDiagnostic),
|
||||
...storageReadiness.errors,
|
||||
...mailReadiness.errors,
|
||||
...importReadiness.errors,
|
||||
...pluginRuntimeReadiness.errors,
|
||||
...requiredModeReadiness.errors,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function getWorkerGroup(env: WorkerHealthEnv): WorkerQueueGroup | null {
|
||||
const group = env.WORKER_QUEUE_GROUP;
|
||||
return typeof group === "string" && isWorkerQueueGroup(group) ? group : null;
|
||||
}
|
||||
|
||||
export async function runWorkerHealthCli(
|
||||
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
|
||||
io: WorkerHealthCliIo = process,
|
||||
dependencies: WorkerHealthDependencies = {}
|
||||
): Promise<number> {
|
||||
const report = await buildWorkerHealthReport(env, dependencies);
|
||||
io.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
return report.status === "misconfigured" ? 1 : 0;
|
||||
}
|
||||
|
||||
function toWorkerHealthStatus(
|
||||
workerEnabled: boolean,
|
||||
redisReady: boolean,
|
||||
redisStatus: WorkerHealthRedisStatus,
|
||||
databaseStatus: WorkerHealthDatabaseStatus,
|
||||
storageStatus: WorkerHealthStorageStatus,
|
||||
mailStatus: WorkerHealthMailStatus,
|
||||
importStatus: WorkerHealthImportStatus,
|
||||
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus,
|
||||
requiredModeStatus: WorkerHealthRequiredModeStatus
|
||||
): WorkerHealthStatus {
|
||||
if (!workerEnabled) {
|
||||
return "disabled";
|
||||
}
|
||||
|
||||
if (
|
||||
!redisReady ||
|
||||
redisStatus === "misconfigured" ||
|
||||
databaseStatus === "misconfigured" ||
|
||||
storageStatus === "misconfigured" ||
|
||||
mailStatus === "misconfigured" ||
|
||||
importStatus === "misconfigured" ||
|
||||
pluginRuntimeStatus === "misconfigured" ||
|
||||
requiredModeStatus === "misconfigured"
|
||||
) {
|
||||
return "misconfigured";
|
||||
}
|
||||
|
||||
return "ready";
|
||||
}
|
||||
|
||||
function checkPluginRuntimeReadiness(
|
||||
shouldCheck: boolean,
|
||||
workerGroup: WorkerQueueGroup | null,
|
||||
env: WorkerHealthEnv
|
||||
): { configured: boolean; errors: string[]; status: WorkerHealthPluginRuntimeStatus } {
|
||||
const productionRuntimeEnabled = env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
|
||||
if (!shouldCheck || !productionRuntimeEnabled || (workerGroup !== "general" && workerGroup !== "plugin")) {
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
if (!hasTrimmedValue(env.PLUGIN_TRUSTED_PUBLIC_KEYS)) {
|
||||
errors.push("PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.");
|
||||
}
|
||||
|
||||
if (workerGroup === "general") {
|
||||
if (!hasTrimmedValue(env.BULLMQ_REDIS_URL) && !hasTrimmedValue(env.REDIS_URL)) {
|
||||
errors.push(
|
||||
"BULLMQ_REDIS_URL:missing-plugin-runtime-redis:Required for plugin hook fanout and capability-use subscribers when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
||||
);
|
||||
}
|
||||
if (env.OUTBOX_DISPATCHER_ENABLED !== "true") {
|
||||
errors.push(
|
||||
"OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
||||
);
|
||||
}
|
||||
if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") {
|
||||
errors.push(
|
||||
"PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
||||
);
|
||||
}
|
||||
if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") {
|
||||
errors.push(
|
||||
"PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { configured: true, errors, status: errors.length > 0 ? "misconfigured" : "ready" };
|
||||
}
|
||||
|
||||
function isMailProviderRequired(env: WorkerHealthEnv): boolean {
|
||||
return env.WORKER_QUEUE_GROUP === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true";
|
||||
}
|
||||
|
||||
function checkRequiredGeneralWorkerReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { errors: string[]; status: WorkerHealthRequiredModeStatus } {
|
||||
if (!shouldCheck) {
|
||||
return { errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
if (env.OUTBOX_DISPATCHER_REQUIRED === "true" && env.OUTBOX_DISPATCHER_ENABLED !== "true") {
|
||||
errors.push("OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true.");
|
||||
}
|
||||
|
||||
const workflowTriggerRequired = env.WORKFLOW_TRIGGER_OUTBOX_REQUIRED === "true";
|
||||
const workflowTriggerEnabled = env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true";
|
||||
const workflowTriggerModeLabel = workflowTriggerRequired ? "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true" : "WORKFLOW_TRIGGER_OUTBOX_ENABLED=true";
|
||||
|
||||
if (workflowTriggerRequired || workflowTriggerEnabled) {
|
||||
if (env.OUTBOX_DISPATCHER_ENABLED !== "true" && env.OUTBOX_DISPATCHER_REQUIRED !== "true") {
|
||||
errors.push(`OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`);
|
||||
}
|
||||
|
||||
if (workflowTriggerRequired && !workflowTriggerEnabled) {
|
||||
errors.push(
|
||||
"WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true."
|
||||
);
|
||||
}
|
||||
|
||||
const executorMode = parseWorkflowTriggerExecutorModeForHealth(env.WORKFLOW_TRIGGER_EXECUTOR_MODE);
|
||||
if (!executorMode) {
|
||||
errors.push(
|
||||
`WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`
|
||||
);
|
||||
} else if (workflowTriggerExecutorNeedsHttpBridge(executorMode)) {
|
||||
if (!hasTrimmedValue(env.API_ACCESS_TOKEN)) {
|
||||
errors.push(
|
||||
`API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
|
||||
);
|
||||
}
|
||||
|
||||
if (!hasTrimmedValue(env.WORKFLOW_TRIGGER_EXECUTOR_URL) && !hasTrimmedValue(env.APP_URL) && !hasTrimmedValue(env.WEB_ORIGIN)) {
|
||||
errors.push(
|
||||
`WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { errors, status: errors.length > 0 ? "misconfigured" : "ready" };
|
||||
}
|
||||
|
||||
function parseWorkflowTriggerExecutorModeForHealth(value: unknown): "http" | "native" | "hybrid" | null {
|
||||
if (typeof value !== "string" || value.trim().length === 0) {
|
||||
return "http";
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized === "http" || normalized === "native" || normalized === "hybrid" ? normalized : null;
|
||||
}
|
||||
|
||||
function workflowTriggerExecutorNeedsHttpBridge(mode: "http" | "native" | "hybrid"): boolean {
|
||||
return mode !== "native";
|
||||
}
|
||||
|
||||
function hasTrimmedValue(value: string | undefined): boolean {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function checkMailReadiness(
|
||||
shouldCheck: boolean,
|
||||
configured: boolean,
|
||||
required: boolean,
|
||||
env: WorkerHealthEnv
|
||||
): { configured: boolean; errors: string[]; status: WorkerHealthMailStatus } {
|
||||
if (!shouldCheck) {
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
if (!configured) {
|
||||
if (required) {
|
||||
const missing = ["MAILJET_API_KEY", "MAILJET_SECRET_KEY", "MAILJET_FROM_EMAIL"].filter((name) => !env[name]?.trim());
|
||||
return {
|
||||
configured: false,
|
||||
errors: missing.map(
|
||||
(name) => `${name}:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.`
|
||||
),
|
||||
status: "misconfigured",
|
||||
};
|
||||
}
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
return { configured: true, errors: [], status: "ready" };
|
||||
}
|
||||
|
||||
function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthImportStatus } {
|
||||
if (!shouldCheck) {
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
|
||||
const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
|
||||
if (!importEnabled && !runtimeReady) {
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
if (!importEnabled || !runtimeReady) {
|
||||
const errors: string[] = [];
|
||||
if (!importEnabled) {
|
||||
errors.push(
|
||||
"IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume toolctrl.imports/toolctrl.exports."
|
||||
);
|
||||
}
|
||||
if (!runtimeReady) {
|
||||
errors.push(
|
||||
"IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired."
|
||||
);
|
||||
}
|
||||
return {
|
||||
configured: false,
|
||||
errors,
|
||||
status: "misconfigured",
|
||||
};
|
||||
}
|
||||
|
||||
return { configured: true, errors: [], status: "ready" };
|
||||
}
|
||||
|
||||
function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
|
||||
if (!shouldCheck) {
|
||||
return { configured: false, errors: [], status: "skipped" };
|
||||
}
|
||||
|
||||
const missing = getMissingWorkerStorageEnv(env);
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
configured: false,
|
||||
errors: missing.map((name) => `${name}:missing-storage-config:Required when EXPORT_RETENTION_QUEUE_ENABLED=true.`),
|
||||
status: "misconfigured",
|
||||
};
|
||||
}
|
||||
|
||||
return { configured: true, errors: [], status: "ready" };
|
||||
}
|
||||
|
||||
async function checkRedisReadiness(
|
||||
shouldProbe: boolean,
|
||||
redisUrl: string | undefined,
|
||||
redisProbe: WorkerRedisProbe = defaultRedisProbe
|
||||
): Promise<{ status: WorkerHealthRedisStatus; errorCode?: string }> {
|
||||
if (!shouldProbe || !redisUrl) {
|
||||
return { status: "skipped" };
|
||||
}
|
||||
|
||||
try {
|
||||
await redisProbe(redisUrl);
|
||||
return { status: "ready" };
|
||||
} catch (error) {
|
||||
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error, "REDIS_READINESS_FAILED") };
|
||||
}
|
||||
}
|
||||
|
||||
async function checkDatabaseReadiness(
|
||||
shouldProbe: boolean,
|
||||
databaseProbe: WorkerDatabaseProbe = defaultDatabaseProbe
|
||||
): Promise<{ status: WorkerHealthDatabaseStatus; errorCode?: string }> {
|
||||
if (!shouldProbe) {
|
||||
return { status: "skipped" };
|
||||
}
|
||||
|
||||
try {
|
||||
await databaseProbe();
|
||||
return { status: "ready" };
|
||||
} catch (error) {
|
||||
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error) };
|
||||
}
|
||||
}
|
||||
|
||||
async function defaultDatabaseProbe(): Promise<void> {
|
||||
const db = getDbClient();
|
||||
|
||||
try {
|
||||
await db.$queryRaw`SELECT 1`;
|
||||
} finally {
|
||||
await db.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
async function defaultRedisProbe(redisUrl: string): Promise<void> {
|
||||
const queue = new Queue("__toolctrl_worker_health__", {
|
||||
connection: parseRedisUrl(redisUrl),
|
||||
});
|
||||
|
||||
try {
|
||||
await withTimeout(queue.waitUntilReady(), 2_000);
|
||||
} finally {
|
||||
await queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
function toHealthRedisParseStatus(status: WorkerConfigRedisParseStatus): WorkerHealthReport["redisParseStatus"] {
|
||||
if (status === "valid") {
|
||||
return "valid";
|
||||
}
|
||||
|
||||
if (status === "not-provided") {
|
||||
return "not_configured";
|
||||
}
|
||||
|
||||
return "invalid";
|
||||
}
|
||||
|
||||
function toRedactedDependencyErrorCode(error: unknown, fallback = "DATABASE_READINESS_FAILED"): string {
|
||||
if (typeof error === "object" && error !== null && "code" in error) {
|
||||
const code = (error as { code?: unknown }).code;
|
||||
if (typeof code === "string" && /^[A-Z0-9_-]{1,64}$/i.test(code)) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
|
||||
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => reject(Object.assign(new Error("Redis readiness timed out."), { code: "REDIS_TIMEOUT" })), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function formatWorkerConfigDiagnostic(diagnostic: WorkerConfigDiagnostic): string {
|
||||
return `${diagnostic.name}:${diagnostic.code}:${diagnostic.message}`;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runWorkerHealthCli()
|
||||
.then((exitCode) => {
|
||||
process.exitCode = exitCode;
|
||||
})
|
||||
.catch(() => {
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user