461 lines
18 KiB
JavaScript
461 lines
18 KiB
JavaScript
import { describe, expect, it } from "vitest";
|
|
import {
|
|
GROUPS,
|
|
WORKER_ENV_OVERRIDE_KEYS,
|
|
buildWorkerApplicationBody,
|
|
desiredWorkerEnv,
|
|
envEntriesToRecord,
|
|
getWorkerAppConfigDrift,
|
|
getWorkerEnvDrift,
|
|
} from "./coolify-ensure-worker-groups.mjs";
|
|
|
|
function group(name) {
|
|
const found = GROUPS.find((item) => item.name === name);
|
|
if (!found) {
|
|
throw new Error(`Missing test group ${name}`);
|
|
}
|
|
return found;
|
|
}
|
|
|
|
function entriesFromRecord(record) {
|
|
return Object.entries(record).map(([key, value]) => ({ key, value }));
|
|
}
|
|
|
|
function desiredWorkerEnvWithRuntime(workerGroup, overrides = {}) {
|
|
return {
|
|
...desiredWorkerEnv(workerGroup),
|
|
DATABASE_URL: "postgres://redacted",
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe("coolify worker group env drift", () => {
|
|
it("provisions worker apps with readiness health checks on the group health port", () => {
|
|
expect(
|
|
buildWorkerApplicationBody({
|
|
group: group("mail"),
|
|
projectUuid: "project_1",
|
|
serverUuid: "server_1",
|
|
destinationUuid: "destination_1",
|
|
environmentName: "production",
|
|
environmentUuid: "environment_1",
|
|
workerImage: "registry.example.test/toolctrl/worker",
|
|
imageTag: "sha-abc123",
|
|
})
|
|
).toMatchObject({
|
|
project_uuid: "project_1",
|
|
server_uuid: "server_1",
|
|
destination_uuid: "destination_1",
|
|
environment_name: "production",
|
|
environment_uuid: "environment_1",
|
|
docker_registry_image_name: "registry.example.test/toolctrl/worker",
|
|
docker_registry_image_tag: "sha-abc123",
|
|
name: "worker-mail",
|
|
ports_exposes: "9102",
|
|
health_check_enabled: true,
|
|
health_check_path: "/readyz",
|
|
health_check_port: "9102",
|
|
health_check_host: "127.0.0.1",
|
|
connect_to_docker_network: true,
|
|
});
|
|
});
|
|
|
|
it("builds the expected mail worker environment", () => {
|
|
expect(desiredWorkerEnv(group("mail"))).toEqual({
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "mail",
|
|
WORKER_HEALTH_PORT: "9102",
|
|
WORKER_ADMIN_PORT: "9202",
|
|
WORKER_CONCURRENCY_MAIL: "5",
|
|
MARKETING_SEQUENCES_ENABLED: "false",
|
|
TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
|
|
});
|
|
});
|
|
|
|
it("builds the expected general worker production outbox environment", () => {
|
|
expect(desiredWorkerEnv(group("general"))).toMatchObject({
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "general",
|
|
WORKER_HEALTH_PORT: "9101",
|
|
WORKER_ADMIN_PORT: "9201",
|
|
WORKER_CONCURRENCY_GENERAL: "10",
|
|
OUTBOX_DISPATCHER_ENABLED: "true",
|
|
OUTBOX_DISPATCHER_REQUIRED: "true",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "http",
|
|
WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
|
|
WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY: "5",
|
|
WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
|
|
});
|
|
});
|
|
|
|
it("protects every managed worker env key from baseline-env cloning", () => {
|
|
for (const item of GROUPS) {
|
|
for (const key of Object.keys(desiredWorkerEnv(item))) {
|
|
expect(WORKER_ENV_OVERRIDE_KEYS.has(key)).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("normalizes quoted Coolify env values", () => {
|
|
expect(
|
|
envEntriesToRecord([
|
|
{ key: "WORKER_ENABLED", value: '"true"' },
|
|
{ key: "WORKER_QUEUE_GROUP", value: "'mail'" },
|
|
])
|
|
).toEqual({
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "mail",
|
|
});
|
|
});
|
|
|
|
it("returns no drift when an existing worker app matches the desired env", () => {
|
|
const mail = group("mail");
|
|
const drift = getWorkerEnvDrift({ uuid: "worker-mail", name: "worker-mail" }, mail, entriesFromRecord(desiredWorkerEnvWithRuntime(mail)));
|
|
|
|
expect(drift).toEqual([]);
|
|
});
|
|
|
|
it("returns no app config drift when an existing worker app uses readyz on the group health port", () => {
|
|
const mail = group("mail");
|
|
|
|
expect(
|
|
getWorkerAppConfigDrift(
|
|
{
|
|
uuid: "worker-mail",
|
|
name: "worker-mail",
|
|
ports_exposes: "9102",
|
|
health_check_enabled: true,
|
|
health_check_path: "/readyz",
|
|
health_check_port: 9102,
|
|
health_check_host: "127.0.0.1",
|
|
},
|
|
mail
|
|
)
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("reports existing worker app health-check drift", () => {
|
|
const mail = group("mail");
|
|
|
|
expect(
|
|
getWorkerAppConfigDrift(
|
|
{
|
|
uuid: "worker-mail",
|
|
name: "worker-mail",
|
|
ports_exposes: "9101",
|
|
health_check_enabled: false,
|
|
health_check_path: "/healthz",
|
|
health_check_port: "9101",
|
|
health_check_host: "0.0.0.0",
|
|
},
|
|
mail
|
|
)
|
|
).toEqual([
|
|
expect.objectContaining({ key: "ports_exposes", expected: "9102", actual: "9101", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_enabled", expected: "true", actual: "false", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_path", expected: "/readyz", actual: "/healthz", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_port", expected: "9102", actual: "9101", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_host", expected: "127.0.0.1", actual: "0.0.0.0", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("accepts a fully configured general worker HTTP bridge", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts WEB_ORIGIN as the general worker HTTP bridge fallback", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WEB_ORIGIN: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("does not require plugin hook fanout before the production plugin runtime gate flips", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "false",
|
|
PLUGIN_HOOK_FANOUT_ENABLED: "false",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("requires plugin hook fanout fail-closed when production plugin runtime is enabled", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
|
|
PLUGIN_HOOK_FANOUT_ENABLED: "false",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({
|
|
key: "PLUGIN_HOOK_FANOUT_ENABLED",
|
|
expected: "unset or true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true",
|
|
actual: "false",
|
|
group: "general",
|
|
}),
|
|
expect.objectContaining({
|
|
key: "PLUGIN_HOOK_FANOUT_REQUIRED",
|
|
expected: "true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("accepts production plugin runtime when general worker hook fanout is required", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
|
|
PLUGIN_HOOK_FANOUT_REQUIRED: "true",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts native workflow execution as an explicit general worker opt-in", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "native",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("normalizes native workflow execution mode like the worker runtime", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: " NATIVE ",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts hybrid workflow execution when the HTTP fallback is configured", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("normalizes hybrid workflow execution mode before checking HTTP fallback dependencies", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: " HYBRID ",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("requires the HTTP fallback dependencies for hybrid workflow execution", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("rejects legacy-only workflow executor token configuration", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("rejects legacy workflow executor token even when API_ACCESS_TOKEN is configured", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({
|
|
key: "WORKFLOW_TRIGGER_EXECUTOR_TOKEN",
|
|
expected: "unset; API_ACCESS_TOKEN is the supported workflow bridge credential",
|
|
actual: "<configured>",
|
|
group: "general",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("reports disabled existing worker apps", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_ENABLED: "false" };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_ENABLED", expected: "true", actual: "false", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("reports wrong queue groups", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_QUEUE_GROUP: "general" };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_QUEUE_GROUP", expected: "mail", actual: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("reports mail workers without the marketing sequence killswitch", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail) };
|
|
delete entries.MARKETING_SEQUENCES_ENABLED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "MARKETING_SEQUENCES_ENABLED", expected: "false", actual: "<missing>", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("reports mail workers without fail-loud transactional outbox bootstrap", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail) };
|
|
delete entries.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED", expected: "true", actual: "<missing>", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("requires DATABASE_URL and BULLMQ_REDIS_URL on every managed worker group", () => {
|
|
for (const workerGroup of GROUPS) {
|
|
const missingDatabase = desiredWorkerEnvWithRuntime(workerGroup);
|
|
delete missingDatabase.DATABASE_URL;
|
|
const missingRedis = desiredWorkerEnvWithRuntime(workerGroup);
|
|
delete missingRedis.BULLMQ_REDIS_URL;
|
|
missingRedis.REDIS_URL = "redis://fallback-not-accepted-for-coolify-workers";
|
|
|
|
expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingDatabase))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "<missing>", group: workerGroup.name }),
|
|
])
|
|
);
|
|
expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingRedis))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
key: "BULLMQ_REDIS_URL",
|
|
expected: "configured BULLMQ_REDIS_URL",
|
|
actual: "<missing; REDIS_URL fallback not accepted for managed workers>",
|
|
group: workerGroup.name,
|
|
}),
|
|
])
|
|
);
|
|
}
|
|
});
|
|
|
|
it("reports missing required general-worker outbox activation", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
OUTBOX_DISPATCHER_REQUIRED: "false",
|
|
};
|
|
delete entries.WORKFLOW_TRIGGER_EXECUTOR_MODE;
|
|
entries.WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY = "1";
|
|
delete entries.WORKFLOW_TRIGGER_OUTBOX_REQUIRED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "OUTBOX_DISPATCHER_REQUIRED", expected: "true", actual: "false", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_MODE", expected: "http|hybrid|native", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY", expected: "5", actual: "1", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_REQUIRED", expected: "true", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("reports missing general worker Redis and HTTP bridge dependencies", () => {
|
|
const general = group("general");
|
|
const entries = { ...desiredWorkerEnv(general) };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "BULLMQ_REDIS_URL", expected: "configured BULLMQ_REDIS_URL", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("reports swapped or missing worker ports", () => {
|
|
const plugin = group("plugin");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(plugin), WORKER_HEALTH_PORT: "9205" };
|
|
delete entries.WORKER_ADMIN_PORT;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-plugin" }, plugin, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_HEALTH_PORT", expected: "9105", actual: "9205" }),
|
|
expect.objectContaining({ key: "WORKER_ADMIN_PORT", expected: "9205", actual: "<missing>" }),
|
|
]);
|
|
});
|
|
|
|
it("reports missing or wrong group concurrency", () => {
|
|
const webhook = group("webhook");
|
|
const wrong = { ...desiredWorkerEnvWithRuntime(webhook), WORKER_CONCURRENCY_WEBHOOK: "5" };
|
|
const missing = { ...desiredWorkerEnvWithRuntime(webhook) };
|
|
delete missing.WORKER_CONCURRENCY_WEBHOOK;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(wrong))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "5" }),
|
|
]);
|
|
expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(missing))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "<missing>" }),
|
|
]);
|
|
});
|
|
});
|