Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/lib/run-command-with-timeout.test.mjs
T

590 lines
22 KiB
JavaScript

import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync, spawn, spawnSync } from "node:child_process";
import { afterEach, describe, it } from "vitest";
const runnerPath = path.resolve("scripts/lib/run-command-with-timeout.mjs");
const helperPath = path.resolve("scripts/lib/docker-runtime-smoke.sh");
const roleFixturePath = path.resolve("scripts/lib/docker-rls-role-fixture.sh");
const temporaryDirectories = [];
const remainingProcesses = new Set();
afterEach(() => {
for (const pid of remainingProcesses) {
try {
process.kill(pid, "SIGKILL");
} catch {
// The timeout runner should already have reaped the fixture process.
}
}
remainingProcesses.clear();
for (const directory of temporaryDirectories.splice(0)) {
fs.rmSync(directory, { recursive: true, force: true });
}
});
function runRunner(stage, timeoutSeconds, command, { graceSeconds = 0.1, env = process.env } = {}) {
return spawnSync(
process.execPath,
[runnerPath, "--stage", stage, "--timeout-seconds", String(timeoutSeconds), "--grace-seconds", String(graceSeconds), "--", ...command],
{ encoding: "utf8", env, timeout: 5_000 },
);
}
function createProcessTreeFixture() {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-timeout-tree-"));
temporaryDirectories.push(directory);
const pidFile = path.join(directory, "pids.json");
const grandchildPath = path.join(directory, "grandchild.mjs");
const parentPath = path.join(directory, "parent.mjs");
fs.writeFileSync(grandchildPath, ['import process from "node:process";', 'process.on("SIGTERM", () => {});', "setInterval(() => {}, 1_000);", ""].join("\n"));
fs.writeFileSync(
parentPath,
[
'import fs from "node:fs";',
'import process from "node:process";',
'import { spawn } from "node:child_process";',
"const [pidFile, grandchildPath] = process.argv.slice(2);",
'const grandchild = spawn(process.execPath, [grandchildPath], { stdio: "ignore" });',
"fs.writeFileSync(pidFile, JSON.stringify({ parent: process.pid, grandchild: grandchild.pid }));",
'process.on("SIGTERM", () => {});',
"setInterval(() => {}, 1_000);",
"",
].join("\n")
);
return { parentPath, pidFile, grandchildPath };
}
function readFixturePids(pidFile) {
const pids = JSON.parse(fs.readFileSync(pidFile, "utf8"));
remainingProcesses.add(pids.parent);
remainingProcesses.add(pids.grandchild);
return pids;
}
async function waitForFile(filePath, timeoutMilliseconds = 2_000) {
const deadline = Date.now() + timeoutMilliseconds;
while (Date.now() < deadline) {
if (fs.existsSync(filePath)) {
return;
}
await new Promise((resolve) => setTimeout(resolve, 20));
}
throw new Error("process-tree fixture did not become ready");
}
function isTerminatedLinuxProcessStat(stat) {
const commandEnd = stat.lastIndexOf(")");
const state = commandEnd >= 0 ? stat.slice(commandEnd + 2).split(" ", 1)[0] : "";
return state === "Z" || state === "X" || state === "x";
}
async function waitForProcessExit(pid, timeoutMilliseconds = 2_000) {
const deadline = Date.now() + timeoutMilliseconds;
while (Date.now() < deadline) {
try {
process.kill(pid, 0);
} catch (error) {
if (error?.code === "ESRCH") {
remainingProcesses.delete(pid);
return;
}
throw error;
}
// Linux keeps an exited process in /proc while its parent (or container PID 1)
// has not reaped the zombie yet. kill(pid, 0) still succeeds in that state even
// though the process can no longer execute, so it is valid evidence that the
// timeout runner killed the fixture. This occurs on the Gitea act runner, whose
// job container does not necessarily reap adopted grandchildren immediately.
if (process.platform === "linux") {
try {
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
if (isTerminatedLinuxProcessStat(stat)) {
remainingProcesses.delete(pid);
return;
}
} catch (error) {
if (error?.code === "ENOENT") {
remainingProcesses.delete(pid);
return;
}
throw error;
}
}
await new Promise((resolve) => setTimeout(resolve, 20));
}
throw new Error("timed-out process remained runnable");
}
describe("run-command-with-timeout", () => {
it("recognizes Linux zombie and dead states without accepting runnable states", () => {
for (const state of ["Z", "X", "x"]) {
assert.equal(isTerminatedLinuxProcessStat(`123 (fixture with ) paren) ${state} 1 2 3`), true);
}
for (const state of ["R", "S", "D", "T"]) {
assert.equal(isTerminatedLinuxProcessStat(`123 (fixture) ${state} 1 2 3`), false);
}
assert.equal(isTerminatedLinuxProcessStat("malformed"), false);
});
it("forwards output and reports stage-only metadata for successful commands", () => {
const secret = "secret-sentinel-success-42";
const result = runRunner(
"worker:inspect",
2,
[process.execPath, "-e", 'process.stdout.write("child-stdout\\n"); process.stderr.write("child-stderr\\n");', secret],
{ env: { ...process.env, SECRET_SENTINEL: secret } }
);
assert.equal(result.status, 0);
assert.equal(result.stdout, "child-stdout\n");
assert.match(result.stderr, /child-stderr/u);
assert.match(result.stderr, /\[docker-smoke\] START stage=worker:inspect timeout_seconds=2 grace_seconds=0\.1/u);
assert.match(result.stderr, /\[docker-smoke\] PASS stage=worker:inspect duration_ms=\d+ exit_code=0/u);
assert.doesNotMatch(result.stderr, new RegExp(secret, "u"));
});
it("preserves a failing command's exit code and streams", () => {
const result = runRunner("worker:migrate", 2, [
process.execPath,
"-e",
'process.stdout.write("migration-stdout\\n"); process.stderr.write("migration-stderr\\n"); process.exit(7);',
]);
assert.equal(result.status, 7);
assert.equal(result.stdout, "migration-stdout\n");
assert.match(result.stderr, /migration-stderr/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=worker:migrate duration_ms=\d+ exit_code=7/u);
});
it("returns 124 and kills a SIGTERM-ignoring process group", async () => {
const fixture = createProcessTreeFixture();
const result = runRunner("worker:timeout", 0.25, [process.execPath, fixture.parentPath, fixture.pidFile, fixture.grandchildPath], { graceSeconds: 0.1 });
const pids = readFixturePids(fixture.pidFile);
assert.equal(result.status, 124);
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:timeout duration_ms=\d+ exit_code=124/u);
await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]);
});
it("terminates the child group and exits signal-typically when the suite sends SIGTERM", async () => {
const fixture = createProcessTreeFixture();
const runner = spawn(
process.execPath,
[
runnerPath,
"--stage",
"suite:interrupt",
"--timeout-seconds",
"10",
"--grace-seconds",
"0.1",
"--",
process.execPath,
fixture.parentPath,
fixture.pidFile,
fixture.grandchildPath,
],
{ stdio: ["ignore", "pipe", "pipe"] }
);
let stderr = "";
runner.stderr.setEncoding("utf8");
runner.stderr.on("data", (chunk) => {
stderr += chunk;
});
try {
await waitForFile(fixture.pidFile);
const pids = readFixturePids(fixture.pidFile);
runner.kill("SIGTERM");
const result = await new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error("timeout runner did not exit after SIGTERM")), 3_000);
runner.once("error", reject);
runner.once("close", (code, signal) => {
clearTimeout(timeout);
resolve({ code, signal });
});
});
assert.deepEqual(result, { code: 143, signal: null });
assert.match(stderr, /\[docker-smoke\] FAIL stage=suite:interrupt duration_ms=\d+ signal=SIGTERM exit_code=143/u);
await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]);
} finally {
if (runner.exitCode === null && runner.signalCode === null) {
runner.kill("SIGKILL");
}
}
});
it("rejects malformed metadata labels without echoing them", () => {
const secret = "secret sentinel invalid stage";
const result = spawnSync(process.execPath, [runnerPath, "--stage", secret, "--timeout-seconds", "1", "--", process.execPath, "-e", ""], {
encoding: "utf8",
});
assert.equal(result.status, 2);
assert.match(result.stderr, /reason=invalid_arguments/u);
assert.doesNotMatch(result.stderr, new RegExp(secret, "u"));
});
});
describe("docker-runtime-smoke helper", () => {
it("replays captured stdout to stderr when the command fails", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-capture-"));
temporaryDirectories.push(fixture);
const scriptPath = path.join(fixture, "capture.sh");
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
'captured=""',
"if docker_smoke_capture captured capture.failure 2 bash -c 'printf captured-stdout; printf child-stderr >&2; exit 7'; then",
" exit 99",
"else",
" status=$?",
"fi",
'printf "captured-variable=%s\\n" "$captured"',
'exit "$status"',
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], { encoding: "utf8", timeout: 5_000 });
assert.equal(result.status, 7);
assert.equal(result.stdout, "captured-variable=\n");
assert.match(result.stderr, /child-stderr/u);
assert.match(result.stderr, /captured-stdout/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=capture.failure duration_ms=\d+ exit_code=7/u);
});
it("requires an in-container rejection marker and rejects Docker transport failures", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "rejection.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "${FAKE_DOCKER_TRANSPORT_FAILURE:-0}" == "1" ]]; then',
' printf "Docker transport unavailable\\n" >&2',
" exit 1",
"fi",
'[[ "$1" == "exec" ]] || exit 2',
"shift",
"shift",
'"$@"',
"",
].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_expect_docker_exec_rejection rejection.product 2 fixture-container bash -c 'printf product-rejected >&2; exit 1'",
"",
].join("\n"),
{ mode: 0o755 },
);
const baseEnv = { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` };
const productRejection = spawnSync("bash", [scriptPath], { encoding: "utf8", env: baseEnv, timeout: 5_000 });
const transportFailure = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...baseEnv, FAKE_DOCKER_TRANSPORT_FAILURE: "1" },
timeout: 5_000,
});
assert.equal(productRejection.status, 0);
assert.match(productRejection.stderr, /\[docker-smoke\] ASSERT stage=rejection\.product outcome=expected_rejection/u);
assert.equal(transportFailure.status, 1);
assert.match(transportFailure.stderr, /Docker transport unavailable/u);
assert.match(transportFailure.stderr, /\[docker-smoke\] FAIL stage=rejection\.product duration_ms=\d+ exit_code=1/u);
assert.doesNotMatch(transportFailure.stderr, /outcome=expected_rejection/u);
});
it("rejects an unexpectedly successful in-container negative assertion", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-success-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "rejection-success.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
["#!/usr/bin/env bash", '[[ "$1" == "exec" ]] || exit 2', "shift", "shift", '"$@"', ""].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_expect_docker_exec_rejection rejection.unexpected-success 2 fixture-container bash -c 'exit 0'",
"",
].join("\n"),
{ mode: 0o755 },
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 64);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=rejection\.unexpected-success duration_ms=\d+ exit_code=64/u);
assert.doesNotMatch(result.stderr, /outcome=expected_rejection/u);
});
it("bounds hanging Docker cleanup while preserving the original exit status", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 });
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"export DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS=0.2",
"export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1",
"cleanup() {",
" docker_smoke_cleanup_container cleanup.container fixture-container",
"}",
"docker_smoke_install_cleanup_traps cleanup",
"exit 23",
"",
].join("\n"),
{ mode: 0o755 }
);
const startedAt = Date.now();
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 23);
assert.ok(Date.now() - startedAt < 3_000, "cleanup must be bounded by the helper timeout");
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=cleanup\.container duration_ms=\d+ exit_code=124/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.container reason=cleanup_incomplete exit_code=124/u);
});
it("treats already absent cleanup resources as an idempotent success", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-absent-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-absent.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "$1" == "container" && "$2" == "rm" ]]; then',
' printf "Error response from daemon: No such container: %s\\n" "$4" >&2',
" exit 1",
"fi",
'if [[ "$1" == "container" && "$2" == "inspect" ]]; then',
' printf "Error: No such object: %s\\n" "$3" >&2',
" exit 1",
"fi",
"exit 2",
"",
].join("\n"),
{ mode: 0o755 }
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_cleanup_container cleanup.absent absent-container",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 0);
assert.match(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.absent duration_ms=\d+ exit_code=0/u);
assert.doesNotMatch(result.stderr, /(?:FAIL|WARN|RETAINED|No such)/u);
});
it("does not misclassify an unknown Docker context failure as an absent resource", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-context-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-context.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'printf "unable to resolve docker endpoint: context review-context: context not found\\n" >&2',
"exit 1",
"",
].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_cleanup_container cleanup.context fixture-container",
"",
].join("\n"),
{ mode: 0o755 },
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 0);
assert.match(result.stderr, /context review-context: context not found/u);
assert.match(result.stderr, /\[docker-smoke\] CLEANUP_CHECK_FAILED kind=container name=fixture-container/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.context reason=cleanup_incomplete exit_code=1/u);
assert.doesNotMatch(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.context/u);
});
it("reports retained cleanup resources without replacing the original exit status", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-retained-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-retained.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "$1" == "container" && "$2" == "rm" ]]; then',
' printf "removal already in progress\\n" >&2',
" exit 1",
"fi",
'if [[ "$1" == "container" && "$2" == "inspect" ]]; then',
" exit 0",
"fi",
"exit 2",
"",
].join("\n"),
{ mode: 0o755 }
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"cleanup() {",
" docker_smoke_cleanup_container cleanup.retained retained-container",
"}",
"docker_smoke_install_cleanup_traps cleanup",
"exit 23",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 23);
assert.match(result.stderr, /\[docker-smoke\] RETAINED kind=container name=retained-container/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=cleanup\.retained duration_ms=\d+ exit_code=1/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.retained reason=cleanup_incomplete exit_code=1/u);
});
it("bounds RLS role provisioning with a named stage without exposing fixture credentials", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-role-provision-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "provision.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 });
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
`source ${JSON.stringify(roleFixturePath)}`,
"export DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS=0.2",
"export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1",
"provision_docker_smoke_rls_roles worker:provision-rls-roles fixture-db fixture-admin fixture-database migrator-secret app-secret worker-secret authentik-secret",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 124);
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:provision-rls-roles duration_ms=\d+ exit_code=124/u);
assert.doesNotMatch(result.stderr, /(?:migrator|app|worker|authentik)-secret/u);
});
it("exports overridable timeout defaults and remains valid Bash syntax", () => {
execFileSync("bash", ["-n", helperPath]);
const result = spawnSync(
"bash",
[
"-c",
[
"set -euo pipefail",
"export DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS=41",
`source ${JSON.stringify(helperPath)}`,
'printf \'%s,%s,%s,%s,%s,%s\' "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" "$DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS" "$DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS" "$DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS"',
].join("\n"),
],
{ encoding: "utf8" }
);
assert.equal(result.status, 0);
assert.equal(result.stdout, "41,5,10,15,90,60");
});
});