import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { execFileSync, spawn, spawnSync } from "node:child_process"; import { afterEach, describe, it } from "vitest"; const runnerPath = path.resolve("scripts/lib/run-command-with-timeout.mjs"); const helperPath = path.resolve("scripts/lib/docker-runtime-smoke.sh"); const roleFixturePath = path.resolve("scripts/lib/docker-rls-role-fixture.sh"); const temporaryDirectories = []; const remainingProcesses = new Set(); afterEach(() => { for (const pid of remainingProcesses) { try { process.kill(pid, "SIGKILL"); } catch { // The timeout runner should already have reaped the fixture process. } } remainingProcesses.clear(); for (const directory of temporaryDirectories.splice(0)) { fs.rmSync(directory, { recursive: true, force: true }); } }); function runRunner(stage, timeoutSeconds, command, { graceSeconds = 0.1, env = process.env } = {}) { return spawnSync( process.execPath, [runnerPath, "--stage", stage, "--timeout-seconds", String(timeoutSeconds), "--grace-seconds", String(graceSeconds), "--", ...command], { encoding: "utf8", env, timeout: 5_000 }, ); } function createProcessTreeFixture() { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-timeout-tree-")); temporaryDirectories.push(directory); const pidFile = path.join(directory, "pids.json"); const grandchildPath = path.join(directory, "grandchild.mjs"); const parentPath = path.join(directory, "parent.mjs"); fs.writeFileSync(grandchildPath, ['import process from "node:process";', 'process.on("SIGTERM", () => {});', "setInterval(() => {}, 1_000);", ""].join("\n")); fs.writeFileSync( parentPath, [ 'import fs from "node:fs";', 'import process from "node:process";', 'import { spawn } from "node:child_process";', "const [pidFile, grandchildPath] = process.argv.slice(2);", 'const grandchild = spawn(process.execPath, [grandchildPath], { stdio: "ignore" });', "fs.writeFileSync(pidFile, JSON.stringify({ parent: process.pid, grandchild: grandchild.pid }));", 'process.on("SIGTERM", () => {});', "setInterval(() => {}, 1_000);", "", ].join("\n") ); return { parentPath, pidFile, grandchildPath }; } function readFixturePids(pidFile) { const pids = JSON.parse(fs.readFileSync(pidFile, "utf8")); remainingProcesses.add(pids.parent); remainingProcesses.add(pids.grandchild); return pids; } async function waitForFile(filePath, timeoutMilliseconds = 2_000) { const deadline = Date.now() + timeoutMilliseconds; while (Date.now() < deadline) { if (fs.existsSync(filePath)) { return; } await new Promise((resolve) => setTimeout(resolve, 20)); } throw new Error("process-tree fixture did not become ready"); } function isTerminatedLinuxProcessStat(stat) { const commandEnd = stat.lastIndexOf(")"); const state = commandEnd >= 0 ? stat.slice(commandEnd + 2).split(" ", 1)[0] : ""; return state === "Z" || state === "X" || state === "x"; } async function waitForProcessExit(pid, timeoutMilliseconds = 2_000) { const deadline = Date.now() + timeoutMilliseconds; while (Date.now() < deadline) { try { process.kill(pid, 0); } catch (error) { if (error?.code === "ESRCH") { remainingProcesses.delete(pid); return; } throw error; } // Linux keeps an exited process in /proc while its parent (or container PID 1) // has not reaped the zombie yet. kill(pid, 0) still succeeds in that state even // though the process can no longer execute, so it is valid evidence that the // timeout runner killed the fixture. This occurs on the Gitea act runner, whose // job container does not necessarily reap adopted grandchildren immediately. if (process.platform === "linux") { try { const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); if (isTerminatedLinuxProcessStat(stat)) { remainingProcesses.delete(pid); return; } } catch (error) { if (error?.code === "ENOENT") { remainingProcesses.delete(pid); return; } throw error; } } await new Promise((resolve) => setTimeout(resolve, 20)); } throw new Error("timed-out process remained runnable"); } describe("run-command-with-timeout", () => { it("recognizes Linux zombie and dead states without accepting runnable states", () => { for (const state of ["Z", "X", "x"]) { assert.equal(isTerminatedLinuxProcessStat(`123 (fixture with ) paren) ${state} 1 2 3`), true); } for (const state of ["R", "S", "D", "T"]) { assert.equal(isTerminatedLinuxProcessStat(`123 (fixture) ${state} 1 2 3`), false); } assert.equal(isTerminatedLinuxProcessStat("malformed"), false); }); it("forwards output and reports stage-only metadata for successful commands", () => { const secret = "secret-sentinel-success-42"; const result = runRunner( "worker:inspect", 2, [process.execPath, "-e", 'process.stdout.write("child-stdout\\n"); process.stderr.write("child-stderr\\n");', secret], { env: { ...process.env, SECRET_SENTINEL: secret } } ); assert.equal(result.status, 0); assert.equal(result.stdout, "child-stdout\n"); assert.match(result.stderr, /child-stderr/u); assert.match(result.stderr, /\[docker-smoke\] START stage=worker:inspect timeout_seconds=2 grace_seconds=0\.1/u); assert.match(result.stderr, /\[docker-smoke\] PASS stage=worker:inspect duration_ms=\d+ exit_code=0/u); assert.doesNotMatch(result.stderr, new RegExp(secret, "u")); }); it("preserves a failing command's exit code and streams", () => { const result = runRunner("worker:migrate", 2, [ process.execPath, "-e", 'process.stdout.write("migration-stdout\\n"); process.stderr.write("migration-stderr\\n"); process.exit(7);', ]); assert.equal(result.status, 7); assert.equal(result.stdout, "migration-stdout\n"); assert.match(result.stderr, /migration-stderr/u); assert.match(result.stderr, /\[docker-smoke\] FAIL stage=worker:migrate duration_ms=\d+ exit_code=7/u); }); it("returns 124 and kills a SIGTERM-ignoring process group", async () => { const fixture = createProcessTreeFixture(); const result = runRunner("worker:timeout", 0.25, [process.execPath, fixture.parentPath, fixture.pidFile, fixture.grandchildPath], { graceSeconds: 0.1 }); const pids = readFixturePids(fixture.pidFile); assert.equal(result.status, 124); assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:timeout duration_ms=\d+ exit_code=124/u); await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]); }); it("terminates the child group and exits signal-typically when the suite sends SIGTERM", async () => { const fixture = createProcessTreeFixture(); const runner = spawn( process.execPath, [ runnerPath, "--stage", "suite:interrupt", "--timeout-seconds", "10", "--grace-seconds", "0.1", "--", process.execPath, fixture.parentPath, fixture.pidFile, fixture.grandchildPath, ], { stdio: ["ignore", "pipe", "pipe"] } ); let stderr = ""; runner.stderr.setEncoding("utf8"); runner.stderr.on("data", (chunk) => { stderr += chunk; }); try { await waitForFile(fixture.pidFile); const pids = readFixturePids(fixture.pidFile); runner.kill("SIGTERM"); const result = await new Promise((resolve, reject) => { const timeout = setTimeout(() => reject(new Error("timeout runner did not exit after SIGTERM")), 3_000); runner.once("error", reject); runner.once("close", (code, signal) => { clearTimeout(timeout); resolve({ code, signal }); }); }); assert.deepEqual(result, { code: 143, signal: null }); assert.match(stderr, /\[docker-smoke\] FAIL stage=suite:interrupt duration_ms=\d+ signal=SIGTERM exit_code=143/u); await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]); } finally { if (runner.exitCode === null && runner.signalCode === null) { runner.kill("SIGKILL"); } } }); it("rejects malformed metadata labels without echoing them", () => { const secret = "secret sentinel invalid stage"; const result = spawnSync(process.execPath, [runnerPath, "--stage", secret, "--timeout-seconds", "1", "--", process.execPath, "-e", ""], { encoding: "utf8", }); assert.equal(result.status, 2); assert.match(result.stderr, /reason=invalid_arguments/u); assert.doesNotMatch(result.stderr, new RegExp(secret, "u")); }); }); describe("docker-runtime-smoke helper", () => { it("replays captured stdout to stderr when the command fails", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-capture-")); temporaryDirectories.push(fixture); const scriptPath = path.join(fixture, "capture.sh"); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, 'captured=""', "if docker_smoke_capture captured capture.failure 2 bash -c 'printf captured-stdout; printf child-stderr >&2; exit 7'; then", " exit 99", "else", " status=$?", "fi", 'printf "captured-variable=%s\\n" "$captured"', 'exit "$status"', "", ].join("\n"), { mode: 0o755 } ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", timeout: 5_000 }); assert.equal(result.status, 7); assert.equal(result.stdout, "captured-variable=\n"); assert.match(result.stderr, /child-stderr/u); assert.match(result.stderr, /captured-stdout/u); assert.match(result.stderr, /\[docker-smoke\] FAIL stage=capture.failure duration_ms=\d+ exit_code=7/u); }); it("requires an in-container rejection marker and rejects Docker transport failures", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "rejection.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync( fakeDockerPath, [ "#!/usr/bin/env bash", 'if [[ "${FAKE_DOCKER_TRANSPORT_FAILURE:-0}" == "1" ]]; then', ' printf "Docker transport unavailable\\n" >&2', " exit 1", "fi", '[[ "$1" == "exec" ]] || exit 2', "shift", "shift", '"$@"', "", ].join("\n"), { mode: 0o755 }, ); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "docker_smoke_expect_docker_exec_rejection rejection.product 2 fixture-container bash -c 'printf product-rejected >&2; exit 1'", "", ].join("\n"), { mode: 0o755 }, ); const baseEnv = { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }; const productRejection = spawnSync("bash", [scriptPath], { encoding: "utf8", env: baseEnv, timeout: 5_000 }); const transportFailure = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...baseEnv, FAKE_DOCKER_TRANSPORT_FAILURE: "1" }, timeout: 5_000, }); assert.equal(productRejection.status, 0); assert.match(productRejection.stderr, /\[docker-smoke\] ASSERT stage=rejection\.product outcome=expected_rejection/u); assert.equal(transportFailure.status, 1); assert.match(transportFailure.stderr, /Docker transport unavailable/u); assert.match(transportFailure.stderr, /\[docker-smoke\] FAIL stage=rejection\.product duration_ms=\d+ exit_code=1/u); assert.doesNotMatch(transportFailure.stderr, /outcome=expected_rejection/u); }); it("rejects an unexpectedly successful in-container negative assertion", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-success-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "rejection-success.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync( fakeDockerPath, ["#!/usr/bin/env bash", '[[ "$1" == "exec" ]] || exit 2', "shift", "shift", '"$@"', ""].join("\n"), { mode: 0o755 }, ); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "docker_smoke_expect_docker_exec_rejection rejection.unexpected-success 2 fixture-container bash -c 'exit 0'", "", ].join("\n"), { mode: 0o755 }, ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 64); assert.match(result.stderr, /\[docker-smoke\] FAIL stage=rejection\.unexpected-success duration_ms=\d+ exit_code=64/u); assert.doesNotMatch(result.stderr, /outcome=expected_rejection/u); }); it("bounds hanging Docker cleanup while preserving the original exit status", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "cleanup.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 }); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "export DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS=0.2", "export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1", "cleanup() {", " docker_smoke_cleanup_container cleanup.container fixture-container", "}", "docker_smoke_install_cleanup_traps cleanup", "exit 23", "", ].join("\n"), { mode: 0o755 } ); const startedAt = Date.now(); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 23); assert.ok(Date.now() - startedAt < 3_000, "cleanup must be bounded by the helper timeout"); assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=cleanup\.container duration_ms=\d+ exit_code=124/u); assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.container reason=cleanup_incomplete exit_code=124/u); }); it("treats already absent cleanup resources as an idempotent success", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-absent-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "cleanup-absent.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync( fakeDockerPath, [ "#!/usr/bin/env bash", 'if [[ "$1" == "container" && "$2" == "rm" ]]; then', ' printf "Error response from daemon: No such container: %s\\n" "$4" >&2', " exit 1", "fi", 'if [[ "$1" == "container" && "$2" == "inspect" ]]; then', ' printf "Error: No such object: %s\\n" "$3" >&2', " exit 1", "fi", "exit 2", "", ].join("\n"), { mode: 0o755 } ); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "docker_smoke_cleanup_container cleanup.absent absent-container", "", ].join("\n"), { mode: 0o755 } ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 0); assert.match(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.absent duration_ms=\d+ exit_code=0/u); assert.doesNotMatch(result.stderr, /(?:FAIL|WARN|RETAINED|No such)/u); }); it("does not misclassify an unknown Docker context failure as an absent resource", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-context-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "cleanup-context.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync( fakeDockerPath, [ "#!/usr/bin/env bash", 'printf "unable to resolve docker endpoint: context review-context: context not found\\n" >&2', "exit 1", "", ].join("\n"), { mode: 0o755 }, ); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "docker_smoke_cleanup_container cleanup.context fixture-container", "", ].join("\n"), { mode: 0o755 }, ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 0); assert.match(result.stderr, /context review-context: context not found/u); assert.match(result.stderr, /\[docker-smoke\] CLEANUP_CHECK_FAILED kind=container name=fixture-container/u); assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.context reason=cleanup_incomplete exit_code=1/u); assert.doesNotMatch(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.context/u); }); it("reports retained cleanup resources without replacing the original exit status", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-retained-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "cleanup-retained.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync( fakeDockerPath, [ "#!/usr/bin/env bash", 'if [[ "$1" == "container" && "$2" == "rm" ]]; then', ' printf "removal already in progress\\n" >&2', " exit 1", "fi", 'if [[ "$1" == "container" && "$2" == "inspect" ]]; then', " exit 0", "fi", "exit 2", "", ].join("\n"), { mode: 0o755 } ); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, "cleanup() {", " docker_smoke_cleanup_container cleanup.retained retained-container", "}", "docker_smoke_install_cleanup_traps cleanup", "exit 23", "", ].join("\n"), { mode: 0o755 } ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 23); assert.match(result.stderr, /\[docker-smoke\] RETAINED kind=container name=retained-container/u); assert.match(result.stderr, /\[docker-smoke\] FAIL stage=cleanup\.retained duration_ms=\d+ exit_code=1/u); assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.retained reason=cleanup_incomplete exit_code=1/u); }); it("bounds RLS role provisioning with a named stage without exposing fixture credentials", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-role-provision-")); temporaryDirectories.push(fixture); const binDirectory = path.join(fixture, "bin"); const fakeDockerPath = path.join(binDirectory, "docker"); const scriptPath = path.join(fixture, "provision.sh"); fs.mkdirSync(binDirectory); fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 }); fs.writeFileSync( scriptPath, [ "#!/usr/bin/env bash", "set -euo pipefail", `source ${JSON.stringify(helperPath)}`, `source ${JSON.stringify(roleFixturePath)}`, "export DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS=0.2", "export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1", "provision_docker_smoke_rls_roles worker:provision-rls-roles fixture-db fixture-admin fixture-database migrator-secret app-secret worker-secret authentik-secret", "", ].join("\n"), { mode: 0o755 } ); const result = spawnSync("bash", [scriptPath], { encoding: "utf8", env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` }, timeout: 5_000, }); assert.equal(result.status, 124); assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:provision-rls-roles duration_ms=\d+ exit_code=124/u); assert.doesNotMatch(result.stderr, /(?:migrator|app|worker|authentik)-secret/u); }); it("exports overridable timeout defaults and remains valid Bash syntax", () => { execFileSync("bash", ["-n", helperPath]); const result = spawnSync( "bash", [ "-c", [ "set -euo pipefail", "export DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS=41", `source ${JSON.stringify(helperPath)}`, 'printf \'%s,%s,%s,%s,%s,%s\' "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" "$DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS" "$DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS" "$DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS"', ].join("\n"), ], { encoding: "utf8" } ); assert.equal(result.status, 0); assert.equal(result.stdout, "41,5,10,15,90,60"); }); });