Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -0,0 +1,79 @@
#!/usr/bin/env bash
# Provision the exact production role shape in an ephemeral PostgreSQL smoke
# container. Passwords are fixed test fixtures supplied by the calling script.
DOCKER_SMOKE_PORTAL_DB_PASSWORD="${DOCKER_SMOKE_PORTAL_DB_PASSWORD:-portal_smoke_0123456789abcdef012345}"
provision_docker_smoke_rls_roles() {
local stage="$1"
local container_name="$2"
local admin_user="$3"
local database_name="$4"
local migrator_password="$5"
local app_password="$6"
local worker_password="$7"
local authentik_jit_password="$8"
local portal_password="$DOCKER_SMOKE_PORTAL_DB_PASSWORD"
docker_smoke_run "$stage" "$DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS" \
docker exec -i "$container_name" \
psql -v ON_ERROR_STOP=1 -U "$admin_user" -d "$database_name" \
-v migrator_password="$migrator_password" \
-v app_password="$app_password" \
-v portal_password="$portal_password" \
-v worker_password="$worker_password" \
-v authentik_jit_password="$authentik_jit_password" >/dev/null <<'SQL'
SELECT
set_config('hivegrid.smoke.migrator_password', :'migrator_password', false),
set_config('hivegrid.smoke.app_password', :'app_password', false),
set_config('hivegrid.smoke.portal_password', :'portal_password', false),
set_config('hivegrid.smoke.worker_password', :'worker_password', false),
set_config('hivegrid.smoke.authentik_jit_password', :'authentik_jit_password', false);
DO $hivegrid_smoke_roles$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_migrator') THEN
CREATE ROLE hivegrid_migrator;
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_app') THEN
CREATE ROLE hivegrid_app;
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_portal') THEN
CREATE ROLE hivegrid_portal;
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_worker') THEN
CREATE ROLE hivegrid_worker;
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_authentik_jit') THEN
CREATE ROLE hivegrid_authentik_jit;
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'hivegrid_rls_definer') THEN
CREATE ROLE hivegrid_rls_definer;
END IF;
ALTER ROLE hivegrid_migrator LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS;
ALTER ROLE hivegrid_app LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS;
ALTER ROLE hivegrid_portal LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS;
ALTER ROLE hivegrid_worker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS;
ALTER ROLE hivegrid_authentik_jit LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS;
ALTER ROLE hivegrid_rls_definer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION BYPASSRLS PASSWORD NULL;
EXECUTE format('ALTER ROLE hivegrid_migrator PASSWORD %L', current_setting('hivegrid.smoke.migrator_password'));
EXECUTE format('ALTER ROLE hivegrid_app PASSWORD %L', current_setting('hivegrid.smoke.app_password'));
EXECUTE format('ALTER ROLE hivegrid_portal PASSWORD %L', current_setting('hivegrid.smoke.portal_password'));
EXECUTE format('ALTER ROLE hivegrid_worker PASSWORD %L', current_setting('hivegrid.smoke.worker_password'));
EXECUTE format('ALTER ROLE hivegrid_authentik_jit PASSWORD %L', current_setting('hivegrid.smoke.authentik_jit_password'));
END
$hivegrid_smoke_roles$;
GRANT hivegrid_rls_definer TO hivegrid_migrator
WITH ADMIN FALSE, INHERIT FALSE, SET TRUE;
ALTER DATABASE :DBNAME OWNER TO hivegrid_migrator;
ALTER SCHEMA public OWNER TO hivegrid_migrator;
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
GRANT USAGE, CREATE ON SCHEMA public TO hivegrid_migrator;
REVOKE CONNECT, CREATE, TEMPORARY ON DATABASE :DBNAME FROM PUBLIC;
GRANT CONNECT, TEMPORARY ON DATABASE :DBNAME TO hivegrid_migrator;
GRANT CONNECT ON DATABASE :DBNAME TO hivegrid_app, hivegrid_portal, hivegrid_worker, hivegrid_authentik_jit;
SQL
}
@@ -0,0 +1,362 @@
#!/usr/bin/env bash
# Shared, Bash 3.2-compatible execution boundaries for Docker runtime smokes.
# The Node runner deliberately logs stage metadata only; command arguments and
# environment values must never be copied into wrapper diagnostics.
_DOCKER_SMOKE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export DOCKER_SMOKE_NODE_BIN="${DOCKER_SMOKE_NODE_BIN:-node}"
export DOCKER_SMOKE_TIMEOUT_RUNNER="${DOCKER_SMOKE_TIMEOUT_RUNNER:-${_DOCKER_SMOKE_LIB_DIR}/run-command-with-timeout.mjs}"
export DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS="${DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS:-30}"
export DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS="${DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS:-5}"
export DOCKER_SMOKE_LOG_TIMEOUT_SECONDS="${DOCKER_SMOKE_LOG_TIMEOUT_SECONDS:-10}"
export DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS="${DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS:-15}"
export DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS="${DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS:-90}"
export DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS="${DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS:-60}"
export DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS="${DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS:-300}"
export DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS="${DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS:-30}"
export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS="${DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS:-2}"
export DOCKER_SMOKE_LOG_TAIL="${DOCKER_SMOKE_LOG_TAIL:-200}"
DOCKER_SMOKE_EXPECTED_REJECTION_MARKER="hivegrid-smoke-expected-rejection-v1"
_DOCKER_SMOKE_CLEANUP_CALLBACK=""
_DOCKER_SMOKE_CLEANUP_ACTIVE=0
docker_smoke_run() {
if [[ "$#" -lt 3 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
shift 2
docker_smoke_run_with_grace "$stage" "$timeout_seconds" "$DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS" "$@"
}
docker_smoke_run_with_grace() {
if [[ "$#" -lt 4 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
local grace_seconds="${3:-$DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS}"
shift 3
"$DOCKER_SMOKE_NODE_BIN" "$DOCKER_SMOKE_TIMEOUT_RUNNER" \
--stage "$stage" \
--timeout-seconds "$timeout_seconds" \
--grace-seconds "$grace_seconds" \
-- "$@"
}
docker_smoke_expect_docker_exec_rejection() {
if [[ "$#" -lt 4 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
local container_name="$3"
local observed_output=""
local command_status
shift 3
if docker_smoke_capture observed_output "$stage" "$timeout_seconds" \
docker exec "$container_name" sh -c '
marker="$1"
shift
"$@"
command_status=$?
if [ "$command_status" -eq 1 ]; then
printf "%s" "$marker"
exit 0
fi
if [ "$command_status" -eq 0 ]; then
exit 64
fi
exit "$command_status"
' docker-smoke-rejection "$DOCKER_SMOKE_EXPECTED_REJECTION_MARKER" "$@"; then
command_status=0
else
command_status=$?
fi
if [[ "$command_status" -ne 0 ]]; then
return "$command_status"
fi
if [[ "$observed_output" != "$DOCKER_SMOKE_EXPECTED_REJECTION_MARKER" ]]; then
printf '%s\n' "[docker-smoke] FAIL stage=${stage} reason=missing_expected_rejection_marker exit_code=1" >&2
return 1
fi
printf '%s\n' "[docker-smoke] ASSERT stage=${stage} outcome=expected_rejection" >&2
}
docker_smoke_capture() {
if [[ "$#" -lt 4 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local output_variable="$1"
local stage="$2"
local timeout_seconds="${3:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
local _docker_smoke_captured_value
local command_status
shift 3
if [[ ! "$output_variable" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_output_variable exit_code=2' >&2
return 2
fi
if _docker_smoke_captured_value="$(docker_smoke_run "$stage" "$timeout_seconds" "$@")"; then
command_status=0
else
command_status=$?
fi
if [[ "$command_status" -ne 0 ]]; then
if [[ -n "$_docker_smoke_captured_value" ]]; then
printf '%s\n' "$_docker_smoke_captured_value" >&2
fi
return "$command_status"
fi
printf -v "$output_variable" '%s' "$_docker_smoke_captured_value"
}
docker_smoke_probe() {
if [[ "$#" -lt 2 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
shift
docker_smoke_run "$stage" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$@"
}
docker_smoke_wait_for_postgres() {
if [[ "$#" -ne 4 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=postgres-readiness reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
local container_name="$2"
local database_user="$3"
local database_name="$4"
local deadline_seconds=$((SECONDS + DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS))
while [[ "$SECONDS" -lt "$deadline_seconds" ]]; do
if docker_smoke_probe "$stage" docker exec "$container_name" \
pg_isready -h 127.0.0.1 -U "$database_user" -d "$database_name" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
return 1
}
docker_smoke_logs() {
if [[ "$#" -ne 2 ]]; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
return 2
fi
local stage="$1"
local container_name="$2"
docker_smoke_run "$stage" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" \
docker logs --tail "$DOCKER_SMOKE_LOG_TAIL" "$container_name"
}
docker_smoke_cleanup_run() {
if [[ "$#" -lt 2 ]]; then
return 0
fi
local stage="$1"
local cleanup_status
shift
if docker_smoke_run "$stage" "$DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS" "$@" >/dev/null; then
return 0
else
cleanup_status=$?
fi
printf '%s\n' "[docker-smoke] WARN stage=${stage} reason=cleanup_incomplete exit_code=${cleanup_status}" >&2
return 0
}
_docker_smoke_cleanup_resources() {
if [[ "$#" -lt 3 ]]; then
return 0
fi
local resource_kind="$1"
local stage="$2"
shift 2
docker_smoke_cleanup_run "$stage" bash -c '
set -u
resource_kind="$1"
shift
removal_status=0
removal_output=""
case "$resource_kind" in
container)
removal_output="$(docker container rm -f "$@" 2>&1 >/dev/null)" || removal_status=$?
;;
network|volume)
removal_output="$(docker "$resource_kind" rm "$@" 2>&1 >/dev/null)" || removal_status=$?
;;
*)
exit 2
;;
esac
cleanup_incomplete=0
for resource_name in "$@"; do
inspect_status=0
resource_absent=0
inspect_output="$(docker "$resource_kind" inspect "$resource_name" 2>&1 >/dev/null)" || inspect_status=$?
if [[ "$inspect_status" -eq 0 ]]; then
printf "[docker-smoke] RETAINED kind=%s name=%s\n" "$resource_kind" "$resource_name" >&2
cleanup_incomplete=1
continue
fi
case "$resource_kind" in
container)
case "$inspect_output" in
*"No such object: $resource_name"*|*"No such container: $resource_name"*) resource_absent=1 ;;
esac
;;
network)
case "$inspect_output" in
*"No such network: $resource_name"*|*"network $resource_name not found"*) resource_absent=1 ;;
esac
;;
volume)
case "$inspect_output" in
*"No such volume: $resource_name"*|*"no such volume: $resource_name"*|*"get $resource_name: no such volume"*) resource_absent=1 ;;
esac
;;
esac
if [[ "$resource_absent" -ne 1 ]]; then
if [[ -n "$inspect_output" ]]; then
printf "%s\n" "$inspect_output" >&2
fi
printf "[docker-smoke] CLEANUP_CHECK_FAILED kind=%s name=%s\n" "$resource_kind" "$resource_name" >&2
cleanup_incomplete=1
fi
done
if [[ "$cleanup_incomplete" -eq 1 ]]; then
if [[ -n "$removal_output" ]]; then
printf "%s\n" "$removal_output" >&2
fi
if [[ "$removal_status" -eq 0 ]]; then
removal_status=1
fi
exit "$removal_status"
fi
' docker-smoke-cleanup "$resource_kind" "$@"
}
docker_smoke_cleanup_container() {
if [[ "$#" -lt 2 ]]; then
return 0
fi
local stage="$1"
shift
_docker_smoke_cleanup_resources container "$stage" "$@"
}
docker_smoke_cleanup_network() {
if [[ "$#" -lt 2 ]]; then
return 0
fi
local stage="$1"
shift
_docker_smoke_cleanup_resources network "$stage" "$@"
}
docker_smoke_cleanup_volume() {
if [[ "$#" -lt 2 ]]; then
return 0
fi
local stage="$1"
shift
_docker_smoke_cleanup_resources volume "$stage" "$@"
}
_docker_smoke_invoke_cleanup() {
if [[ "$_DOCKER_SMOKE_CLEANUP_ACTIVE" -eq 1 || -z "$_DOCKER_SMOKE_CLEANUP_CALLBACK" ]]; then
return 0
fi
_DOCKER_SMOKE_CLEANUP_ACTIVE=1
if "$_DOCKER_SMOKE_CLEANUP_CALLBACK"; then
:
else
:
fi
_DOCKER_SMOKE_CLEANUP_ACTIVE=0
return 0
}
_docker_smoke_handle_exit() {
local original_status="$1"
trap - EXIT INT TERM
_docker_smoke_invoke_cleanup
exit "$original_status"
}
_docker_smoke_handle_signal() {
local signal_name="$1"
local signal_status
case "$signal_name" in
INT)
signal_status=130
;;
TERM)
signal_status=143
;;
*)
signal_status=1
;;
esac
trap - EXIT INT TERM
_docker_smoke_invoke_cleanup
exit "$signal_status"
}
docker_smoke_install_cleanup_traps() {
if [[ "$#" -ne 1 || -z "$1" ]] || ! declare -F "$1" >/dev/null 2>&1; then
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_cleanup_callback exit_code=2' >&2
return 2
fi
_DOCKER_SMOKE_CLEANUP_CALLBACK="$1"
trap '_docker_smoke_handle_exit "$?"' EXIT
trap '_docker_smoke_handle_signal INT' INT
trap '_docker_smoke_handle_signal TERM' TERM
}
unset _DOCKER_SMOKE_LIB_DIR
@@ -0,0 +1,231 @@
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { beforeAll, describe, expect, it, vi } from "vitest";
import {
PORTAL_APP_WORKER_DELTA_TABLES,
assertPortalAppWorkerPolicyParity,
normalizePolicyExpression,
parsePortalAppWorkerPolicyContract,
readPortalAppWorkerPolicyState,
type PortalAppWorkerPolicyContract,
type PortalAppWorkerPolicyState,
} from "./portal-app-worker-policy-parity.js";
const CANONICAL_ARTIFACT = join(process.cwd(), "prisma/rls/tenant-policy-contract-isolation.sql");
const APP_WORKER_ROLES = ["hivegrid_app", "hivegrid_worker"] as const;
const PRIVILEGES = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE", "REFERENCES", "TRIGGER"] as const;
let canonicalSql: string;
let contract: PortalAppWorkerPolicyContract;
beforeAll(async () => {
canonicalSql = await readFile(CANONICAL_ARTIFACT, "utf8");
contract = parsePortalAppWorkerPolicyContract(canonicalSql);
});
describe("portal app/worker policy contract parser", () => {
it("derives the complete portal delta contract from the generated artifact", () => {
expect(contract.tables.map(({ table }) => table)).toEqual(PORTAL_APP_WORKER_DELTA_TABLES);
expect(contract.policies).toHaveLength(69);
expect(contract.grants).toHaveLength(144);
expect(contract.tables).toEqual(
PORTAL_APP_WORKER_DELTA_TABLES.map((table) => ({
table,
rowSecurity: true,
forceRowSecurity: true,
}))
);
const portalPolicy = contract.policies.find(({ table, policy }) => table === "Portal" && policy === "hg_Portal_all");
expect(portalPolicy).toMatchObject({
permissive: true,
command: "*",
roles: ["hivegrid_app", "hivegrid_worker"],
});
expect(portalPolicy?.usingExpression).not.toBeNull();
expect(portalPolicy?.usingExpression).toBe(portalPolicy?.withCheckExpression);
});
it("fails closed when ENABLE/FORCE, grants, or SQL structure are malformed", () => {
expect(() => parsePortalAppWorkerPolicyContract(canonicalSql.replace('ALTER TABLE "Portal" FORCE ROW LEVEL SECURITY;', ""))).toThrow(
"must ENABLE and FORCE RLS exactly once for Portal"
);
expect(() => parsePortalAppWorkerPolicyContract(`${canonicalSql}\nGRANT SELECT ON "Portal" TO hivegrid_app;`)).toThrow(
"duplicates SELECT on Portal to hivegrid_app"
);
expect(() => parsePortalAppWorkerPolicyContract(`${canonicalSql}\nGRANT TRUNCATE ON "Portal" TO hivegrid_app;`)).toThrow(
"grants forbidden TRUNCATE on Portal to hivegrid_app"
);
expect(() =>
parsePortalAppWorkerPolicyContract(
canonicalSql.replace('CREATE POLICY "hg_Portal_all" ON "Portal"', 'CREATE POLICY "hg_Portal_all" ON "Portal" AS MAYBE')
)
).toThrow("invalid AS mode");
});
it("normalizes PostgreSQL deparser casts and aliases without erasing subquery boundaries", () => {
const artifactExpression = "\"tenantId\" = NULLIF(current_setting('app.tenant_id', true), '')";
const postgresExpression = "(\"tenantId\" = NULLIF(current_setting('app.tenant_id'::text, true), ''::text))";
expect(normalizePolicyExpression(postgresExpression)).toBe(normalizePolicyExpression(artifactExpression));
const artifactExistsExpression =
'("tenantId" = NULLIF(current_setting(\'app.tenant_id\', true), \'\') AND EXISTS (SELECT 1 FROM "Portal" AS "owner_0" WHERE "owner_0"."id" = "PortalUser"."portalId" AND "owner_0"."tenantId" = NULLIF(current_setting(\'app.tenant_id\', true), \'\')))';
const postgresExistsExpression =
'(("tenantId" = NULLIF(current_setting(\'app.tenant_id\'::text, true), \'\'::text)) AND (EXISTS ( SELECT 1 FROM "Portal" owner_0 WHERE ((owner_0.id = "PortalUser"."portalId") AND (owner_0."tenantId" = NULLIF(current_setting(\'app.tenant_id\'::text, true), \'\'::text))))))';
expect(normalizePolicyExpression(postgresExistsExpression)).toBe(normalizePolicyExpression(artifactExistsExpression));
const artifactNullableRelation =
'"PortalServiceInboxItem"."routingRuleId" IS NULL OR EXISTS (SELECT 1 FROM "PortalActivityRoutingRule" AS "owner_0" WHERE "owner_0"."id" = "PortalServiceInboxItem"."routingRuleId")';
const postgresNullableRelation =
'"routingRuleId" IS NULL OR EXISTS (SELECT 1 FROM "PortalActivityRoutingRule" owner_0 WHERE owner_0.id = "PortalServiceInboxItem"."routingRuleId")';
expect(normalizePolicyExpression(postgresNullableRelation, "PortalServiceInboxItem")).toBe(
normalizePolicyExpression(artifactNullableRelation, "PortalServiceInboxItem")
);
expect(normalizePolicyExpression('"wrong_alias"."routingRuleId" IS NULL', "PortalServiceInboxItem")).not.toBe(
normalizePolicyExpression('"routingRuleId" IS NULL', "PortalServiceInboxItem")
);
const siblingSubqueries =
'EXISTS (SELECT 1 FROM "Portal" AS "owner_0" WHERE "owner_0"."id" = "PortalUser"."portalId") AND EXISTS (SELECT 1 FROM "Portal" AS "owner_1" WHERE "owner_1"."id" = "PortalUser"."portalId")';
const nestedSubquery =
'EXISTS (SELECT 1 FROM "Portal" AS "owner_0" WHERE "owner_0"."id" = "PortalUser"."portalId" AND EXISTS (SELECT 1 FROM "Portal" AS "owner_1" WHERE "owner_1"."id" = "PortalUser"."portalId"))';
expect(normalizePolicyExpression(siblingSubqueries)).not.toBe(normalizePolicyExpression(nestedSubquery));
});
});
describe("portal app/worker exact database-state parity", () => {
it("accepts only the exact state derived from the canonical artifact", () => {
expect(() => assertPortalAppWorkerPolicyParity(contract, stateFromContract(contract))).not.toThrow();
});
it("rejects permissive true expressions, missing/extra policies, and role or command drift", () => {
const state = stateFromContract(contract);
const policy = state.policies.find(({ table, policy: name }) => table === "PortalEmailTemplate" && name === "hg_PortalEmailTemplate_update");
if (!policy) throw new Error("test fixture policy missing");
policy.usingExpression = normalizePolicyExpression("true");
policy.command = "*";
policy.roles = ["PUBLIC"];
state.policies.pop();
state.policies.push({
table: "Portal",
policy: "allow_everything",
permissive: true,
command: "*",
roles: ["hivegrid_app", "hivegrid_worker"],
usingExpression: normalizePolicyExpression("true"),
withCheckExpression: normalizePolicyExpression("true"),
});
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow(
/using_expression_mismatch.*command_mismatch|command_mismatch.*using_expression_mismatch/
);
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("roles_mismatch");
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("unexpected_policy");
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("policy_missing");
});
it("rejects missing FORCE RLS, a missing required grant, and every extra table privilege", () => {
const state = stateFromContract(contract);
const portal = state.tables.find(({ table }) => table === "Portal");
if (!portal) throw new Error("test fixture table missing");
portal.forceRowSecurity = false;
state.grants = state.grants.filter(({ table, role, privilege }) => !(table === "Portal" && role === "hivegrid_app" && privilege === "DELETE"));
state.grants.push({
table: "Portal",
role: "hivegrid_worker",
privilege: "TRUNCATE",
});
state.columnGrants.push({
table: "PortalUser",
column: "passwordHash",
role: "hivegrid_app",
privilege: "UPDATE",
});
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("Portal:rls_not_forced");
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("Portal:hivegrid_app:DELETE:grant_missing");
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("Portal:hivegrid_worker:TRUNCATE:unexpected_grant");
expect(() => assertPortalAppWorkerPolicyParity(contract, state)).toThrow("PortalUser.passwordHash:hivegrid_app:UPDATE:unexpected_column_grant");
});
it("reads table flags, all app/worker-affecting policies, and the full effective privilege matrix", async () => {
const queries: string[] = [];
const queryRawUnsafe = vi.fn(async (query: string): Promise<unknown[]> => {
queries.push(query);
if (query.includes(":tables */")) {
return PORTAL_APP_WORKER_DELTA_TABLES.map((table) => ({
table_name: table,
row_security: true,
force_row_security: true,
}));
}
if (query.includes(":policies */")) {
return [
{
table_name: "Portal",
policy_name: "hg_Portal_all",
permissive: true,
command: "*",
roles: ["hivegrid_app", "hivegrid_worker"],
using_expression: "(\"tenantId\" = NULLIF(current_setting('app.tenant_id'::text, true), ''::text))",
with_check_expression: "(\"tenantId\" = NULLIF(current_setting('app.tenant_id'::text, true), ''::text))",
},
];
}
if (query.includes(":privileges */")) {
return PORTAL_APP_WORKER_DELTA_TABLES.flatMap((table) =>
APP_WORKER_ROLES.flatMap((role) =>
PRIVILEGES.map((privilege) => ({
table_name: table,
role_name: role,
privilege_type: privilege,
granted: ["SELECT", "INSERT", "UPDATE", "DELETE"].includes(privilege),
}))
)
);
}
if (query.includes(":column-grants */")) return [];
throw new Error("unexpected test query");
});
const state = await readPortalAppWorkerPolicyState({ $queryRawUnsafe: queryRawUnsafe });
expect(queryRawUnsafe).toHaveBeenCalledTimes(4);
expect(queries[0]).toContain("class.relrowsecurity");
expect(queries[0]).toContain("class.relforcerowsecurity");
expect(queries[1]).toContain("pg_get_expr(policy.polqual");
expect(queries[1]).toContain("policy_role.role_oid = 0");
expect(queries[1]).toContain("pg_has_role(runtime_role.oid, role.oid, 'MEMBER')");
expect(queries[1]).toContain("hivegrid_app");
expect(queries[1]).toContain("hivegrid_worker");
expect(queries[2]).toContain("has_table_privilege");
expect(queries[2]).toContain("TRUNCATE");
expect(queries[2]).toContain("REFERENCES");
expect(queries[2]).toContain("TRIGGER");
expect(queries[3]).toContain("attribute.attacl");
expect(queries[3]).toContain("aclexplode");
expect(queries[3]).toContain("access.grantee = 0");
expect(state.tables).toHaveLength(18);
expect(state.policies).toHaveLength(1);
expect(state.grants).toHaveLength(144);
expect(state.columnGrants).toEqual([]);
expect(state.policies[0]?.usingExpression).toBe(contract.policies.find(({ policy }) => policy === "hg_Portal_all")?.usingExpression);
});
});
function stateFromContract(source: PortalAppWorkerPolicyContract): PortalAppWorkerPolicyState {
return {
tables: source.tables.map((table) => ({ ...table })),
policies: source.policies.map((policy) => ({
...policy,
roles: [...policy.roles],
})),
grants: source.grants.map((grant) => ({ ...grant })),
columnGrants: [],
};
}
@@ -0,0 +1,212 @@
import { readFileSync, readdirSync } from "node:fs";
import path from "node:path";
export const TENANT_SCOPE_KINDS = Object.freeze([
"tenant-required",
"tenant-or-global",
"derived-child",
"tenant-root",
"global-catalog",
"global-shared",
"platform-admin",
]);
export const TENANT_RLS_ENFORCEMENTS = Object.freeze(["tenant-policy", "mixed-policy", "join-policy", "root-policy"]);
const VALID_SCOPES = new Set(TENANT_SCOPE_KINDS);
const VALID_ENFORCEMENTS = new Set([...TENANT_RLS_ENFORCEMENTS, "none", "platform-role"]);
function listSchemaFiles(schemaDirectory) {
return readdirSync(schemaDirectory, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.endsWith(".prisma"))
.map((entry) => path.join(schemaDirectory, entry.name))
.sort();
}
function parseOwningRelations(body) {
const relations = [];
const relationField = /^\s*(\w+)\s+([A-Za-z_]\w*)(\?)?\s+@relation\(([\s\S]*?)\)/gm;
for (const relation of body.matchAll(relationField)) {
const fields = /\bfields\s*:\s*\[([^\]]+)\]/m.exec(relation[4])?.[1];
const references = /\breferences\s*:\s*\[([^\]]+)\]/m.exec(relation[4])?.[1];
if (!fields || !references) continue;
relations.push({
field: relation[1],
target: relation[2],
nullable: relation[3] === "?",
fields: fields.split(",").map((field) => field.trim()),
references: references.split(",").map((field) => field.trim()),
});
}
return relations.sort((left, right) => left.field.localeCompare(right.field));
}
function parseFieldNames(body) {
const fields = [];
for (const line of body.split("\n")) {
const field = /^\s*(\w+)\s+[A-Za-z_]\w*(?:\[\])?\??(?:\s|$)/u.exec(line)?.[1];
if (field) fields.push(field);
}
return [...new Set(fields)].sort();
}
function parseNullableFieldNames(body) {
const fields = [];
for (const line of body.split("\n")) {
const field = /^\s*(\w+)\s+[A-Za-z_]\w*(?:\[\])?\?(?:\s|$)/u.exec(line)?.[1];
if (field) fields.push(field);
}
return [...new Set(fields)].sort();
}
/** Parse the model and owning-FK surface needed by the static tenant gates. */
export function parsePrismaSchemaDirectory(schemaDirectory) {
const models = [];
for (const file of listSchemaFiles(schemaDirectory)) {
const source = readFileSync(file, "utf8");
for (const match of source.matchAll(/model\s+(\w+)\s*\{([\s\S]*?)\n\}/g)) {
const body = match[2];
const tenantField = /^\s*tenantId\s+String(\?)?/m.exec(body);
const owningRelations = parseOwningRelations(body);
models.push({
model: match[1],
table: /@@map\(\s*"([^"]+)"\s*\)/m.exec(body)?.[1] ?? match[1],
schema: path.basename(file),
tenantField: tenantField ? { nullable: tenantField[1] === "?" } : null,
fieldNames: parseFieldNames(body),
nullableFieldNames: parseNullableFieldNames(body),
owningRelations,
owningRelationTargets: [...new Set(owningRelations.map((relation) => relation.target))].sort(),
});
}
}
return models.sort((left, right) => left.schema.localeCompare(right.schema) || left.model.localeCompare(right.model));
}
/**
* Discover models whose rows inherit a tenant boundary through an owning FK.
* Tenant roots and direct tenant models seed the graph; traversal then follows
* child -> parent relations until no new descendants are found.
*/
export function discoverIndirectTenantModels(models, manifest) {
const modelNames = new Set(models.map((model) => model.model));
const tenantBoundary = new Set(
models
.filter((model) => {
const scope = manifest[model.model]?.scope;
return scope === "tenant-required" || scope === "tenant-or-global" || scope === "tenant-root";
})
.map((model) => model.model)
);
const indirect = new Set();
let changed = true;
while (changed) {
changed = false;
for (const model of models) {
if (model.tenantField || tenantBoundary.has(model.model) || indirect.has(model.model)) continue;
if (model.owningRelationTargets.some((target) => modelNames.has(target) && (tenantBoundary.has(target) || indirect.has(target)))) {
indirect.add(model.model);
changed = true;
}
}
}
return indirect;
}
export function modelRequiresTenantRls(entry) {
return TENANT_RLS_ENFORCEMENTS.includes(entry.enforcement);
}
/** Validate schema coverage, direct-field semantics, and indirect FK drift. */
export function validateTenantScopeManifest(models, manifest) {
const errors = [];
const byName = new Map(models.map((model) => [model.model, model]));
const schemaNames = new Set(byName.keys());
const manifestNames = new Set(Object.keys(manifest));
for (const model of models) {
if (!manifestNames.has(model.model)) errors.push(`${model.schema}: ${model.model} is missing from the tenant scope manifest`);
}
for (const model of manifestNames) {
if (!schemaNames.has(model)) errors.push(`tenant scope manifest contains unknown model ${model}`);
}
for (const model of models) {
const entry = manifest[model.model];
if (!entry) continue;
if (!VALID_SCOPES.has(entry.scope)) errors.push(`${model.model} has invalid tenant scope ${String(entry.scope)}`);
if (!VALID_ENFORCEMENTS.has(entry.enforcement)) {
errors.push(`${model.model} has invalid tenant enforcement ${String(entry.enforcement)}`);
}
if (entry.scope === "tenant-required" && model.tenantField?.nullable !== false) {
errors.push(`${model.schema}: ${model.model} is tenant-required but lacks a required tenantId String field`);
}
if (entry.scope === "tenant-or-global" && model.tenantField?.nullable !== true) {
errors.push(`${model.schema}: ${model.model} is tenant-or-global but lacks a nullable tenantId String? field`);
}
if (entry.scope === "derived-child" && model.tenantField) {
errors.push(`${model.schema}: ${model.model} is derived-child but declares its own tenantId field`);
}
if ((entry.scope === "global-catalog" || entry.scope === "global-shared" || entry.scope === "tenant-root") && model.tenantField) {
errors.push(`${model.schema}: ${model.model} is ${entry.scope} but declares a tenantId field`);
}
const expectedEnforcement = {
"tenant-required": "tenant-policy",
"tenant-or-global": "mixed-policy",
"derived-child": "join-policy",
"tenant-root": "root-policy",
"global-catalog": "none",
"global-shared": "none",
"platform-admin": "platform-role",
}[entry.scope];
if (entry.enforcement !== expectedEnforcement && !entry.reason) {
errors.push(`${model.model} overrides ${entry.scope} enforcement without an explicit reason`);
}
if (entry.tenantParentBoundary === true && !entry.reason) {
errors.push(`${model.model} declares a tenant-parent boundary exception without an explicit reason`);
}
}
const tenantRoots = models.filter((model) => manifest[model.model]?.scope === "tenant-root").map((model) => model.model);
if ((byName.has("Tenant") || tenantRoots.length > 0) && (tenantRoots.length !== 1 || tenantRoots[0] !== "Tenant")) {
errors.push(`tenant-root classification must contain exactly Tenant, found: ${tenantRoots.join(", ") || "none"}`);
}
const indirectModels = discoverIndirectTenantModels(models, manifest);
for (const modelName of indirectModels) {
const entry = manifest[modelName];
if (!entry) continue;
const classifiedBoundary = entry.scope === "derived-child" || entry.scope === "platform-admin";
if (!classifiedBoundary && entry.tenantParentBoundary !== true) {
errors.push(`${byName.get(modelName)?.schema}: ${modelName} has an owning FK path to tenant data but is classified ${entry.scope}`);
}
}
for (const model of models) {
const entry = manifest[model.model];
if (entry?.scope === "derived-child" && !indirectModels.has(model.model)) {
errors.push(`${model.schema}: ${model.model} is derived-child but no owning FK path reaches a tenant boundary`);
}
if (entry?.tenantParentBoundary === true && !indirectModels.has(model.model)) {
errors.push(`${model.schema}: ${model.model} declares an unnecessary tenant-parent boundary exception`);
}
}
const scopeCounts = Object.fromEntries(TENANT_SCOPE_KINDS.map((scope) => [scope, 0]));
for (const model of models) {
const scope = manifest[model.model]?.scope;
if (scopeCounts[scope] !== undefined) scopeCounts[scope] += 1;
}
return {
ok: errors.length === 0,
errors,
indirectModels,
scopeCounts,
classifiedCount: models.filter((model) => manifest[model.model]).length,
modelCount: models.length,
};
}
@@ -0,0 +1,76 @@
import path from "node:path";
import { describe, expect, it } from "vitest";
import { TENANT_SCOPE_MANIFEST } from "../../prisma/tenant-scope-manifest.mjs";
import { discoverIndirectTenantModels, parsePrismaSchemaDirectory, validateTenantScopeManifest } from "./prisma-tenant-scope.mjs";
function model(model, options = {}) {
return {
model,
table: model,
schema: options.schema ?? "fixture.prisma",
tenantField: options.tenantField ?? null,
owningRelationTargets: options.owningRelationTargets ?? [],
};
}
describe("tenant scope manifest", () => {
it("classifies every current Prisma model and every discovered indirect boundary", () => {
const models = parsePrismaSchemaDirectory(path.join(process.cwd(), "prisma", "schema"));
const result = validateTenantScopeManifest(models, TENANT_SCOPE_MANIFEST);
expect(result.errors).toEqual([]);
expect(result.modelCount).toBe(377);
expect(result.classifiedCount).toBe(377);
expect(result.scopeCounts).toEqual({
"tenant-required": 195,
"tenant-or-global": 29,
"derived-child": 36,
"tenant-root": 1,
"global-catalog": 11,
"global-shared": 16,
"platform-admin": 89,
});
expect(result.indirectModels.size).toBe(37);
expect(result.indirectModels).toContain("Registration");
expect(result.indirectModels).toContain("KnowledgeBaseArticleDownvotes");
});
it("reports an unclassified schema model", () => {
const models = [model("Known", { tenantField: { nullable: false } }), model("NewModel")];
const manifest = { Known: { scope: "tenant-required", enforcement: "tenant-policy" } };
const result = validateTenantScopeManifest(models, manifest);
expect(result.errors).toContain("fixture.prisma: NewModel is missing from the tenant scope manifest");
});
it("detects recursively derived FK children classified as global", () => {
const models = [
model("TenantRow", { tenantField: { nullable: false } }),
model("Child", { owningRelationTargets: ["TenantRow"] }),
model("GrandChild", { owningRelationTargets: ["Child"] }),
];
const manifest = {
TenantRow: { scope: "tenant-required", enforcement: "tenant-policy" },
Child: { scope: "derived-child", enforcement: "join-policy" },
GrandChild: { scope: "global-shared", enforcement: "none" },
};
const indirect = discoverIndirectTenantModels(models, manifest);
const result = validateTenantScopeManifest(models, manifest);
expect(indirect).toEqual(new Set(["Child", "GrandChild"]));
expect(result.errors).toContain("fixture.prisma: GrandChild has an owning FK path to tenant data but is classified global-shared");
});
it("requires a reason for enforcement overrides", () => {
const models = [model("Hybrid", { tenantField: { nullable: true } })];
const manifest = { Hybrid: { scope: "tenant-or-global", enforcement: "none" } };
const result = validateTenantScopeManifest(models, manifest);
expect(result.errors).toContain("Hybrid overrides tenant-or-global enforcement without an explicit reason");
});
});
@@ -0,0 +1,253 @@
#!/usr/bin/env node
import os from "node:os";
import process from "node:process";
import { spawn } from "node:child_process";
const TIMEOUT_EXIT_CODE = 124;
const SPAWN_ERROR_EXIT_CODE = 127;
const INVALID_ARGUMENT_EXIT_CODE = 2;
const DEFAULT_GRACE_SECONDS = 2;
const MAX_TIMEOUT_SECONDS = 86_400;
const MAX_GRACE_SECONDS = 60;
const STAGE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/u;
function failArguments() {
process.stderr.write(`[docker-smoke] FAIL stage=timeout-runner reason=invalid_arguments exit_code=${INVALID_ARGUMENT_EXIT_CODE}\n`);
process.exit(INVALID_ARGUMENT_EXIT_CODE);
}
function parseSeconds(value, { allowZero, maximum }) {
if (typeof value !== "string" || !/^(?:0|[1-9]\d*)(?:\.\d+)?$/u.test(value)) {
failArguments();
}
const parsed = Number(value);
const validLowerBound = allowZero ? parsed >= 0 : parsed > 0;
if (!Number.isFinite(parsed) || !validLowerBound || parsed > maximum) {
failArguments();
}
return parsed;
}
function parseArguments(argv) {
let stage;
let timeoutSeconds;
let graceSeconds = DEFAULT_GRACE_SECONDS;
let sawGraceSeconds = false;
let command;
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index];
if (argument === "--") {
command = argv.slice(index + 1);
break;
}
if (argument === "--stage" && stage === undefined) {
stage = argv[index + 1];
index += 1;
continue;
}
if (argument === "--timeout-seconds" && timeoutSeconds === undefined) {
timeoutSeconds = argv[index + 1];
index += 1;
continue;
}
if (argument === "--grace-seconds" && !sawGraceSeconds) {
graceSeconds = argv[index + 1];
sawGraceSeconds = true;
index += 1;
continue;
}
failArguments();
}
if (!STAGE_PATTERN.test(stage ?? "") || timeoutSeconds === undefined || !command?.length) {
failArguments();
}
return {
stage,
timeoutSeconds: parseSeconds(timeoutSeconds, { allowZero: false, maximum: MAX_TIMEOUT_SECONDS }),
graceSeconds: parseSeconds(String(graceSeconds), { allowZero: true, maximum: MAX_GRACE_SECONDS }),
command,
};
}
function signalExitCode(signal) {
const signalNumber = os.constants.signals[signal];
return typeof signalNumber === "number" ? 128 + signalNumber : 1;
}
const { stage, timeoutSeconds, graceSeconds, command } = parseArguments(process.argv.slice(2));
const startedAt = performance.now();
const timeoutMilliseconds = timeoutSeconds * 1_000;
const graceMilliseconds = graceSeconds * 1_000;
process.stderr.write(`[docker-smoke] START stage=${stage} timeout_seconds=${timeoutSeconds} grace_seconds=${graceSeconds}\n`);
const child = spawn(command[0], command.slice(1), {
detached: true,
env: process.env,
shell: false,
stdio: "inherit",
});
let childClosed = false;
let finished = false;
let termination;
let timeoutTimer;
let forceKillTimer;
function durationMilliseconds() {
return Math.max(0, Math.round(performance.now() - startedAt));
}
function processGroupExists() {
if (!child.pid) {
return false;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return error?.code === "EPERM";
}
}
function signalProcessGroup(signal) {
if (!child.pid) {
return;
}
try {
process.kill(-child.pid, signal);
} catch (error) {
if (error?.code !== "ESRCH") {
try {
child.kill(signal);
} catch {
// The child may have exited between the process-group and direct kill.
}
}
}
}
function clearTimers() {
if (timeoutTimer) {
clearTimeout(timeoutTimer);
}
if (forceKillTimer) {
clearTimeout(forceKillTimer);
}
}
function finish(exitCode, outcome, extraMetadata = "") {
if (finished) {
return;
}
finished = true;
clearTimers();
process.removeListener("SIGINT", handleInterrupt);
process.removeListener("SIGTERM", handleTerminate);
const suffix = extraMetadata ? ` ${extraMetadata}` : "";
process.stderr.write(`[docker-smoke] ${outcome} stage=${stage} duration_ms=${durationMilliseconds()}${suffix} exit_code=${exitCode}\n`);
process.exitCode = exitCode;
}
function finishTerminationIfReady() {
if (!termination || (!childClosed && !termination.forceKillSent)) {
return;
}
if (processGroupExists() && !termination.forceKillSent) {
return;
}
if (termination.kind === "timeout") {
finish(TIMEOUT_EXIT_CODE, "TIMEOUT");
return;
}
const exitCode = signalExitCode(termination.signal);
finish(exitCode, "FAIL", `signal=${termination.signal}`);
}
function forceKillTermination() {
if (!termination || finished) {
return;
}
termination.forceKillSent = true;
signalProcessGroup("SIGKILL");
child.unref();
finishTerminationIfReady();
}
function beginTermination(nextTermination) {
if (finished) {
return;
}
if (termination) {
forceKillTermination();
return;
}
termination = { ...nextTermination, forceKillSent: false };
if (timeoutTimer) {
clearTimeout(timeoutTimer);
}
signalProcessGroup("SIGTERM");
forceKillTimer = setTimeout(forceKillTermination, graceMilliseconds);
finishTerminationIfReady();
}
function handleInterrupt() {
beginTermination({ kind: "signal", signal: "SIGINT" });
}
function handleTerminate() {
beginTermination({ kind: "signal", signal: "SIGTERM" });
}
process.on("SIGINT", handleInterrupt);
process.on("SIGTERM", handleTerminate);
child.once("error", () => {
childClosed = true;
if (termination) {
finishTerminationIfReady();
return;
}
finish(SPAWN_ERROR_EXIT_CODE, "FAIL", "reason=spawn_error");
});
child.once("close", (code, signal) => {
childClosed = true;
if (termination) {
finishTerminationIfReady();
return;
}
if (signal) {
const exitCode = signalExitCode(signal);
finish(exitCode, "FAIL", `signal=${signal}`);
return;
}
const exitCode = typeof code === "number" ? code : 1;
finish(exitCode, exitCode === 0 ? "PASS" : "FAIL");
});
timeoutTimer = setTimeout(() => {
beginTermination({ kind: "timeout" });
}, timeoutMilliseconds);
@@ -0,0 +1,589 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync, spawn, spawnSync } from "node:child_process";
import { afterEach, describe, it } from "vitest";
const runnerPath = path.resolve("scripts/lib/run-command-with-timeout.mjs");
const helperPath = path.resolve("scripts/lib/docker-runtime-smoke.sh");
const roleFixturePath = path.resolve("scripts/lib/docker-rls-role-fixture.sh");
const temporaryDirectories = [];
const remainingProcesses = new Set();
afterEach(() => {
for (const pid of remainingProcesses) {
try {
process.kill(pid, "SIGKILL");
} catch {
// The timeout runner should already have reaped the fixture process.
}
}
remainingProcesses.clear();
for (const directory of temporaryDirectories.splice(0)) {
fs.rmSync(directory, { recursive: true, force: true });
}
});
function runRunner(stage, timeoutSeconds, command, { graceSeconds = 0.1, env = process.env } = {}) {
return spawnSync(
process.execPath,
[runnerPath, "--stage", stage, "--timeout-seconds", String(timeoutSeconds), "--grace-seconds", String(graceSeconds), "--", ...command],
{ encoding: "utf8", env, timeout: 5_000 },
);
}
function createProcessTreeFixture() {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-timeout-tree-"));
temporaryDirectories.push(directory);
const pidFile = path.join(directory, "pids.json");
const grandchildPath = path.join(directory, "grandchild.mjs");
const parentPath = path.join(directory, "parent.mjs");
fs.writeFileSync(grandchildPath, ['import process from "node:process";', 'process.on("SIGTERM", () => {});', "setInterval(() => {}, 1_000);", ""].join("\n"));
fs.writeFileSync(
parentPath,
[
'import fs from "node:fs";',
'import process from "node:process";',
'import { spawn } from "node:child_process";',
"const [pidFile, grandchildPath] = process.argv.slice(2);",
'const grandchild = spawn(process.execPath, [grandchildPath], { stdio: "ignore" });',
"fs.writeFileSync(pidFile, JSON.stringify({ parent: process.pid, grandchild: grandchild.pid }));",
'process.on("SIGTERM", () => {});',
"setInterval(() => {}, 1_000);",
"",
].join("\n")
);
return { parentPath, pidFile, grandchildPath };
}
function readFixturePids(pidFile) {
const pids = JSON.parse(fs.readFileSync(pidFile, "utf8"));
remainingProcesses.add(pids.parent);
remainingProcesses.add(pids.grandchild);
return pids;
}
async function waitForFile(filePath, timeoutMilliseconds = 2_000) {
const deadline = Date.now() + timeoutMilliseconds;
while (Date.now() < deadline) {
if (fs.existsSync(filePath)) {
return;
}
await new Promise((resolve) => setTimeout(resolve, 20));
}
throw new Error("process-tree fixture did not become ready");
}
function isTerminatedLinuxProcessStat(stat) {
const commandEnd = stat.lastIndexOf(")");
const state = commandEnd >= 0 ? stat.slice(commandEnd + 2).split(" ", 1)[0] : "";
return state === "Z" || state === "X" || state === "x";
}
async function waitForProcessExit(pid, timeoutMilliseconds = 2_000) {
const deadline = Date.now() + timeoutMilliseconds;
while (Date.now() < deadline) {
try {
process.kill(pid, 0);
} catch (error) {
if (error?.code === "ESRCH") {
remainingProcesses.delete(pid);
return;
}
throw error;
}
// Linux keeps an exited process in /proc while its parent (or container PID 1)
// has not reaped the zombie yet. kill(pid, 0) still succeeds in that state even
// though the process can no longer execute, so it is valid evidence that the
// timeout runner killed the fixture. This occurs on the Gitea act runner, whose
// job container does not necessarily reap adopted grandchildren immediately.
if (process.platform === "linux") {
try {
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
if (isTerminatedLinuxProcessStat(stat)) {
remainingProcesses.delete(pid);
return;
}
} catch (error) {
if (error?.code === "ENOENT") {
remainingProcesses.delete(pid);
return;
}
throw error;
}
}
await new Promise((resolve) => setTimeout(resolve, 20));
}
throw new Error("timed-out process remained runnable");
}
describe("run-command-with-timeout", () => {
it("recognizes Linux zombie and dead states without accepting runnable states", () => {
for (const state of ["Z", "X", "x"]) {
assert.equal(isTerminatedLinuxProcessStat(`123 (fixture with ) paren) ${state} 1 2 3`), true);
}
for (const state of ["R", "S", "D", "T"]) {
assert.equal(isTerminatedLinuxProcessStat(`123 (fixture) ${state} 1 2 3`), false);
}
assert.equal(isTerminatedLinuxProcessStat("malformed"), false);
});
it("forwards output and reports stage-only metadata for successful commands", () => {
const secret = "secret-sentinel-success-42";
const result = runRunner(
"worker:inspect",
2,
[process.execPath, "-e", 'process.stdout.write("child-stdout\\n"); process.stderr.write("child-stderr\\n");', secret],
{ env: { ...process.env, SECRET_SENTINEL: secret } }
);
assert.equal(result.status, 0);
assert.equal(result.stdout, "child-stdout\n");
assert.match(result.stderr, /child-stderr/u);
assert.match(result.stderr, /\[docker-smoke\] START stage=worker:inspect timeout_seconds=2 grace_seconds=0\.1/u);
assert.match(result.stderr, /\[docker-smoke\] PASS stage=worker:inspect duration_ms=\d+ exit_code=0/u);
assert.doesNotMatch(result.stderr, new RegExp(secret, "u"));
});
it("preserves a failing command's exit code and streams", () => {
const result = runRunner("worker:migrate", 2, [
process.execPath,
"-e",
'process.stdout.write("migration-stdout\\n"); process.stderr.write("migration-stderr\\n"); process.exit(7);',
]);
assert.equal(result.status, 7);
assert.equal(result.stdout, "migration-stdout\n");
assert.match(result.stderr, /migration-stderr/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=worker:migrate duration_ms=\d+ exit_code=7/u);
});
it("returns 124 and kills a SIGTERM-ignoring process group", async () => {
const fixture = createProcessTreeFixture();
const result = runRunner("worker:timeout", 0.25, [process.execPath, fixture.parentPath, fixture.pidFile, fixture.grandchildPath], { graceSeconds: 0.1 });
const pids = readFixturePids(fixture.pidFile);
assert.equal(result.status, 124);
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:timeout duration_ms=\d+ exit_code=124/u);
await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]);
});
it("terminates the child group and exits signal-typically when the suite sends SIGTERM", async () => {
const fixture = createProcessTreeFixture();
const runner = spawn(
process.execPath,
[
runnerPath,
"--stage",
"suite:interrupt",
"--timeout-seconds",
"10",
"--grace-seconds",
"0.1",
"--",
process.execPath,
fixture.parentPath,
fixture.pidFile,
fixture.grandchildPath,
],
{ stdio: ["ignore", "pipe", "pipe"] }
);
let stderr = "";
runner.stderr.setEncoding("utf8");
runner.stderr.on("data", (chunk) => {
stderr += chunk;
});
try {
await waitForFile(fixture.pidFile);
const pids = readFixturePids(fixture.pidFile);
runner.kill("SIGTERM");
const result = await new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error("timeout runner did not exit after SIGTERM")), 3_000);
runner.once("error", reject);
runner.once("close", (code, signal) => {
clearTimeout(timeout);
resolve({ code, signal });
});
});
assert.deepEqual(result, { code: 143, signal: null });
assert.match(stderr, /\[docker-smoke\] FAIL stage=suite:interrupt duration_ms=\d+ signal=SIGTERM exit_code=143/u);
await Promise.all([waitForProcessExit(pids.parent), waitForProcessExit(pids.grandchild)]);
} finally {
if (runner.exitCode === null && runner.signalCode === null) {
runner.kill("SIGKILL");
}
}
});
it("rejects malformed metadata labels without echoing them", () => {
const secret = "secret sentinel invalid stage";
const result = spawnSync(process.execPath, [runnerPath, "--stage", secret, "--timeout-seconds", "1", "--", process.execPath, "-e", ""], {
encoding: "utf8",
});
assert.equal(result.status, 2);
assert.match(result.stderr, /reason=invalid_arguments/u);
assert.doesNotMatch(result.stderr, new RegExp(secret, "u"));
});
});
describe("docker-runtime-smoke helper", () => {
it("replays captured stdout to stderr when the command fails", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-capture-"));
temporaryDirectories.push(fixture);
const scriptPath = path.join(fixture, "capture.sh");
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
'captured=""',
"if docker_smoke_capture captured capture.failure 2 bash -c 'printf captured-stdout; printf child-stderr >&2; exit 7'; then",
" exit 99",
"else",
" status=$?",
"fi",
'printf "captured-variable=%s\\n" "$captured"',
'exit "$status"',
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], { encoding: "utf8", timeout: 5_000 });
assert.equal(result.status, 7);
assert.equal(result.stdout, "captured-variable=\n");
assert.match(result.stderr, /child-stderr/u);
assert.match(result.stderr, /captured-stdout/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=capture.failure duration_ms=\d+ exit_code=7/u);
});
it("requires an in-container rejection marker and rejects Docker transport failures", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "rejection.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "${FAKE_DOCKER_TRANSPORT_FAILURE:-0}" == "1" ]]; then',
' printf "Docker transport unavailable\\n" >&2',
" exit 1",
"fi",
'[[ "$1" == "exec" ]] || exit 2',
"shift",
"shift",
'"$@"',
"",
].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_expect_docker_exec_rejection rejection.product 2 fixture-container bash -c 'printf product-rejected >&2; exit 1'",
"",
].join("\n"),
{ mode: 0o755 },
);
const baseEnv = { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` };
const productRejection = spawnSync("bash", [scriptPath], { encoding: "utf8", env: baseEnv, timeout: 5_000 });
const transportFailure = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...baseEnv, FAKE_DOCKER_TRANSPORT_FAILURE: "1" },
timeout: 5_000,
});
assert.equal(productRejection.status, 0);
assert.match(productRejection.stderr, /\[docker-smoke\] ASSERT stage=rejection\.product outcome=expected_rejection/u);
assert.equal(transportFailure.status, 1);
assert.match(transportFailure.stderr, /Docker transport unavailable/u);
assert.match(transportFailure.stderr, /\[docker-smoke\] FAIL stage=rejection\.product duration_ms=\d+ exit_code=1/u);
assert.doesNotMatch(transportFailure.stderr, /outcome=expected_rejection/u);
});
it("rejects an unexpectedly successful in-container negative assertion", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-rejection-success-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "rejection-success.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
["#!/usr/bin/env bash", '[[ "$1" == "exec" ]] || exit 2', "shift", "shift", '"$@"', ""].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_expect_docker_exec_rejection rejection.unexpected-success 2 fixture-container bash -c 'exit 0'",
"",
].join("\n"),
{ mode: 0o755 },
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 64);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=rejection\.unexpected-success duration_ms=\d+ exit_code=64/u);
assert.doesNotMatch(result.stderr, /outcome=expected_rejection/u);
});
it("bounds hanging Docker cleanup while preserving the original exit status", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 });
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"export DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS=0.2",
"export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1",
"cleanup() {",
" docker_smoke_cleanup_container cleanup.container fixture-container",
"}",
"docker_smoke_install_cleanup_traps cleanup",
"exit 23",
"",
].join("\n"),
{ mode: 0o755 }
);
const startedAt = Date.now();
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 23);
assert.ok(Date.now() - startedAt < 3_000, "cleanup must be bounded by the helper timeout");
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=cleanup\.container duration_ms=\d+ exit_code=124/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.container reason=cleanup_incomplete exit_code=124/u);
});
it("treats already absent cleanup resources as an idempotent success", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-absent-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-absent.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "$1" == "container" && "$2" == "rm" ]]; then',
' printf "Error response from daemon: No such container: %s\\n" "$4" >&2',
" exit 1",
"fi",
'if [[ "$1" == "container" && "$2" == "inspect" ]]; then',
' printf "Error: No such object: %s\\n" "$3" >&2',
" exit 1",
"fi",
"exit 2",
"",
].join("\n"),
{ mode: 0o755 }
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_cleanup_container cleanup.absent absent-container",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 0);
assert.match(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.absent duration_ms=\d+ exit_code=0/u);
assert.doesNotMatch(result.stderr, /(?:FAIL|WARN|RETAINED|No such)/u);
});
it("does not misclassify an unknown Docker context failure as an absent resource", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-context-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-context.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'printf "unable to resolve docker endpoint: context review-context: context not found\\n" >&2',
"exit 1",
"",
].join("\n"),
{ mode: 0o755 },
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"docker_smoke_cleanup_container cleanup.context fixture-container",
"",
].join("\n"),
{ mode: 0o755 },
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 0);
assert.match(result.stderr, /context review-context: context not found/u);
assert.match(result.stderr, /\[docker-smoke\] CLEANUP_CHECK_FAILED kind=container name=fixture-container/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.context reason=cleanup_incomplete exit_code=1/u);
assert.doesNotMatch(result.stderr, /\[docker-smoke\] PASS stage=cleanup\.context/u);
});
it("reports retained cleanup resources without replacing the original exit status", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-cleanup-retained-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "cleanup-retained.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(
fakeDockerPath,
[
"#!/usr/bin/env bash",
'if [[ "$1" == "container" && "$2" == "rm" ]]; then',
' printf "removal already in progress\\n" >&2',
" exit 1",
"fi",
'if [[ "$1" == "container" && "$2" == "inspect" ]]; then',
" exit 0",
"fi",
"exit 2",
"",
].join("\n"),
{ mode: 0o755 }
);
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
"cleanup() {",
" docker_smoke_cleanup_container cleanup.retained retained-container",
"}",
"docker_smoke_install_cleanup_traps cleanup",
"exit 23",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 23);
assert.match(result.stderr, /\[docker-smoke\] RETAINED kind=container name=retained-container/u);
assert.match(result.stderr, /\[docker-smoke\] FAIL stage=cleanup\.retained duration_ms=\d+ exit_code=1/u);
assert.match(result.stderr, /\[docker-smoke\] WARN stage=cleanup\.retained reason=cleanup_incomplete exit_code=1/u);
});
it("bounds RLS role provisioning with a named stage without exposing fixture credentials", () => {
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "hivegrid-smoke-role-provision-"));
temporaryDirectories.push(fixture);
const binDirectory = path.join(fixture, "bin");
const fakeDockerPath = path.join(binDirectory, "docker");
const scriptPath = path.join(fixture, "provision.sh");
fs.mkdirSync(binDirectory);
fs.writeFileSync(fakeDockerPath, ["#!/usr/bin/env bash", "trap '' TERM", "while :; do", " sleep 1", "done", ""].join("\n"), { mode: 0o755 });
fs.writeFileSync(
scriptPath,
[
"#!/usr/bin/env bash",
"set -euo pipefail",
`source ${JSON.stringify(helperPath)}`,
`source ${JSON.stringify(roleFixturePath)}`,
"export DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS=0.2",
"export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS=0.1",
"provision_docker_smoke_rls_roles worker:provision-rls-roles fixture-db fixture-admin fixture-database migrator-secret app-secret worker-secret authentik-secret",
"",
].join("\n"),
{ mode: 0o755 }
);
const result = spawnSync("bash", [scriptPath], {
encoding: "utf8",
env: { ...process.env, PATH: `${binDirectory}${path.delimiter}${process.env.PATH}` },
timeout: 5_000,
});
assert.equal(result.status, 124);
assert.match(result.stderr, /\[docker-smoke\] TIMEOUT stage=worker:provision-rls-roles duration_ms=\d+ exit_code=124/u);
assert.doesNotMatch(result.stderr, /(?:migrator|app|worker|authentik)-secret/u);
});
it("exports overridable timeout defaults and remains valid Bash syntax", () => {
execFileSync("bash", ["-n", helperPath]);
const result = spawnSync(
"bash",
[
"-c",
[
"set -euo pipefail",
"export DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS=41",
`source ${JSON.stringify(helperPath)}`,
'printf \'%s,%s,%s,%s,%s,%s\' "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" "$DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS" "$DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS" "$DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS"',
].join("\n"),
],
{ encoding: "utf8" }
);
assert.equal(result.status, 0);
assert.equal(result.stdout, "41,5,10,15,90,60");
});
});