331 lines
14 KiB
Bash
Executable File
331 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
DOCKER_SMOKE_RUN_ID="${DOCKER_SMOKE_RUN_ID:-$$}"
|
|
if [[ -z "$DOCKER_SMOKE_RUN_ID" || ${#DOCKER_SMOKE_RUN_ID} -gt 80 || ! $DOCKER_SMOKE_RUN_ID =~ ^[A-Za-z0-9_.-]+$ ]]; then
|
|
echo "DOCKER_SMOKE_RUN_ID must contain 1-80 Docker-safe characters" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# shellcheck source=scripts/lib/docker-rls-role-fixture.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-rls-role-fixture.sh"
|
|
# shellcheck source=scripts/lib/docker-runtime-smoke.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-runtime-smoke.sh"
|
|
|
|
IMAGE="${1:-hivegrid-mcp:smoke}"
|
|
MIGRATOR_IMAGE="${DOCKER_SMOKE_MIGRATOR_IMAGE:-hivegrid-web:smoke}"
|
|
PORT="${DOCKER_MCP_SMOKE_PORT:-}"
|
|
CONTAINER_NAME="${DOCKER_MCP_SMOKE_CONTAINER:-hivegrid-mcp-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}"
|
|
REDIS_CONTAINER_NAME="${DOCKER_MCP_SMOKE_REDIS_CONTAINER:-hivegrid-mcp-runtime-smoke-redis-${DOCKER_SMOKE_RUN_ID}}"
|
|
DB_CONTAINER_NAME="${DOCKER_MCP_SMOKE_DB_CONTAINER:-hivegrid-mcp-runtime-smoke-db-${DOCKER_SMOKE_RUN_ID}}"
|
|
NETWORK_NAME="${DOCKER_MCP_SMOKE_NETWORK:-hivegrid-mcp-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}"
|
|
MIGRATION_CONTAINER_NAME="${DOCKER_MCP_SMOKE_MIGRATION_CONTAINER:-hivegrid-mcp-runtime-smoke-migrate-${DOCKER_SMOKE_RUN_ID}}"
|
|
ASSERT_CONTAINER_NAME="${DOCKER_MCP_SMOKE_ASSERT_CONTAINER:-hivegrid-mcp-runtime-smoke-assert-${DOCKER_SMOKE_RUN_ID}}"
|
|
TRANSPORT="${DOCKER_MCP_SMOKE_TRANSPORT:-host}"
|
|
EXPECTED_SMOKE_REVISION="${DOCKER_MCP_SMOKE_REVISION:-}"
|
|
SMOKE_REVISION=""
|
|
HEALTH_ATTEMPTS="${DOCKER_MCP_SMOKE_HEALTH_ATTEMPTS:-120}"
|
|
DEPENDENCY_ATTEMPTS="${DOCKER_MCP_SMOKE_DEPENDENCY_ATTEMPTS:-30}"
|
|
DB_PASSWORD="hivegrid"
|
|
DB_NAME="hivegrid"
|
|
DB_USER="hivegrid"
|
|
APP_DB_PASSWORD="hivegrid-app-smoke"
|
|
WORKER_DB_PASSWORD="hivegrid-worker-smoke"
|
|
MIGRATOR_DB_PASSWORD="hivegrid-migrator-smoke"
|
|
AUTHENTIK_JIT_DB_PASSWORD="hivegrid-authentik-jit-smoke"
|
|
|
|
cleanup() {
|
|
docker_smoke_cleanup_container "mcp:cleanup-containers" \
|
|
"$CONTAINER_NAME" "$MIGRATION_CONTAINER_NAME" "$ASSERT_CONTAINER_NAME" \
|
|
"$REDIS_CONTAINER_NAME" "$DB_CONTAINER_NAME"
|
|
docker_smoke_cleanup_network "mcp:cleanup-network" "$NETWORK_NAME"
|
|
}
|
|
|
|
resolve_smoke_revision() {
|
|
local image_revision
|
|
|
|
docker_smoke_capture image_revision "mcp:inspect-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE"
|
|
if [[ -n "$EXPECTED_SMOKE_REVISION" && "$EXPECTED_SMOKE_REVISION" != "$image_revision" ]]; then
|
|
echo "Image ${IMAGE} revision ${image_revision:-<empty>} does not match expected revision ${EXPECTED_SMOKE_REVISION}" >&2
|
|
return 1
|
|
fi
|
|
|
|
printf "%s" "$image_revision"
|
|
}
|
|
|
|
verify_migrator_revision() {
|
|
local migrator_revision
|
|
|
|
docker_smoke_capture migrator_revision "mcp:inspect-migrator-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$MIGRATOR_IMAGE"
|
|
if [[ "$migrator_revision" != "$SMOKE_REVISION" ]]; then
|
|
echo "Migrator image ${MIGRATOR_IMAGE} revision ${migrator_revision:-<empty>} does not match MCP image revision ${SMOKE_REVISION}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
http_status() {
|
|
local path="$1"
|
|
local method="${2:-GET}"
|
|
local code
|
|
|
|
if [[ "$TRANSPORT" == "exec" ]]; then
|
|
if ! docker_smoke_capture code "mcp:http-status" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker exec "$CONTAINER_NAME" node -e 'const http = require("node:http"); const [path, method] = process.argv.slice(1); const request = http.request({ hostname: "127.0.0.1", port: 8080, path, method, headers: { Host: "mcp.hivegrid.de" } }, (response) => { process.stdout.write(String(response.statusCode ?? "000")); response.resume(); }); request.on("error", () => process.stdout.write("000")); request.end();' "$path" "$method" 2>/dev/null; then
|
|
code="000"
|
|
fi
|
|
else
|
|
if ! docker_smoke_capture code "mcp:http-status" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
curl -s -o /dev/null -w "%{http_code}" -X "$method" -H "Host: mcp.hivegrid.de" "http://127.0.0.1:${PORT}${path}" 2>/dev/null; then
|
|
code="000"
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$code" ]]; then
|
|
printf "000"
|
|
return
|
|
fi
|
|
|
|
printf "%s" "$code"
|
|
}
|
|
|
|
http_body() {
|
|
local path="$1"
|
|
local body
|
|
|
|
if [[ "$TRANSPORT" == "exec" ]]; then
|
|
docker_smoke_capture body "mcp:http-body" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker exec "$CONTAINER_NAME" node -e 'const http = require("node:http"); const [path] = process.argv.slice(1); const request = http.request({ hostname: "127.0.0.1", port: 8080, path, headers: { Host: "mcp.hivegrid.de" } }, (response) => { let body = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { body += chunk; }); response.on("end", () => process.stdout.write(body)); }); request.on("error", () => process.exit(1)); request.end();' "$path"
|
|
else
|
|
docker_smoke_capture body "mcp:http-body" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
curl --fail --silent --show-error -H "Host: mcp.hivegrid.de" "http://127.0.0.1:${PORT}${path}"
|
|
fi
|
|
|
|
printf "%s" "$body"
|
|
}
|
|
|
|
expect_status() {
|
|
local path="$1"
|
|
local expected="$2"
|
|
local method="${3:-GET}"
|
|
local actual
|
|
|
|
actual="$(http_status "$path" "$method")"
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo "Expected ${method} ${path} to return ${expected}, got ${actual}" >&2
|
|
docker_smoke_logs "mcp:logs-status" "$CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
wait_for_status() {
|
|
local path="$1"
|
|
local expected="$2"
|
|
local description="$3"
|
|
|
|
for attempt in $(seq 1 "$DEPENDENCY_ATTEMPTS"); do
|
|
if [[ "$(http_status "$path")" == "$expected" ]]; then
|
|
return
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
echo "MCP ${description} did not reach HTTP ${expected} within ${DEPENDENCY_ATTEMPTS}s" >&2
|
|
docker_smoke_logs "mcp:logs-readiness" "$CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
expect_health_revision() {
|
|
local body
|
|
local actual
|
|
local service
|
|
|
|
body="$(http_body /healthz)"
|
|
actual="$(printf "%s" "$body" | node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(0, "utf8")); process.stdout.write(payload.revision || "");')"
|
|
service="$(printf "%s" "$body" | node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(0, "utf8")); process.stdout.write(payload.service || "");')"
|
|
if [[ "$actual" != "$SMOKE_REVISION" || "$service" != "mcp" ]]; then
|
|
echo "Expected /healthz service=mcp revision=${SMOKE_REVISION}, got service=${service:-<empty>} revision=${actual:-<empty>}" >&2
|
|
docker_smoke_logs "mcp:logs-revision" "$CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
expect_deploy_health() {
|
|
local output="$1"
|
|
|
|
printf "%s" "$output" | node -e '
|
|
const expectedRevision = process.argv[1];
|
|
const payload = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
|
|
if (payload.status !== "ready" || payload.service !== "mcp" || payload.revision !== expectedRevision) {
|
|
console.error("MCP deployment health did not prove the expected live revision");
|
|
process.exit(1);
|
|
}
|
|
' "$SMOKE_REVISION"
|
|
}
|
|
|
|
docker_smoke_install_cleanup_traps cleanup
|
|
cleanup
|
|
SMOKE_REVISION="$(resolve_smoke_revision)"
|
|
if [[ -z "$SMOKE_REVISION" || "$SMOKE_REVISION" == "<no value>" || "$SMOKE_REVISION" == "unknown" ]]; then
|
|
echo "Image ${IMAGE} does not expose a concrete org.opencontainers.image.revision; build with --build-arg APP_REVISION" >&2
|
|
exit 1
|
|
fi
|
|
verify_migrator_revision
|
|
|
|
docker_smoke_run "mcp:create-network" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
|
|
docker network create "$NETWORK_NAME" >/dev/null
|
|
docker_smoke_run "mcp:start-redis" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
|
|
docker run -d --name "$REDIS_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias redis redis:7-alpine >/dev/null
|
|
docker_smoke_run "mcp:start-db" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$DB_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias db \
|
|
-e POSTGRES_USER="$DB_USER" \
|
|
-e POSTGRES_PASSWORD="$DB_PASSWORD" \
|
|
-e POSTGRES_DB="$DB_NAME" \
|
|
postgres:16-alpine >/dev/null
|
|
|
|
MIGRATOR_DATABASE_URL="postgresql://hivegrid_migrator:${MIGRATOR_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public"
|
|
APP_DATABASE_URL="postgresql://hivegrid_app:${APP_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public"
|
|
|
|
for attempt in $(seq 1 30); do
|
|
if docker_smoke_probe "mcp:probe-redis" docker exec "$REDIS_CONTAINER_NAME" redis-cli ping 2>/dev/null | grep -q PONG; then
|
|
break
|
|
fi
|
|
|
|
if [[ "$attempt" == "30" ]]; then
|
|
echo "Redis smoke service did not become ready" >&2
|
|
docker_smoke_logs "mcp:logs-redis" "$REDIS_CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
# Probe over TCP, not the unix socket. On first boot the postgres image starts a
|
|
# temporary server that listens on the socket ONLY (initdb + init scripts), then
|
|
# shuts it down before starting the real one. A socket-based pg_isready reports
|
|
# ready during that window, so the very next psql can hit
|
|
# "FATAL: the database system is shutting down". The temp server sets
|
|
# listen_addresses='', so a TCP probe cannot succeed too early.
|
|
if ! docker_smoke_wait_for_postgres "mcp:probe-db" \
|
|
"$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME"; then
|
|
echo "Postgres smoke database did not become ready within the ${DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS}-second budget" >&2
|
|
docker_smoke_logs "mcp:logs-db" "$DB_CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
provision_docker_smoke_rls_roles \
|
|
"mcp:provision-rls-roles" \
|
|
"$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME" \
|
|
"$MIGRATOR_DB_PASSWORD" "$APP_DB_PASSWORD" "$WORKER_DB_PASSWORD" "$AUTHENTIK_JIT_DB_PASSWORD"
|
|
|
|
# Exercise the same reviewed migration closure as deployment. MCP readiness
|
|
# queries both credential tables, so an unmigrated database must stay unready.
|
|
docker_smoke_run "mcp:migrate" "$DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS" docker run --rm --name "$MIGRATION_CONTAINER_NAME" --network "$NETWORK_NAME" \
|
|
-e DIRECT_URL="$MIGRATOR_DATABASE_URL" \
|
|
"$MIGRATOR_IMAGE" migrate
|
|
|
|
# MCP instances never migrate schemas or compile source. Keep build-only
|
|
# toolchains out of the remotely reachable runtime closure.
|
|
docker_smoke_run "mcp:assert-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run --rm --name "$ASSERT_CONTAINER_NAME" --entrypoint node "$IMAGE" -e '
|
|
for (const specifier of ["prisma", "typescript", "@prisma/engines"]) {
|
|
try {
|
|
require.resolve(specifier);
|
|
throw new Error(`MCP runtime retained build-only package ${specifier}`);
|
|
} catch (error) {
|
|
if (error?.code !== "MODULE_NOT_FOUND") throw error;
|
|
}
|
|
}
|
|
' >/dev/null
|
|
|
|
publish_args=(-p 127.0.0.1::8080)
|
|
if [[ -n "$PORT" ]]; then
|
|
publish_args=(-p "${PORT}:8080")
|
|
fi
|
|
|
|
docker_smoke_run "mcp:start-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$CONTAINER_NAME" --network "$NETWORK_NAME" "${publish_args[@]}" \
|
|
-e NODE_ENV=production \
|
|
-e PORT=8080 \
|
|
-e MCP_PORT=8080 \
|
|
-e DATABASE_URL="$APP_DATABASE_URL" \
|
|
-e DATABASE_RUNTIME_ROLE=hivegrid_app \
|
|
-e RLS_RUNTIME_MODE=required \
|
|
-e REDIS_URL='redis://redis:6379/0' \
|
|
-e MCP_OAUTH_TOKEN_HASH_SECRET="smoke-only-mcp-token-hash-secret-not-real" \
|
|
-e AUDIT_CHAIN_SECRET='deploy-smoke-audit-chain-secret-minimum-32' \
|
|
"$IMAGE" >/dev/null
|
|
|
|
if [[ -z "$PORT" ]]; then
|
|
docker_smoke_capture PORT "mcp:resolve-port" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker port "$CONTAINER_NAME" 8080/tcp
|
|
PORT="$(printf "%s\n" "$PORT" | awk -F: 'NR == 1 { print $NF }')"
|
|
fi
|
|
|
|
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
|
if [[ "$(http_status /healthz)" == "200" ]]; then
|
|
break
|
|
fi
|
|
|
|
if [[ "$attempt" == "$HEALTH_ATTEMPTS" ]]; then
|
|
echo "MCP container did not become healthy on port ${PORT} after ${HEALTH_ATTEMPTS}s" >&2
|
|
docker_smoke_logs "mcp:logs-health" "$CONTAINER_NAME" >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
expect_status /healthz 200
|
|
expect_health_revision
|
|
wait_for_status /readyz 200 "initial dependency readiness"
|
|
docker_smoke_run "mcp:assert-revision-file" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker exec "$CONTAINER_NAME" node -e '
|
|
const fs = require("node:fs");
|
|
const expectedRevision = process.argv[1];
|
|
const path = "/myapp/.image-revision";
|
|
const stat = fs.statSync(path);
|
|
const actualRevision = fs.readFileSync(path, "utf8").trim();
|
|
if (stat.uid !== 0 || (stat.mode & 0o022) !== 0 || actualRevision !== expectedRevision) {
|
|
console.error("MCP immutable image revision file is not root-owned, read-only, and exact");
|
|
process.exit(1);
|
|
}
|
|
' "$SMOKE_REVISION"
|
|
deploy_health_output=""
|
|
docker_smoke_capture deploy_health_output "mcp:deploy-health" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker exec "$CONTAINER_NAME" node apps/mcp/dist/deploy-health.js "$SMOKE_REVISION"
|
|
expect_deploy_health "$deploy_health_output"
|
|
MISMATCHED_REVISION="$(printf '0%.0s' {1..40})"
|
|
if [[ "$MISMATCHED_REVISION" == "$SMOKE_REVISION" ]]; then
|
|
MISMATCHED_REVISION="$(printf 'f%.0s' {1..40})"
|
|
fi
|
|
docker_smoke_expect_docker_exec_rejection "mcp:reject-spoofed-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
|
|
env APP_REVISION="$MISMATCHED_REVISION" \
|
|
node apps/mcp/dist/deploy-health.js "$MISMATCHED_REVISION"
|
|
expect_status /mcp 404 POST
|
|
expect_status / 401 POST
|
|
|
|
# Readiness must reflect the exact Redis and Postgres dependencies used by
|
|
# authenticated MCP traffic while liveness remains process-only.
|
|
docker_smoke_run "mcp:stop-redis" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker stop "$REDIS_CONTAINER_NAME" >/dev/null
|
|
wait_for_status /readyz 503 "Redis outage readiness"
|
|
expect_status /healthz 200
|
|
docker_smoke_expect_docker_exec_rejection "mcp:reject-redis-outage" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
|
|
node apps/mcp/dist/deploy-health.js "$SMOKE_REVISION"
|
|
|
|
docker_smoke_run "mcp:start-redis-recovery" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker start "$REDIS_CONTAINER_NAME" >/dev/null
|
|
for attempt in $(seq 1 30); do
|
|
if docker_smoke_probe "mcp:probe-redis-recovery" docker exec "$REDIS_CONTAINER_NAME" redis-cli ping 2>/dev/null | grep -q PONG; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
wait_for_status /readyz 200 "Redis recovery readiness"
|
|
docker_smoke_capture deploy_health_output "mcp:deploy-health-recovery" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
|
|
docker exec "$CONTAINER_NAME" node apps/mcp/dist/deploy-health.js "$SMOKE_REVISION"
|
|
expect_deploy_health "$deploy_health_output"
|
|
|
|
docker_smoke_run "mcp:stop-db" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker stop "$DB_CONTAINER_NAME" >/dev/null
|
|
wait_for_status /readyz 503 "Postgres outage readiness"
|
|
expect_status /healthz 200
|
|
docker_smoke_expect_docker_exec_rejection "mcp:reject-db-outage" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
|
|
node apps/mcp/dist/deploy-health.js "$SMOKE_REVISION"
|
|
|
|
echo "Docker MCP runtime smoke passed for ${IMAGE}"
|