363 lines
10 KiB
Bash
363 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Shared, Bash 3.2-compatible execution boundaries for Docker runtime smokes.
|
|
# The Node runner deliberately logs stage metadata only; command arguments and
|
|
# environment values must never be copied into wrapper diagnostics.
|
|
|
|
_DOCKER_SMOKE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
export DOCKER_SMOKE_NODE_BIN="${DOCKER_SMOKE_NODE_BIN:-node}"
|
|
export DOCKER_SMOKE_TIMEOUT_RUNNER="${DOCKER_SMOKE_TIMEOUT_RUNNER:-${_DOCKER_SMOKE_LIB_DIR}/run-command-with-timeout.mjs}"
|
|
export DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS="${DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS:-30}"
|
|
export DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS="${DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS:-5}"
|
|
export DOCKER_SMOKE_LOG_TIMEOUT_SECONDS="${DOCKER_SMOKE_LOG_TIMEOUT_SECONDS:-10}"
|
|
export DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS="${DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS:-15}"
|
|
export DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS="${DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS:-90}"
|
|
export DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS="${DOCKER_SMOKE_REDIS_READY_TIMEOUT_SECONDS:-60}"
|
|
export DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS="${DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS:-300}"
|
|
export DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS="${DOCKER_SMOKE_ROLE_PROVISION_TIMEOUT_SECONDS:-30}"
|
|
export DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS="${DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS:-2}"
|
|
export DOCKER_SMOKE_LOG_TAIL="${DOCKER_SMOKE_LOG_TAIL:-200}"
|
|
DOCKER_SMOKE_EXPECTED_REJECTION_MARKER="hivegrid-smoke-expected-rejection-v1"
|
|
|
|
_DOCKER_SMOKE_CLEANUP_CALLBACK=""
|
|
_DOCKER_SMOKE_CLEANUP_ACTIVE=0
|
|
|
|
docker_smoke_run() {
|
|
if [[ "$#" -lt 3 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
|
|
shift 2
|
|
|
|
docker_smoke_run_with_grace "$stage" "$timeout_seconds" "$DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS" "$@"
|
|
}
|
|
|
|
docker_smoke_run_with_grace() {
|
|
if [[ "$#" -lt 4 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
|
|
local grace_seconds="${3:-$DOCKER_SMOKE_TIMEOUT_GRACE_SECONDS}"
|
|
shift 3
|
|
|
|
"$DOCKER_SMOKE_NODE_BIN" "$DOCKER_SMOKE_TIMEOUT_RUNNER" \
|
|
--stage "$stage" \
|
|
--timeout-seconds "$timeout_seconds" \
|
|
--grace-seconds "$grace_seconds" \
|
|
-- "$@"
|
|
}
|
|
|
|
docker_smoke_expect_docker_exec_rejection() {
|
|
if [[ "$#" -lt 4 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
local timeout_seconds="${2:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
|
|
local container_name="$3"
|
|
local observed_output=""
|
|
local command_status
|
|
shift 3
|
|
|
|
if docker_smoke_capture observed_output "$stage" "$timeout_seconds" \
|
|
docker exec "$container_name" sh -c '
|
|
marker="$1"
|
|
shift
|
|
"$@"
|
|
command_status=$?
|
|
if [ "$command_status" -eq 1 ]; then
|
|
printf "%s" "$marker"
|
|
exit 0
|
|
fi
|
|
if [ "$command_status" -eq 0 ]; then
|
|
exit 64
|
|
fi
|
|
exit "$command_status"
|
|
' docker-smoke-rejection "$DOCKER_SMOKE_EXPECTED_REJECTION_MARKER" "$@"; then
|
|
command_status=0
|
|
else
|
|
command_status=$?
|
|
fi
|
|
|
|
if [[ "$command_status" -ne 0 ]]; then
|
|
return "$command_status"
|
|
fi
|
|
if [[ "$observed_output" != "$DOCKER_SMOKE_EXPECTED_REJECTION_MARKER" ]]; then
|
|
printf '%s\n' "[docker-smoke] FAIL stage=${stage} reason=missing_expected_rejection_marker exit_code=1" >&2
|
|
return 1
|
|
fi
|
|
|
|
printf '%s\n' "[docker-smoke] ASSERT stage=${stage} outcome=expected_rejection" >&2
|
|
}
|
|
|
|
docker_smoke_capture() {
|
|
if [[ "$#" -lt 4 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local output_variable="$1"
|
|
local stage="$2"
|
|
local timeout_seconds="${3:-$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS}"
|
|
local _docker_smoke_captured_value
|
|
local command_status
|
|
shift 3
|
|
|
|
if [[ ! "$output_variable" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_output_variable exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
if _docker_smoke_captured_value="$(docker_smoke_run "$stage" "$timeout_seconds" "$@")"; then
|
|
command_status=0
|
|
else
|
|
command_status=$?
|
|
fi
|
|
|
|
if [[ "$command_status" -ne 0 ]]; then
|
|
if [[ -n "$_docker_smoke_captured_value" ]]; then
|
|
printf '%s\n' "$_docker_smoke_captured_value" >&2
|
|
fi
|
|
return "$command_status"
|
|
fi
|
|
|
|
printf -v "$output_variable" '%s' "$_docker_smoke_captured_value"
|
|
}
|
|
|
|
docker_smoke_probe() {
|
|
if [[ "$#" -lt 2 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
shift
|
|
docker_smoke_run "$stage" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$@"
|
|
}
|
|
|
|
docker_smoke_wait_for_postgres() {
|
|
if [[ "$#" -ne 4 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=postgres-readiness reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
local container_name="$2"
|
|
local database_user="$3"
|
|
local database_name="$4"
|
|
local deadline_seconds=$((SECONDS + DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS))
|
|
|
|
while [[ "$SECONDS" -lt "$deadline_seconds" ]]; do
|
|
if docker_smoke_probe "$stage" docker exec "$container_name" \
|
|
pg_isready -h 127.0.0.1 -U "$database_user" -d "$database_name" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
docker_smoke_logs() {
|
|
if [[ "$#" -ne 2 ]]; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_arguments exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
local stage="$1"
|
|
local container_name="$2"
|
|
docker_smoke_run "$stage" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" \
|
|
docker logs --tail "$DOCKER_SMOKE_LOG_TAIL" "$container_name"
|
|
}
|
|
|
|
docker_smoke_cleanup_run() {
|
|
if [[ "$#" -lt 2 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local stage="$1"
|
|
local cleanup_status
|
|
shift
|
|
if docker_smoke_run "$stage" "$DOCKER_SMOKE_CLEANUP_TIMEOUT_SECONDS" "$@" >/dev/null; then
|
|
return 0
|
|
else
|
|
cleanup_status=$?
|
|
fi
|
|
|
|
printf '%s\n' "[docker-smoke] WARN stage=${stage} reason=cleanup_incomplete exit_code=${cleanup_status}" >&2
|
|
return 0
|
|
}
|
|
|
|
_docker_smoke_cleanup_resources() {
|
|
if [[ "$#" -lt 3 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local resource_kind="$1"
|
|
local stage="$2"
|
|
shift 2
|
|
|
|
docker_smoke_cleanup_run "$stage" bash -c '
|
|
set -u
|
|
resource_kind="$1"
|
|
shift
|
|
removal_status=0
|
|
removal_output=""
|
|
|
|
case "$resource_kind" in
|
|
container)
|
|
removal_output="$(docker container rm -f "$@" 2>&1 >/dev/null)" || removal_status=$?
|
|
;;
|
|
network|volume)
|
|
removal_output="$(docker "$resource_kind" rm "$@" 2>&1 >/dev/null)" || removal_status=$?
|
|
;;
|
|
*)
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
cleanup_incomplete=0
|
|
for resource_name in "$@"; do
|
|
inspect_status=0
|
|
resource_absent=0
|
|
inspect_output="$(docker "$resource_kind" inspect "$resource_name" 2>&1 >/dev/null)" || inspect_status=$?
|
|
if [[ "$inspect_status" -eq 0 ]]; then
|
|
printf "[docker-smoke] RETAINED kind=%s name=%s\n" "$resource_kind" "$resource_name" >&2
|
|
cleanup_incomplete=1
|
|
continue
|
|
fi
|
|
|
|
case "$resource_kind" in
|
|
container)
|
|
case "$inspect_output" in
|
|
*"No such object: $resource_name"*|*"No such container: $resource_name"*) resource_absent=1 ;;
|
|
esac
|
|
;;
|
|
network)
|
|
case "$inspect_output" in
|
|
*"No such network: $resource_name"*|*"network $resource_name not found"*) resource_absent=1 ;;
|
|
esac
|
|
;;
|
|
volume)
|
|
case "$inspect_output" in
|
|
*"No such volume: $resource_name"*|*"no such volume: $resource_name"*|*"get $resource_name: no such volume"*) resource_absent=1 ;;
|
|
esac
|
|
;;
|
|
esac
|
|
|
|
if [[ "$resource_absent" -ne 1 ]]; then
|
|
if [[ -n "$inspect_output" ]]; then
|
|
printf "%s\n" "$inspect_output" >&2
|
|
fi
|
|
printf "[docker-smoke] CLEANUP_CHECK_FAILED kind=%s name=%s\n" "$resource_kind" "$resource_name" >&2
|
|
cleanup_incomplete=1
|
|
fi
|
|
done
|
|
|
|
if [[ "$cleanup_incomplete" -eq 1 ]]; then
|
|
if [[ -n "$removal_output" ]]; then
|
|
printf "%s\n" "$removal_output" >&2
|
|
fi
|
|
if [[ "$removal_status" -eq 0 ]]; then
|
|
removal_status=1
|
|
fi
|
|
exit "$removal_status"
|
|
fi
|
|
' docker-smoke-cleanup "$resource_kind" "$@"
|
|
}
|
|
|
|
docker_smoke_cleanup_container() {
|
|
if [[ "$#" -lt 2 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local stage="$1"
|
|
shift
|
|
_docker_smoke_cleanup_resources container "$stage" "$@"
|
|
}
|
|
|
|
docker_smoke_cleanup_network() {
|
|
if [[ "$#" -lt 2 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local stage="$1"
|
|
shift
|
|
_docker_smoke_cleanup_resources network "$stage" "$@"
|
|
}
|
|
|
|
docker_smoke_cleanup_volume() {
|
|
if [[ "$#" -lt 2 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local stage="$1"
|
|
shift
|
|
_docker_smoke_cleanup_resources volume "$stage" "$@"
|
|
}
|
|
|
|
_docker_smoke_invoke_cleanup() {
|
|
if [[ "$_DOCKER_SMOKE_CLEANUP_ACTIVE" -eq 1 || -z "$_DOCKER_SMOKE_CLEANUP_CALLBACK" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
_DOCKER_SMOKE_CLEANUP_ACTIVE=1
|
|
if "$_DOCKER_SMOKE_CLEANUP_CALLBACK"; then
|
|
:
|
|
else
|
|
:
|
|
fi
|
|
_DOCKER_SMOKE_CLEANUP_ACTIVE=0
|
|
return 0
|
|
}
|
|
|
|
_docker_smoke_handle_exit() {
|
|
local original_status="$1"
|
|
trap - EXIT INT TERM
|
|
_docker_smoke_invoke_cleanup
|
|
exit "$original_status"
|
|
}
|
|
|
|
_docker_smoke_handle_signal() {
|
|
local signal_name="$1"
|
|
local signal_status
|
|
|
|
case "$signal_name" in
|
|
INT)
|
|
signal_status=130
|
|
;;
|
|
TERM)
|
|
signal_status=143
|
|
;;
|
|
*)
|
|
signal_status=1
|
|
;;
|
|
esac
|
|
|
|
trap - EXIT INT TERM
|
|
_docker_smoke_invoke_cleanup
|
|
exit "$signal_status"
|
|
}
|
|
|
|
docker_smoke_install_cleanup_traps() {
|
|
if [[ "$#" -ne 1 || -z "$1" ]] || ! declare -F "$1" >/dev/null 2>&1; then
|
|
printf '%s\n' '[docker-smoke] FAIL stage=helper reason=invalid_cleanup_callback exit_code=2' >&2
|
|
return 2
|
|
fi
|
|
|
|
_DOCKER_SMOKE_CLEANUP_CALLBACK="$1"
|
|
trap '_docker_smoke_handle_exit "$?"' EXIT
|
|
trap '_docker_smoke_handle_signal INT' INT
|
|
trap '_docker_smoke_handle_signal TERM' TERM
|
|
}
|
|
|
|
unset _DOCKER_SMOKE_LIB_DIR
|