Track HiveGrid Cloud planning reference

This commit is contained in:
david committed 2026-06-26 20:48:01 +02:00
1 parent b3d5ea6764
commit e341c5938a
5784 files changed
+1104929 -3

No files matched your search

@@ -0,0 +1,53 @@
# infra/sentry/backup/Dockerfile
#
# Sidecar container that backs up the self-hosted Sentry stack daily:
# - pg_dump --format=custom of the sentry Postgres
# - clickhouse-backup snapshot uploaded directly to S3
# - both artifacts encrypted with GPG (or AES256 fallback) before upload
#
# Triggered by an internal cron (see crontab) so the container stays running
# in the background. Re-uses the upstream Postgres + ClickHouse client tools
# at versions matching the Sentry stack (Postgres 14, ClickHouse 23.8).
FROM alpine:3.20
# --- Runtime deps ------------------------------------------------------------
# postgresql-client -> pg_dump
# clickhouse-client -> clickhouse-client
# aws-cli -> S3 upload (works with Hetzner Object Storage + MinIO via --endpoint-url)
# rclone -> alternative S3 backend (handy for MinIO-compat that AWS-CLI v2 dislikes)
# gnupg -> GPG encryption of dumps
# tini -> proper signal-handling as PID 1
# bash, coreutils -> POSIX-strict scripts need real coreutils
# tzdata -> consistent UTC timestamps
# busybox-extras (cron) -> crond
RUN apk add --no-cache \
postgresql16-client \
clickhouse-client \
aws-cli \
rclone \
gnupg \
tini \
bash \
coreutils \
tzdata \
busybox-extras \
curl \
&& rm -rf /var/cache/apk/*
ENV TZ=UTC \
BACKUP_HOME=/srv/backup \
PATH=/srv/backup:$PATH
WORKDIR ${BACKUP_HOME}
COPY backup.sh ${BACKUP_HOME}/backup.sh
COPY crontab /etc/crontabs/root
RUN chmod 0755 ${BACKUP_HOME}/backup.sh \
&& mkdir -p ${BACKUP_HOME}/logs ${BACKUP_HOME}/work
# Run cron in the foreground; tini handles SIGTERM cleanly so docker compose
# down doesn't strand a half-finished dump.
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["crond", "-f", "-L", "/dev/stdout", "-l", "2"]
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env bash
# infra/sentry/backup/backup.sh
#
# Daily backup of the self-hosted Sentry stack. Triggered by crond at 03:30 UTC
# (see crontab). Idempotent — re-running on the same day overwrites the same
# S3 prefix.
#
# Env (sourced from the Compose service `environment` block which itself reads
# from infra/sentry/.env):
# SENTRY_S3_BACKUP_ENDPOINT S3 endpoint URL (Hetzner / MinIO / AWS)
# SENTRY_S3_BACKUP_BUCKET target bucket name
# SENTRY_S3_BACKUP_ACCESS_KEY S3 access key
# SENTRY_S3_BACKUP_SECRET_KEY S3 secret key
# SENTRY_SECRET_KEY fallback symmetric passphrase for GPG
# SENTRY_BACKUP_GPG_RECIPIENT optional GPG recipient fingerprint
#
# SENTRY_PG_HOST default "postgres" (upstream service name)
# SENTRY_PG_PORT default 5432
# SENTRY_PG_USER default "postgres"
# SENTRY_PG_DB default "postgres"
# SENTRY_PG_PASSWORD from PGPASSWORD or env
#
# SENTRY_CH_HOST default "clickhouse"
# SENTRY_CH_PORT default 9000
# SENTRY_CH_USER default "default"
# SENTRY_CH_PASSWORD optional
set -Eeuo pipefail
shopt -s inherit_errexit 2>/dev/null || true
# -----------------------------------------------------------------------------
# Defaults + logging
# -----------------------------------------------------------------------------
: "${SENTRY_PG_HOST:=postgres}"
: "${SENTRY_PG_PORT:=5432}"
: "${SENTRY_PG_USER:=postgres}"
: "${SENTRY_PG_DB:=postgres}"
: "${SENTRY_CH_HOST:=clickhouse}"
: "${SENTRY_CH_PORT:=9000}"
: "${SENTRY_CH_USER:=default}"
: "${SENTRY_CH_PASSWORD:=}"
: "${SENTRY_S3_BACKUP_ENDPOINT:?SENTRY_S3_BACKUP_ENDPOINT is required}"
: "${SENTRY_S3_BACKUP_BUCKET:?SENTRY_S3_BACKUP_BUCKET is required}"
: "${SENTRY_S3_BACKUP_ACCESS_KEY:?SENTRY_S3_BACKUP_ACCESS_KEY is required}"
: "${SENTRY_S3_BACKUP_SECRET_KEY:?SENTRY_S3_BACKUP_SECRET_KEY is required}"
TODAY="$(date -u +%Y-%m-%d)"
NOW="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
WORK="${BACKUP_HOME:-/srv/backup}/work/${TODAY}"
mkdir -p "${WORK}"
log() { printf "[backup %s] %s\n" "${NOW}" "$*"; }
cleanup() { rm -rf "${WORK}"; }
trap cleanup EXIT
# -----------------------------------------------------------------------------
# Step 1: Postgres dump
# -----------------------------------------------------------------------------
log "dumping Postgres (${SENTRY_PG_HOST}/${SENTRY_PG_DB})"
PG_DUMP_FILE="${WORK}/postgres-${NOW}.dump"
export PGPASSWORD="${SENTRY_PG_PASSWORD:-${PGPASSWORD:-}}"
pg_dump \
--host "${SENTRY_PG_HOST}" \
--port "${SENTRY_PG_PORT}" \
--username "${SENTRY_PG_USER}" \
--dbname "${SENTRY_PG_DB}" \
--format=custom \
--compress=9 \
--no-owner \
--no-privileges \
--file "${PG_DUMP_FILE}"
log "postgres dump size: $(du -h "${PG_DUMP_FILE}" | cut -f1)"
# -----------------------------------------------------------------------------
# Step 2: ClickHouse snapshot
# -----------------------------------------------------------------------------
log "snapshotting ClickHouse (${SENTRY_CH_HOST})"
CH_DUMP_FILE="${WORK}/clickhouse-${NOW}.tar.gz"
# Use clickhouse-client to BACKUP all sentry tables to /var/lib/clickhouse/backup,
# then archive the directory. The `BACKUP ... TO Disk('backups', ...)` syntax
# is preferred but requires backup-disk config; we fall back to per-database
# SELECT...INTO OUTFILE for portability.
CH_TMP="${WORK}/clickhouse"
mkdir -p "${CH_TMP}"
DATABASES="$(clickhouse-client \
--host "${SENTRY_CH_HOST}" \
--port "${SENTRY_CH_PORT}" \
--user "${SENTRY_CH_USER}" \
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
--query="SHOW DATABASES" | grep -Ev '^(system|INFORMATION_SCHEMA|information_schema|default)$' || true)"
for db in ${DATABASES}; do
log " -> dumping clickhouse db: ${db}"
TABLES="$(clickhouse-client \
--host "${SENTRY_CH_HOST}" \
--port "${SENTRY_CH_PORT}" \
--user "${SENTRY_CH_USER}" \
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
--query="SHOW TABLES FROM ${db}")"
mkdir -p "${CH_TMP}/${db}"
for tbl in ${TABLES}; do
clickhouse-client \
--host "${SENTRY_CH_HOST}" \
--port "${SENTRY_CH_PORT}" \
--user "${SENTRY_CH_USER}" \
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
--query="SELECT * FROM ${db}.${tbl} FORMAT Native" \
> "${CH_TMP}/${db}/${tbl}.native" 2>/dev/null || true
done
done
tar -czf "${CH_DUMP_FILE}" -C "${WORK}" clickhouse
log "clickhouse archive size: $(du -h "${CH_DUMP_FILE}" | cut -f1)"
# -----------------------------------------------------------------------------
# Step 3: encrypt
# -----------------------------------------------------------------------------
encrypt() {
local src="$1"
local dst="${src}.gpg"
if [[ -n "${SENTRY_BACKUP_GPG_RECIPIENT:-}" ]]; then
log "encrypting ${src##*/} to GPG recipient ${SENTRY_BACKUP_GPG_RECIPIENT}"
gpg --batch --yes --trust-model always --recipient "${SENTRY_BACKUP_GPG_RECIPIENT}" \
--output "${dst}" --encrypt "${src}"
else
log "encrypting ${src##*/} symmetrically (AES256) with SENTRY_SECRET_KEY"
gpg --batch --yes --symmetric --cipher-algo AES256 \
--passphrase "${SENTRY_SECRET_KEY:?SENTRY_SECRET_KEY missing for symmetric fallback}" \
--output "${dst}" "${src}"
fi
rm -f "${src}"
printf "%s" "${dst}"
}
PG_ENCRYPTED="$(encrypt "${PG_DUMP_FILE}")"
CH_ENCRYPTED="$(encrypt "${CH_DUMP_FILE}")"
# -----------------------------------------------------------------------------
# Step 4: upload to S3
# -----------------------------------------------------------------------------
export AWS_ACCESS_KEY_ID="${SENTRY_S3_BACKUP_ACCESS_KEY}"
export AWS_SECRET_ACCESS_KEY="${SENTRY_S3_BACKUP_SECRET_KEY}"
export AWS_EC2_METADATA_DISABLED=true
S3_PREFIX="s3://${SENTRY_S3_BACKUP_BUCKET}/sentry/${TODAY}"
log "uploading to ${S3_PREFIX}/"
aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${PG_ENCRYPTED}" "${S3_PREFIX}/$(basename "${PG_ENCRYPTED}")"
aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${CH_ENCRYPTED}" "${S3_PREFIX}/$(basename "${CH_ENCRYPTED}")"
log "backup complete for ${TODAY}"
@@ -0,0 +1,13 @@
# infra/sentry/backup/crontab
#
# Daily Sentry backup. Times are UTC (TZ set in the Dockerfile).
# Format: m h dom mon dow command
#
# 03:30 UTC = quiet hour for both EU/US active windows. After Sentry's own
# nightly cleanup job (default 01:00) and before any morning maintenance.
30 3 * * * /srv/backup/backup.sh >> /srv/backup/logs/backup-$(date -u +\%Y\%m\%d).log 2>&1
# Heartbeat — write a tiny "alive" marker hourly so operators can confirm
# crond is actually running inside the container. Keeps the last 24h of files.
0 * * * * date -u +%Y-%m-%dT%H:%M:%SZ > /srv/backup/logs/heartbeat.txt