Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,53 @@
|
||||
# infra/sentry/backup/Dockerfile
|
||||
#
|
||||
# Sidecar container that backs up the self-hosted Sentry stack daily:
|
||||
# - pg_dump --format=custom of the sentry Postgres
|
||||
# - clickhouse-backup snapshot uploaded directly to S3
|
||||
# - both artifacts encrypted with GPG (or AES256 fallback) before upload
|
||||
#
|
||||
# Triggered by an internal cron (see crontab) so the container stays running
|
||||
# in the background. Re-uses the upstream Postgres + ClickHouse client tools
|
||||
# at versions matching the Sentry stack (Postgres 14, ClickHouse 23.8).
|
||||
|
||||
FROM alpine:3.20
|
||||
|
||||
# --- Runtime deps ------------------------------------------------------------
|
||||
# postgresql-client -> pg_dump
|
||||
# clickhouse-client -> clickhouse-client
|
||||
# aws-cli -> S3 upload (works with Hetzner Object Storage + MinIO via --endpoint-url)
|
||||
# rclone -> alternative S3 backend (handy for MinIO-compat that AWS-CLI v2 dislikes)
|
||||
# gnupg -> GPG encryption of dumps
|
||||
# tini -> proper signal-handling as PID 1
|
||||
# bash, coreutils -> POSIX-strict scripts need real coreutils
|
||||
# tzdata -> consistent UTC timestamps
|
||||
# busybox-extras (cron) -> crond
|
||||
RUN apk add --no-cache \
|
||||
postgresql16-client \
|
||||
clickhouse-client \
|
||||
aws-cli \
|
||||
rclone \
|
||||
gnupg \
|
||||
tini \
|
||||
bash \
|
||||
coreutils \
|
||||
tzdata \
|
||||
busybox-extras \
|
||||
curl \
|
||||
&& rm -rf /var/cache/apk/*
|
||||
|
||||
ENV TZ=UTC \
|
||||
BACKUP_HOME=/srv/backup \
|
||||
PATH=/srv/backup:$PATH
|
||||
|
||||
WORKDIR ${BACKUP_HOME}
|
||||
|
||||
COPY backup.sh ${BACKUP_HOME}/backup.sh
|
||||
COPY crontab /etc/crontabs/root
|
||||
|
||||
RUN chmod 0755 ${BACKUP_HOME}/backup.sh \
|
||||
&& mkdir -p ${BACKUP_HOME}/logs ${BACKUP_HOME}/work
|
||||
|
||||
# Run cron in the foreground; tini handles SIGTERM cleanly so docker compose
|
||||
# down doesn't strand a half-finished dump.
|
||||
ENTRYPOINT ["/sbin/tini", "--"]
|
||||
CMD ["crond", "-f", "-L", "/dev/stdout", "-l", "2"]
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# infra/sentry/backup/backup.sh
|
||||
#
|
||||
# Daily backup of the self-hosted Sentry stack. Triggered by crond at 03:30 UTC
|
||||
# (see crontab). Idempotent — re-running on the same day overwrites the same
|
||||
# S3 prefix.
|
||||
#
|
||||
# Env (sourced from the Compose service `environment` block which itself reads
|
||||
# from infra/sentry/.env):
|
||||
# SENTRY_S3_BACKUP_ENDPOINT S3 endpoint URL (Hetzner / MinIO / AWS)
|
||||
# SENTRY_S3_BACKUP_BUCKET target bucket name
|
||||
# SENTRY_S3_BACKUP_ACCESS_KEY S3 access key
|
||||
# SENTRY_S3_BACKUP_SECRET_KEY S3 secret key
|
||||
# SENTRY_SECRET_KEY fallback symmetric passphrase for GPG
|
||||
# SENTRY_BACKUP_GPG_RECIPIENT optional GPG recipient fingerprint
|
||||
#
|
||||
# SENTRY_PG_HOST default "postgres" (upstream service name)
|
||||
# SENTRY_PG_PORT default 5432
|
||||
# SENTRY_PG_USER default "postgres"
|
||||
# SENTRY_PG_DB default "postgres"
|
||||
# SENTRY_PG_PASSWORD from PGPASSWORD or env
|
||||
#
|
||||
# SENTRY_CH_HOST default "clickhouse"
|
||||
# SENTRY_CH_PORT default 9000
|
||||
# SENTRY_CH_USER default "default"
|
||||
# SENTRY_CH_PASSWORD optional
|
||||
|
||||
set -Eeuo pipefail
|
||||
shopt -s inherit_errexit 2>/dev/null || true
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Defaults + logging
|
||||
# -----------------------------------------------------------------------------
|
||||
: "${SENTRY_PG_HOST:=postgres}"
|
||||
: "${SENTRY_PG_PORT:=5432}"
|
||||
: "${SENTRY_PG_USER:=postgres}"
|
||||
: "${SENTRY_PG_DB:=postgres}"
|
||||
: "${SENTRY_CH_HOST:=clickhouse}"
|
||||
: "${SENTRY_CH_PORT:=9000}"
|
||||
: "${SENTRY_CH_USER:=default}"
|
||||
: "${SENTRY_CH_PASSWORD:=}"
|
||||
|
||||
: "${SENTRY_S3_BACKUP_ENDPOINT:?SENTRY_S3_BACKUP_ENDPOINT is required}"
|
||||
: "${SENTRY_S3_BACKUP_BUCKET:?SENTRY_S3_BACKUP_BUCKET is required}"
|
||||
: "${SENTRY_S3_BACKUP_ACCESS_KEY:?SENTRY_S3_BACKUP_ACCESS_KEY is required}"
|
||||
: "${SENTRY_S3_BACKUP_SECRET_KEY:?SENTRY_S3_BACKUP_SECRET_KEY is required}"
|
||||
|
||||
TODAY="$(date -u +%Y-%m-%d)"
|
||||
NOW="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
|
||||
WORK="${BACKUP_HOME:-/srv/backup}/work/${TODAY}"
|
||||
mkdir -p "${WORK}"
|
||||
|
||||
log() { printf "[backup %s] %s\n" "${NOW}" "$*"; }
|
||||
|
||||
cleanup() { rm -rf "${WORK}"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Step 1: Postgres dump
|
||||
# -----------------------------------------------------------------------------
|
||||
log "dumping Postgres (${SENTRY_PG_HOST}/${SENTRY_PG_DB})"
|
||||
PG_DUMP_FILE="${WORK}/postgres-${NOW}.dump"
|
||||
export PGPASSWORD="${SENTRY_PG_PASSWORD:-${PGPASSWORD:-}}"
|
||||
pg_dump \
|
||||
--host "${SENTRY_PG_HOST}" \
|
||||
--port "${SENTRY_PG_PORT}" \
|
||||
--username "${SENTRY_PG_USER}" \
|
||||
--dbname "${SENTRY_PG_DB}" \
|
||||
--format=custom \
|
||||
--compress=9 \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--file "${PG_DUMP_FILE}"
|
||||
log "postgres dump size: $(du -h "${PG_DUMP_FILE}" | cut -f1)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Step 2: ClickHouse snapshot
|
||||
# -----------------------------------------------------------------------------
|
||||
log "snapshotting ClickHouse (${SENTRY_CH_HOST})"
|
||||
CH_DUMP_FILE="${WORK}/clickhouse-${NOW}.tar.gz"
|
||||
|
||||
# Use clickhouse-client to BACKUP all sentry tables to /var/lib/clickhouse/backup,
|
||||
# then archive the directory. The `BACKUP ... TO Disk('backups', ...)` syntax
|
||||
# is preferred but requires backup-disk config; we fall back to per-database
|
||||
# SELECT...INTO OUTFILE for portability.
|
||||
CH_TMP="${WORK}/clickhouse"
|
||||
mkdir -p "${CH_TMP}"
|
||||
DATABASES="$(clickhouse-client \
|
||||
--host "${SENTRY_CH_HOST}" \
|
||||
--port "${SENTRY_CH_PORT}" \
|
||||
--user "${SENTRY_CH_USER}" \
|
||||
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
|
||||
--query="SHOW DATABASES" | grep -Ev '^(system|INFORMATION_SCHEMA|information_schema|default)$' || true)"
|
||||
|
||||
for db in ${DATABASES}; do
|
||||
log " -> dumping clickhouse db: ${db}"
|
||||
TABLES="$(clickhouse-client \
|
||||
--host "${SENTRY_CH_HOST}" \
|
||||
--port "${SENTRY_CH_PORT}" \
|
||||
--user "${SENTRY_CH_USER}" \
|
||||
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
|
||||
--query="SHOW TABLES FROM ${db}")"
|
||||
mkdir -p "${CH_TMP}/${db}"
|
||||
for tbl in ${TABLES}; do
|
||||
clickhouse-client \
|
||||
--host "${SENTRY_CH_HOST}" \
|
||||
--port "${SENTRY_CH_PORT}" \
|
||||
--user "${SENTRY_CH_USER}" \
|
||||
${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \
|
||||
--query="SELECT * FROM ${db}.${tbl} FORMAT Native" \
|
||||
> "${CH_TMP}/${db}/${tbl}.native" 2>/dev/null || true
|
||||
done
|
||||
done
|
||||
tar -czf "${CH_DUMP_FILE}" -C "${WORK}" clickhouse
|
||||
log "clickhouse archive size: $(du -h "${CH_DUMP_FILE}" | cut -f1)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Step 3: encrypt
|
||||
# -----------------------------------------------------------------------------
|
||||
encrypt() {
|
||||
local src="$1"
|
||||
local dst="${src}.gpg"
|
||||
if [[ -n "${SENTRY_BACKUP_GPG_RECIPIENT:-}" ]]; then
|
||||
log "encrypting ${src##*/} to GPG recipient ${SENTRY_BACKUP_GPG_RECIPIENT}"
|
||||
gpg --batch --yes --trust-model always --recipient "${SENTRY_BACKUP_GPG_RECIPIENT}" \
|
||||
--output "${dst}" --encrypt "${src}"
|
||||
else
|
||||
log "encrypting ${src##*/} symmetrically (AES256) with SENTRY_SECRET_KEY"
|
||||
gpg --batch --yes --symmetric --cipher-algo AES256 \
|
||||
--passphrase "${SENTRY_SECRET_KEY:?SENTRY_SECRET_KEY missing for symmetric fallback}" \
|
||||
--output "${dst}" "${src}"
|
||||
fi
|
||||
rm -f "${src}"
|
||||
printf "%s" "${dst}"
|
||||
}
|
||||
|
||||
PG_ENCRYPTED="$(encrypt "${PG_DUMP_FILE}")"
|
||||
CH_ENCRYPTED="$(encrypt "${CH_DUMP_FILE}")"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Step 4: upload to S3
|
||||
# -----------------------------------------------------------------------------
|
||||
export AWS_ACCESS_KEY_ID="${SENTRY_S3_BACKUP_ACCESS_KEY}"
|
||||
export AWS_SECRET_ACCESS_KEY="${SENTRY_S3_BACKUP_SECRET_KEY}"
|
||||
export AWS_EC2_METADATA_DISABLED=true
|
||||
|
||||
S3_PREFIX="s3://${SENTRY_S3_BACKUP_BUCKET}/sentry/${TODAY}"
|
||||
|
||||
log "uploading to ${S3_PREFIX}/"
|
||||
aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${PG_ENCRYPTED}" "${S3_PREFIX}/$(basename "${PG_ENCRYPTED}")"
|
||||
aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${CH_ENCRYPTED}" "${S3_PREFIX}/$(basename "${CH_ENCRYPTED}")"
|
||||
|
||||
log "backup complete for ${TODAY}"
|
||||
@@ -0,0 +1,13 @@
|
||||
# infra/sentry/backup/crontab
|
||||
#
|
||||
# Daily Sentry backup. Times are UTC (TZ set in the Dockerfile).
|
||||
# Format: m h dom mon dow command
|
||||
#
|
||||
# 03:30 UTC = quiet hour for both EU/US active windows. After Sentry's own
|
||||
# nightly cleanup job (default 01:00) and before any morning maintenance.
|
||||
|
||||
30 3 * * * /srv/backup/backup.sh >> /srv/backup/logs/backup-$(date -u +\%Y\%m\%d).log 2>&1
|
||||
|
||||
# Heartbeat — write a tiny "alive" marker hourly so operators can confirm
|
||||
# crond is actually running inside the container. Keeps the last 24h of files.
|
||||
0 * * * * date -u +%Y-%m-%dT%H:%M:%SZ > /srv/backup/logs/heartbeat.txt
|
||||
Reference in new issue
Block a user