#!/usr/bin/env bash # infra/sentry/backup/backup.sh # # Daily backup of the self-hosted Sentry stack. Triggered by crond at 03:30 UTC # (see crontab). Idempotent — re-running on the same day overwrites the same # S3 prefix. # # Env (sourced from the Compose service `environment` block which itself reads # from infra/sentry/.env): # SENTRY_S3_BACKUP_ENDPOINT S3 endpoint URL (Hetzner / MinIO / AWS) # SENTRY_S3_BACKUP_BUCKET target bucket name # SENTRY_S3_BACKUP_ACCESS_KEY S3 access key # SENTRY_S3_BACKUP_SECRET_KEY S3 secret key # SENTRY_SECRET_KEY fallback symmetric passphrase for GPG # SENTRY_BACKUP_GPG_RECIPIENT optional GPG recipient fingerprint # # SENTRY_PG_HOST default "postgres" (upstream service name) # SENTRY_PG_PORT default 5432 # SENTRY_PG_USER default "postgres" # SENTRY_PG_DB default "postgres" # SENTRY_PG_PASSWORD from PGPASSWORD or env # # SENTRY_CH_HOST default "clickhouse" # SENTRY_CH_PORT default 9000 # SENTRY_CH_USER default "default" # SENTRY_CH_PASSWORD optional set -Eeuo pipefail shopt -s inherit_errexit 2>/dev/null || true # ----------------------------------------------------------------------------- # Defaults + logging # ----------------------------------------------------------------------------- : "${SENTRY_PG_HOST:=postgres}" : "${SENTRY_PG_PORT:=5432}" : "${SENTRY_PG_USER:=postgres}" : "${SENTRY_PG_DB:=postgres}" : "${SENTRY_CH_HOST:=clickhouse}" : "${SENTRY_CH_PORT:=9000}" : "${SENTRY_CH_USER:=default}" : "${SENTRY_CH_PASSWORD:=}" : "${SENTRY_S3_BACKUP_ENDPOINT:?SENTRY_S3_BACKUP_ENDPOINT is required}" : "${SENTRY_S3_BACKUP_BUCKET:?SENTRY_S3_BACKUP_BUCKET is required}" : "${SENTRY_S3_BACKUP_ACCESS_KEY:?SENTRY_S3_BACKUP_ACCESS_KEY is required}" : "${SENTRY_S3_BACKUP_SECRET_KEY:?SENTRY_S3_BACKUP_SECRET_KEY is required}" TODAY="$(date -u +%Y-%m-%d)" NOW="$(date -u +%Y-%m-%dT%H-%M-%SZ)" WORK="${BACKUP_HOME:-/srv/backup}/work/${TODAY}" mkdir -p "${WORK}" log() { printf "[backup %s] %s\n" "${NOW}" "$*"; } cleanup() { rm -rf "${WORK}"; } trap cleanup EXIT # ----------------------------------------------------------------------------- # Step 1: Postgres dump # ----------------------------------------------------------------------------- log "dumping Postgres (${SENTRY_PG_HOST}/${SENTRY_PG_DB})" PG_DUMP_FILE="${WORK}/postgres-${NOW}.dump" export PGPASSWORD="${SENTRY_PG_PASSWORD:-${PGPASSWORD:-}}" pg_dump \ --host "${SENTRY_PG_HOST}" \ --port "${SENTRY_PG_PORT}" \ --username "${SENTRY_PG_USER}" \ --dbname "${SENTRY_PG_DB}" \ --format=custom \ --compress=9 \ --no-owner \ --no-privileges \ --file "${PG_DUMP_FILE}" log "postgres dump size: $(du -h "${PG_DUMP_FILE}" | cut -f1)" # ----------------------------------------------------------------------------- # Step 2: ClickHouse snapshot # ----------------------------------------------------------------------------- log "snapshotting ClickHouse (${SENTRY_CH_HOST})" CH_DUMP_FILE="${WORK}/clickhouse-${NOW}.tar.gz" # Use clickhouse-client to BACKUP all sentry tables to /var/lib/clickhouse/backup, # then archive the directory. The `BACKUP ... TO Disk('backups', ...)` syntax # is preferred but requires backup-disk config; we fall back to per-database # SELECT...INTO OUTFILE for portability. CH_TMP="${WORK}/clickhouse" mkdir -p "${CH_TMP}" DATABASES="$(clickhouse-client \ --host "${SENTRY_CH_HOST}" \ --port "${SENTRY_CH_PORT}" \ --user "${SENTRY_CH_USER}" \ ${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \ --query="SHOW DATABASES" | grep -Ev '^(system|INFORMATION_SCHEMA|information_schema|default)$' || true)" for db in ${DATABASES}; do log " -> dumping clickhouse db: ${db}" TABLES="$(clickhouse-client \ --host "${SENTRY_CH_HOST}" \ --port "${SENTRY_CH_PORT}" \ --user "${SENTRY_CH_USER}" \ ${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \ --query="SHOW TABLES FROM ${db}")" mkdir -p "${CH_TMP}/${db}" for tbl in ${TABLES}; do clickhouse-client \ --host "${SENTRY_CH_HOST}" \ --port "${SENTRY_CH_PORT}" \ --user "${SENTRY_CH_USER}" \ ${SENTRY_CH_PASSWORD:+--password "${SENTRY_CH_PASSWORD}"} \ --query="SELECT * FROM ${db}.${tbl} FORMAT Native" \ > "${CH_TMP}/${db}/${tbl}.native" 2>/dev/null || true done done tar -czf "${CH_DUMP_FILE}" -C "${WORK}" clickhouse log "clickhouse archive size: $(du -h "${CH_DUMP_FILE}" | cut -f1)" # ----------------------------------------------------------------------------- # Step 3: encrypt # ----------------------------------------------------------------------------- encrypt() { local src="$1" local dst="${src}.gpg" if [[ -n "${SENTRY_BACKUP_GPG_RECIPIENT:-}" ]]; then log "encrypting ${src##*/} to GPG recipient ${SENTRY_BACKUP_GPG_RECIPIENT}" gpg --batch --yes --trust-model always --recipient "${SENTRY_BACKUP_GPG_RECIPIENT}" \ --output "${dst}" --encrypt "${src}" else log "encrypting ${src##*/} symmetrically (AES256) with SENTRY_SECRET_KEY" gpg --batch --yes --symmetric --cipher-algo AES256 \ --passphrase "${SENTRY_SECRET_KEY:?SENTRY_SECRET_KEY missing for symmetric fallback}" \ --output "${dst}" "${src}" fi rm -f "${src}" printf "%s" "${dst}" } PG_ENCRYPTED="$(encrypt "${PG_DUMP_FILE}")" CH_ENCRYPTED="$(encrypt "${CH_DUMP_FILE}")" # ----------------------------------------------------------------------------- # Step 4: upload to S3 # ----------------------------------------------------------------------------- export AWS_ACCESS_KEY_ID="${SENTRY_S3_BACKUP_ACCESS_KEY}" export AWS_SECRET_ACCESS_KEY="${SENTRY_S3_BACKUP_SECRET_KEY}" export AWS_EC2_METADATA_DISABLED=true S3_PREFIX="s3://${SENTRY_S3_BACKUP_BUCKET}/sentry/${TODAY}" log "uploading to ${S3_PREFIX}/" aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${PG_ENCRYPTED}" "${S3_PREFIX}/$(basename "${PG_ENCRYPTED}")" aws s3 cp --endpoint-url "${SENTRY_S3_BACKUP_ENDPOINT}" "${CH_ENCRYPTED}" "${S3_PREFIX}/$(basename "${CH_ENCRYPTED}")" log "backup complete for ${TODAY}"