44713e4ca0
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018BAUeCFpDkRD6tU5wGsc1C
193 lines
8.0 KiB
Bash
Executable File
193 lines
8.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# Restauration PostgreSQL - production Xpeditis
|
|
# =============================================================================
|
|
# Trois modes, du moins au plus destructeur :
|
|
#
|
|
# verify <fichier.dump.age> Restaure dans une base jetable et controle
|
|
# la coherence. AUCUN impact sur la prod.
|
|
# C'est ce que lance le timer hebdomadaire.
|
|
#
|
|
# logical <fichier.dump.age> Restaure un dump logique dans une base
|
|
# nommee (par defaut une base de secours).
|
|
#
|
|
# pitr <YYYY-MM-DD HH:MM:SS> Restauration a un instant T via WAL-G.
|
|
# DETRUIT le repertoire de donnees courant.
|
|
# Reserve aux incidents majeurs.
|
|
#
|
|
# Lire docs/mise-en-prod/12-sauvegardes-restauration.md AVANT d'utiliser `pitr`.
|
|
set -euo pipefail
|
|
|
|
COMPOSE_DIR="${COMPOSE_DIR:-/opt/xpeditis/data-node}"
|
|
ENV_FILE="${ENV_FILE:-${COMPOSE_DIR}/.env.data}"
|
|
PGDATA_HOST="${PGDATA_HOST:-/var/lib/xpeditis/pgdata}"
|
|
|
|
# shellcheck disable=SC1090
|
|
set -a; source "$ENV_FILE"; set +a
|
|
|
|
log() { printf '\n[%s] %s\n' "$(date -Is)" "$*"; }
|
|
fail() { printf '\nECHEC: %s\n' "$*" >&2; exit 1; }
|
|
dc() { docker compose -f "${COMPOSE_DIR}/docker-compose.data.yml" --env-file "$ENV_FILE" "$@"; }
|
|
|
|
confirm() {
|
|
local prompt="$1" expected="$2" answer
|
|
printf '\n%s\nTapez exactement "%s" pour confirmer : ' "$prompt" "$expected"
|
|
read -r answer
|
|
[[ "$answer" == "$expected" ]] || fail "confirmation refusee"
|
|
}
|
|
|
|
decrypt_dump() {
|
|
local encrypted="$1" out="$2"
|
|
[[ -f "$encrypted" ]] || fail "fichier introuvable : $encrypted"
|
|
[[ -f "${BACKUP_AGE_KEY_FILE:-/root/.config/xpeditis/backup-age.key}" ]] \
|
|
|| fail "cle de dechiffrement absente (BACKUP_AGE_KEY_FILE)"
|
|
age -d -i "${BACKUP_AGE_KEY_FILE:-/root/.config/xpeditis/backup-age.key}" \
|
|
-o "$out" "$encrypted" || fail "dechiffrement age echoue"
|
|
}
|
|
|
|
MODE="${1:-}"
|
|
|
|
case "$MODE" in
|
|
|
|
# ---------------------------------------------------------------------------
|
|
verify)
|
|
ENCRYPTED="${2:-$(ls -1t /var/lib/xpeditis/dumps/xpeditis-*.dump.age 2>/dev/null | head -1)}"
|
|
[[ -n "$ENCRYPTED" ]] || fail "aucun dump trouve"
|
|
SCRATCH_DB="xpeditis_restore_check"
|
|
TMP_DUMP="/tmp/xpeditis-verify-$$.dump"
|
|
|
|
log "Verification de : $ENCRYPTED"
|
|
decrypt_dump "$ENCRYPTED" "$TMP_DUMP"
|
|
trap 'rm -f "$TMP_DUMP"' EXIT
|
|
|
|
log "Creation de la base jetable ${SCRATCH_DB}"
|
|
dc exec -T -u postgres postgres psql -q -c "DROP DATABASE IF EXISTS ${SCRATCH_DB};"
|
|
dc exec -T -u postgres postgres psql -q -c "CREATE DATABASE ${SCRATCH_DB};"
|
|
|
|
log "pg_restore dans la base jetable"
|
|
dc exec -T -u postgres postgres pg_restore \
|
|
--no-owner --no-privileges --exit-on-error -d "$SCRATCH_DB" < "$TMP_DUMP" \
|
|
|| fail "pg_restore a echoue : LA SAUVEGARDE EST INEXPLOITABLE"
|
|
|
|
log "Controles de coherence"
|
|
# Une sauvegarde qui se restaure mais ne contient rien ne vaut rien : on
|
|
# verifie que les tables metier critiques sont peuplees.
|
|
dc exec -T -u postgres postgres psql -d "$SCRATCH_DB" -v ON_ERROR_STOP=1 <<'SQL'
|
|
\pset pager off
|
|
SELECT 'tables' AS objet, count(*) AS n FROM information_schema.tables WHERE table_schema='public';
|
|
SELECT 'users' AS objet, count(*) AS n FROM users;
|
|
SELECT 'organizations' AS objet, count(*) AS n FROM organizations;
|
|
SELECT 'csv_bookings' AS objet, count(*) AS n FROM csv_bookings;
|
|
SELECT 'migrations' AS objet, count(*) AS n FROM migrations;
|
|
DO $$
|
|
DECLARE t int;
|
|
BEGIN
|
|
SELECT count(*) INTO t FROM information_schema.tables WHERE table_schema='public';
|
|
IF t < 10 THEN
|
|
RAISE EXCEPTION 'Sauvegarde suspecte : seulement % tables restaurees', t;
|
|
END IF;
|
|
END $$;
|
|
SQL
|
|
|
|
log "Nettoyage"
|
|
dc exec -T -u postgres postgres psql -q -c "DROP DATABASE ${SCRATCH_DB};"
|
|
log "SAUVEGARDE VALIDE : $ENCRYPTED"
|
|
;;
|
|
|
|
# ---------------------------------------------------------------------------
|
|
logical)
|
|
ENCRYPTED="${2:?usage: $0 logical <fichier.dump.age> [base_cible]}"
|
|
TARGET_DB="${3:-${POSTGRES_DB}_restored}"
|
|
TMP_DUMP="/tmp/xpeditis-restore-$$.dump"
|
|
|
|
confirm "Restauration logique de $ENCRYPTED dans la base '${TARGET_DB}'. La base cible sera SUPPRIMEE si elle existe." "RESTAURER ${TARGET_DB}"
|
|
|
|
decrypt_dump "$ENCRYPTED" "$TMP_DUMP"
|
|
trap 'rm -f "$TMP_DUMP"' EXIT
|
|
|
|
dc exec -T -u postgres postgres psql -q -c "DROP DATABASE IF EXISTS ${TARGET_DB};"
|
|
dc exec -T -u postgres postgres psql -q -c "CREATE DATABASE ${TARGET_DB};"
|
|
dc exec -T -u postgres postgres pg_restore \
|
|
--no-owner --no-privileges --exit-on-error -j 2 -d "$TARGET_DB" < "$TMP_DUMP" \
|
|
|| fail "pg_restore a echoue"
|
|
|
|
log "Restauration terminee dans '${TARGET_DB}'."
|
|
log "La base de production n'a PAS ete touchee. Pour basculer :"
|
|
log " 1. arreter le backend : kubectl -n xpeditis-prod scale deploy/xpeditis-backend --replicas=0"
|
|
log " 2. renommer les bases : ALTER DATABASE ${POSTGRES_DB} RENAME TO ${POSTGRES_DB}_old;"
|
|
log " ALTER DATABASE ${TARGET_DB} RENAME TO ${POSTGRES_DB};"
|
|
log " 3. relancer le backend : kubectl -n xpeditis-prod scale deploy/xpeditis-backend --replicas=2"
|
|
;;
|
|
|
|
# ---------------------------------------------------------------------------
|
|
pitr)
|
|
TARGET_TIME="${2:?usage: $0 pitr \"YYYY-MM-DD HH:MM:SS\"}"
|
|
|
|
cat <<BANNER
|
|
|
|
############################################################
|
|
# RESTAURATION A UN INSTANT T (PITR) #
|
|
# #
|
|
# Cette operation DETRUIT le repertoire de donnees actuel #
|
|
# et rejoue les WAL jusqu'a : #
|
|
# ${TARGET_TIME}
|
|
# #
|
|
# Prerequis : #
|
|
# - backend arrete (replicas=0) : sinon ecritures perdues #
|
|
# - un dump logique frais pris AVANT (voir plus bas) #
|
|
############################################################
|
|
|
|
BANNER
|
|
confirm "Confirmez-vous la destruction de ${PGDATA_HOST} ?" "DETRUIRE ET RESTAURER"
|
|
|
|
log "Sauvegarde de securite du repertoire courant (au cas ou)"
|
|
SAFETY="/var/lib/xpeditis/pgdata-avant-pitr-$(date -u +%Y%m%dT%H%M%SZ)"
|
|
dc stop postgres
|
|
mv "${PGDATA_HOST}/pgdata" "$SAFETY" || fail "impossible de deplacer le repertoire courant"
|
|
mkdir -p "${PGDATA_HOST}/pgdata"
|
|
chown 999:999 "${PGDATA_HOST}/pgdata"
|
|
chmod 700 "${PGDATA_HOST}/pgdata"
|
|
log "Ancien repertoire conserve dans ${SAFETY} (a supprimer une fois la restauration validee)"
|
|
|
|
log "wal-g backup-fetch (derniere sauvegarde complete)"
|
|
dc run --rm --no-deps -u postgres postgres \
|
|
wal-g backup-fetch /var/lib/postgresql/data/pgdata LATEST \
|
|
|| fail "backup-fetch a echoue"
|
|
|
|
log "Configuration du rejeu des WAL jusqu'a ${TARGET_TIME}"
|
|
dc run --rm --no-deps -u postgres postgres bash -c "
|
|
set -e
|
|
D=/var/lib/postgresql/data/pgdata
|
|
touch \$D/recovery.signal
|
|
cat >> \$D/postgresql.auto.conf <<CONF
|
|
restore_command = 'wal-g wal-fetch \"%f\" \"%p\"'
|
|
recovery_target_time = '${TARGET_TIME}'
|
|
recovery_target_action = 'promote'
|
|
recovery_target_timeline = 'latest'
|
|
CONF
|
|
" || fail "configuration de la restauration echouee"
|
|
|
|
log "Redemarrage de PostgreSQL : le rejeu des WAL commence"
|
|
dc up -d postgres
|
|
|
|
log "Suivez la progression : docker compose logs -f postgres"
|
|
log "La base sort du mode restauration quand 'database system is ready to accept connections' apparait."
|
|
log "Verifiez ENSUITE les donnees avant de relancer le backend."
|
|
;;
|
|
|
|
# ---------------------------------------------------------------------------
|
|
*)
|
|
cat <<USAGE
|
|
Usage:
|
|
$0 verify [fichier.dump.age] Test de restauration sans impact
|
|
$0 logical <fichier.dump.age> [base] Restauration logique dans une base a part
|
|
$0 pitr "YYYY-MM-DD HH:MM:SS" Restauration a un instant T (DESTRUCTIF)
|
|
|
|
Lister les sauvegardes disponibles :
|
|
ls -lht /var/lib/xpeditis/dumps/
|
|
docker compose -f ${COMPOSE_DIR}/docker-compose.data.yml exec -u postgres postgres wal-g backup-list --detail
|
|
USAGE
|
|
exit 1
|
|
;;
|
|
esac
|