diff --git a/mariadb-backup/mariadb-backup-full.sh b/mariadb-backup/mariadb-backup-full.sh index 25b9326..fca5ff7 100644 --- a/mariadb-backup/mariadb-backup-full.sh +++ b/mariadb-backup/mariadb-backup-full.sh @@ -1,65 +1,100 @@ #!/bin/bash +# Backup FULL con atomicità, cleanup, log completi e sicurezza operativa +# Sicuro per cluster Galera +# Autore: Marco + Copilot + set -euo pipefail +# Aumenta i file descriptor disponibili ulimit -n 65536 -BACKUP_BASE=/var/lib/mariadb-backup +# Directory base dei backup +BACKUP_BASE=/var/backups/tscale01 + +# Timestamp per la directory del backup TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +# Directory finale del backup FULL TARGET="$BACKUP_BASE/backup-full-$TIMESTAMP" + +# File di log generale LOGFILE=/var/log/mariadb-backup.log + +# Lockfile per evitare esecuzioni concorrenti LOCKFILE=/var/lock/mariadb-backup.lock -# assicurati che il logfile esista e abbia permessi restrittivi +# Nome del server per i log +SERVER_NAME=$(hostname -s) + +# Assicura che il logfile esista e abbia permessi sicuri touch "$LOGFILE" chown root:root "$LOGFILE" chmod 600 "$LOGFILE" ( - flock -n 9 || { echo "[$(date '+%F %T')] SKIP: another backup is running" >> "$LOGFILE"; exit 0; } + # FLOCK: evita che due backup partano insieme + flock -n 9 || { + echo "[$(date '+%F %T')] [$SERVER_NAME] SKIP: another backup is running" >> "$LOGFILE" + exit 0 + } + # Crea directory del backup mkdir -p "$TARGET" chown mysql:mysql "$TARGET" chmod 750 "$TARGET" - TMPLOG=$(mktemp /tmp/mariadb-backup.XXXXXX) + # File temporaneo per catturare errori del backup + TMPLOG=$(mktemp /tmp/mariadb-backup-full.XXXXXX) echo "---------------------" >> "$LOGFILE" - echo "[$(date '+%F %T')] START backup full $TARGET" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] START backup FULL $TARGET" >> "$LOGFILE" - if mariadb-backup --defaults-file=/etc/mysql/backup.conf --backup --target-dir="$TARGET" --verbose >"$TMPLOG" 2>&1; then - echo "[$(date '+%F %T')] NOTE: full created UNPREPARED at $TARGET" >> "$LOGFILE" - echo "[$(date '+%F %T')] END backup full $TARGET" >> "$LOGFILE" + # Esecuzione backup FULL + if mariadb-backup \ + --defaults-file=/etc/mysql/backup.conf \ + --backup \ + --target-dir="$TARGET" \ + --verbose >"$TMPLOG" 2>&1; then - # misura dimensione apparente (byte logici) + echo "[$(date '+%F %T')] [$SERVER_NAME] NOTE: FULL created UNPREPARED at $TARGET" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] END backup FULL $TARGET" >> "$LOGFILE" + + # Misura dimensione apparente (byte logici) size_apparent_bytes=$(find "$TARGET" -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1} END{print s+0}') size_apparent_human=$(numfmt --to=iec --suffix=B "$size_apparent_bytes" 2>/dev/null || echo "${size_apparent_bytes}B") - # misura spazio su disco effettivo (byte allocati) + # Misura spazio su disco effettivo (byte allocati) size_disk_bytes=$(du -s --block-size=1 "$TARGET" 2>/dev/null | cut -f1 || echo 0) size_disk_human=$(numfmt --to=iec --suffix=B "$size_disk_bytes" 2>/dev/null || echo "${size_disk_bytes}B") - echo "[$(date '+%F %T')] SIZE apparent: $size_apparent_human ($size_apparent_bytes bytes) for $TARGET" >> "$LOGFILE" - echo "[$(date '+%F %T')] SIZE on-disk: $size_disk_human ($size_disk_bytes bytes) for $TARGET" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE apparent: $size_apparent_human ($size_apparent_bytes bytes)" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE on-disk: $size_disk_human ($size_disk_bytes bytes)" >> "$LOGFILE" - echo "[$(date '+%F %T')] RESULT: OK, no errors" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] RESULT: OK, no errors" >> "$LOGFILE" rm -f "$TMPLOG" + else - echo "[$(date '+%F %T')] ERROR during backup for $TARGET. See $TMPLOG" >> "$LOGFILE" + # Backup fallito → log + cleanup directory + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR during FULL backup for $TARGET. See $TMPLOG" >> "$LOGFILE" tail -n 200 "$TMPLOG" >> "$LOGFILE" + + echo "[$(date '+%F %T')] [$SERVER_NAME] CLEANUP: removing incomplete FULL backup directory $TARGET" >> "$LOGFILE" + rm -rf "$TARGET" + rm -f "$TMPLOG" exit 1 fi - # rotazione: mantieni 7 giorni solo per i full (log dettagliato come per gli incrementali) + # Rotazione: mantieni 7 giorni di FULL TO_DELETE=$(find "$BACKUP_BASE" -maxdepth 1 -type d -name 'backup-full-*' -mtime +7 -print 2>/dev/null || true) if [ -n "$TO_DELETE" ]; then - echo "[$(date '+%F %T')] ROTATE: removing old fulls:" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: removing old FULL backups:" >> "$LOGFILE" echo "$TO_DELETE" >> "$LOGFILE" echo "$TO_DELETE" | tr '\n' '\0' | xargs -0 -r rm -rf -- else - echo "[$(date '+%F %T')] ROTATE: no fulls to remove (<=7 days)" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: no FULL backups to remove (<=7 days)" >> "$LOGFILE" fi echo " " >> "$LOGFILE" diff --git a/mariadb-backup/mariadb-backup-incremental.sh b/mariadb-backup/mariadb-backup-incremental.sh index 617736e..e4bd59a 100644 --- a/mariadb-backup/mariadb-backup-incremental.sh +++ b/mariadb-backup/mariadb-backup-incremental.sh @@ -1,77 +1,117 @@ #!/bin/bash +# Backup INCREMENTAL con atomicità, cleanup, log completi e sicurezza operativa +# Sicuro per cluster Galera +# Autore: Marco + Copilot + set -euo pipefail +# Aumenta i file descriptor disponibili ulimit -n 65536 -BACKUP_BASE=/var/lib/mariadb-backup +# Directory base dei backup +BACKUP_BASE=/var/backups/tscale01 + +# Prefissi per full e incremental FULL_PREFIX=backup-full- INC_PREFIX=backup-inc- + +# Timestamp per la directory del backup TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +# Directory finale del backup INCREMENTAL TARGET="$BACKUP_BASE/${INC_PREFIX}${TIMESTAMP}" + +# File di log generale LOGFILE=/var/log/mariadb-backup.log + +# Lockfile per evitare esecuzioni concorrenti LOCKFILE=/var/lock/mariadb-backup.lock +# Nome del server per i log +SERVER_NAME=$(hostname -s) + +# Assicura che il logfile esista e abbia permessi sicuri touch "$LOGFILE" chown root:root "$LOGFILE" chmod 600 "$LOGFILE" ( - flock -n 9 || { echo "[$(date '+%F %T')] SKIP: another backup is running" >> "$LOGFILE"; exit 0; } + # FLOCK: evita che due backup partano insieme + flock -n 9 || { + echo "[$(date '+%F %T')] [$SERVER_NAME] SKIP: another backup is running" >> "$LOGFILE" + exit 0 + } + # Trova l'ultimo FULL disponibile LATEST_FULL=$(ls -1d ${BACKUP_BASE}/${FULL_PREFIX}* 2>/dev/null | sort | tail -n1 || true) if [ -z "$LATEST_FULL" ]; then - echo "[$(date '+%F %T')] ERROR: no full backup found in $BACKUP_BASE. Create a full backup first." >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR: no FULL backup found. Create a FULL backup first." >> "$LOGFILE" exit 1 fi BASE_DIR="$LATEST_FULL" + # Crea directory del backup mkdir -p "$TARGET" chown mysql:mysql "$TARGET" chmod 750 "$TARGET" + # File temporaneo per catturare errori del backup TMPLOG=$(mktemp /tmp/mariadb-backup-inc.XXXXXX) echo "---------------------" >> "$LOGFILE" - echo "[$(date '+%F %T')] START incremental backup $TARGET (base: $BASE_DIR)" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] START incremental backup $TARGET (base: $BASE_DIR)" >> "$LOGFILE" - if mariadb-backup --defaults-file=/etc/mysql/backup.conf --backup --target-dir="$TARGET" --incremental-basedir="$BASE_DIR" --verbose >"$TMPLOG" 2>&1; then - echo "[$(date '+%F %T')] END incremental backup $TARGET" >> "$LOGFILE" + # Esecuzione backup INCREMENTAL + if mariadb-backup \ + --defaults-file=/etc/mysql/backup.conf \ + --backup \ + --target-dir="$TARGET" \ + --incremental-basedir="$BASE_DIR" \ + --verbose >"$TMPLOG" 2>&1; then - # misura dimensione apparente (byte logici) + echo "[$(date '+%F %T')] [$SERVER_NAME] END incremental backup $TARGET" >> "$LOGFILE" + + # Misura dimensione apparente (byte logici) size_apparent_bytes=$(find "$TARGET" -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1} END{print s+0}') size_apparent_human=$(numfmt --to=iec --suffix=B "$size_apparent_bytes" 2>/dev/null || echo "${size_apparent_bytes}B") - # misura spazio su disco effettivo (byte allocati) + # Misura spazio su disco effettivo (byte allocati) size_disk_bytes=$(du -s --block-size=1 "$TARGET" 2>/dev/null | cut -f1 || echo 0) size_disk_human=$(numfmt --to=iec --suffix=B "$size_disk_bytes" 2>/dev/null || echo "${size_disk_bytes}B") - echo "[$(date '+%F %T')] SIZE apparent: $size_apparent_human ($size_apparent_bytes bytes) for $TARGET" >> "$LOGFILE" - echo "[$(date '+%F %T')] SIZE on-disk: $size_disk_human ($size_disk_bytes bytes) for $TARGET" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE apparent: $size_apparent_human ($size_apparent_bytes bytes)" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE on-disk: $size_disk_human ($size_disk_bytes bytes)" >> "$LOGFILE" - echo "[$(date '+%F %T')] RESULT: OK, no errors" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] RESULT: OK, no errors" >> "$LOGFILE" rm -f "$TMPLOG" + else - echo "[$(date '+%F %T')] ERROR during incremental backup for $TARGET. See $TMPLOG" >> "$LOGFILE" + # Backup fallito → log + cleanup directory + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR during incremental backup for $TARGET. See $TMPLOG" >> "$LOGFILE" tail -n 200 "$TMPLOG" >> "$LOGFILE" + + echo "[$(date '+%F %T')] [$SERVER_NAME] CLEANUP: removing incomplete incremental backup directory $TARGET" >> "$LOGFILE" + rm -rf "$TARGET" + rm -f "$TMPLOG" exit 1 fi - # rotazione: mantieni esattamente 11 incrementali più recenti + # Rotazione: mantieni esattamente 11 incrementali più recenti cd "$BACKUP_BASE" || exit 1 TO_DELETE=$(ls -1d ${INC_PREFIX}* 2>/dev/null | sort | head -n -11 || true) if [ -n "$TO_DELETE" ]; then - echo "[$(date '+%F %T')] ROTATE: removing old incrementals:" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: removing old incrementals:" >> "$LOGFILE" echo "$TO_DELETE" >> "$LOGFILE" echo "$TO_DELETE" | xargs -r -d '\n' rm -rf -- else - echo "[$(date '+%F %T')] ROTATE: no incrementals to remove (<=11 present)" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: no incrementals to remove (<=11 present)" >> "$LOGFILE" fi echo " " >> "$LOGFILE" -) 9>"$LOCKFILE" \ No newline at end of file +) 9>"$LOCKFILE" diff --git a/mariadb-backup/mariadb-backup-logic.sh b/mariadb-backup/mariadb-backup-logic-old.sh similarity index 97% rename from mariadb-backup/mariadb-backup-logic.sh rename to mariadb-backup/mariadb-backup-logic-old.sh index 35e301e..6a2d06a 100644 --- a/mariadb-backup/mariadb-backup-logic.sh +++ b/mariadb-backup/mariadb-backup-logic-old.sh @@ -3,12 +3,11 @@ set -euo pipefail ulimit -n 65536 -BACKUP_BASE=/var/lib/mariadb-backup +BACKUP_BASE=/var/backups/tscale01 PREFIX=backup-logic- TIMESTAMP=$(date +%Y%m%d-%H%M%S) TARGET="$BACKUP_BASE/${PREFIX}${TIMESTAMP}" DUMPFILE="$TARGET/all-dbs.sql" - LOGFILE=/var/log/mariadb-backup.log LOCKFILE=/var/lock/mariadb-backup.lock @@ -21,6 +20,7 @@ touch "$LOGFILE" chown root:root "$LOGFILE" chmod 600 "$LOGFILE" +#verifico file lock per non sovrapporre backup ( flock -n 9 || { echo "[$(date '+%F %T')] SKIP: another backup is running" >> "$LOGFILE"; exit 0; } diff --git a/mariadb-backup/mariadb-backup-logic-per-database.sh b/mariadb-backup/mariadb-backup-logic-per-database.sh new file mode 100644 index 0000000..660e014 --- /dev/null +++ b/mariadb-backup/mariadb-backup-logic-per-database.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# Backup logico per-database con retry e cleanup automatico +# Sicuro per cluster Galera ad alto carico +# Autore: Marco + Copilot + +set -euo pipefail + +# Aumenta i file descriptor disponibili (mysqldump può aprirne molti) +ulimit -n 65536 + +# Directory base dei backup +BACKUP_BASE=/var/backups/tscale01 + +# Prefisso per distinguere i backup logici di questo nodo +PREFIX=eqn-bck-logic- + +# Timestamp per la directory del backup +TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +# Directory finale del backup +TARGET="$BACKUP_BASE/${PREFIX}${TIMESTAMP}" + +# File di log generale +LOGFILE=/var/log/mariadb-backup.log + +# Lockfile per evitare esecuzioni concorrenti +LOCKFILE=/var/lock/mariadb-backup.lock + +# Nome del server per i log (utile in ambienti multi-nodo) +SERVER_NAME=$(hostname -s) + +# Binari necessari +MYSQL_BIN=$(command -v mariadb || true) +MYSQLDUMP_BIN=$(command -v mariadb-dump || true) + +# Opzioni dump ottimizzate per Galera (no lock, snapshot consistente) +MYSQLDUMP_OPTS="--user=root --single-transaction --quick --skip-lock-tables" + +# Numero massimo di retry per ogni database (deadlock = retry) +MAX_RETRIES=10 + +# Secondi di attesa tra un retry e l'altro +RETRY_SLEEP=30 + +# Assicura che il logfile esista e abbia permessi sicuri +touch "$LOGFILE" +chown root:root "$LOGFILE" +chmod 600 "$LOGFILE" + +( + # FLOCK: evita che due backup partano insieme + flock -n 9 || { echo "[$(date '+%F %T')] [$SERVER_NAME] SKIP: another backup is running" >> "$LOGFILE"; exit 0; } + + # Controllo binari + if [ -z "$MYSQL_BIN" ]; then + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR: mariadb client not found in PATH" >> "$LOGFILE" + exit 1 + fi + + if [ -z "$MYSQLDUMP_BIN" ]; then + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR: mariadb-dump not found in PATH" >> "$LOGFILE" + exit 1 + fi + + # Crea directory del backup + mkdir -p "$TARGET" + chown mysql:mysql "$TARGET" + chmod 750 "$TARGET" + + # File temporaneo per catturare errori del dump + TMPLOG=$(mktemp /tmp/mariadb-backup-logic.XXXXXX) + + echo "---------------------" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] START logic backup (per-database) $TARGET" >> "$LOGFILE" + + # Elenco database utente (escludiamo schemi di sistema) + DB_LIST=$($MYSQL_BIN -N -e "SHOW DATABASES" | grep -vE '^(information_schema|performance_schema|mysql|sys)$' || true) + + if [ -z "$DB_LIST" ]; then + echo "[$(date '+%F %T')] [$SERVER_NAME] WARNING: no user databases found to dump" >> "$LOGFILE" + fi + + FAILED=0 + FAILED_DBS="" + + # Loop su ogni database + for db in $DB_LIST; do + DUMPFILE="$TARGET/${db}.sql" + echo "[$(date '+%F %T')] [$SERVER_NAME] START dump database '$db' -> $DUMPFILE" >> "$LOGFILE" + + attempt=1 + success=0 + + # Retry intelligente in caso di deadlock (errore 1213) + while [ "$attempt" -le "$MAX_RETRIES" ]; do + : >"$TMPLOG" # pulisce il file temporaneo + + if $MYSQLDUMP_BIN $MYSQLDUMP_OPTS "$db" >"$DUMPFILE" 2>"$TMPLOG"; then + echo "[$(date '+%F %T')] [$SERVER_NAME] END dump database '$db' (attempt $attempt)" >> "$LOGFILE" + success=1 + break + else + echo "[$(date '+%F %T')] [$SERVER_NAME] ERROR dumping database '$db' (attempt $attempt). See $TMPLOG" >> "$LOGFILE" + tail -n 50 "$TMPLOG" >> "$LOGFILE" + + # Se è un deadlock, riproviamo + if grep -q "Error 1213" "$TMPLOG"; then + echo "[$(date '+%F %T')] [$SERVER_NAME] RETRY for database '$db' in ${RETRY_SLEEP}s due to deadlock (1213)" >> "$LOGFILE" + sleep "$RETRY_SLEEP" + else + # Errori non recuperabili → stop retry + echo "[$(date '+%F %T')] [$SERVER_NAME] NON-RETRYABLE error for database '$db'" >> "$LOGFILE" + break + fi + fi + + attempt=$((attempt + 1)) + done + + # Se dopo i retry non è andata → segna come fallito + if [ "$success" -ne 1 ]; then + FAILED=1 + FAILED_DBS="$FAILED_DBS $db" + fi + done + + # Calcolo dimensioni del backup + size_apparent_bytes=$(find "$TARGET" -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1} END{print s+0}') + size_apparent_human=$(numfmt --to=iec --suffix=B "$size_apparent_bytes" 2>/dev/null || echo "${size_apparent_bytes}B") + + size_disk_bytes=$(du -s --block-size=1 "$TARGET" 2>/dev/null | cut -f1 || echo 0) + size_disk_human=$(numfmt --to=iec --suffix=B "$size_disk_bytes" 2>/dev/null || echo "${size_disk_bytes}B") + + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE apparent: $size_apparent_human ($size_apparent_bytes bytes) for $TARGET" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] SIZE on-disk: $size_disk_human ($size_disk_bytes bytes) for $TARGET" >> "$LOGFILE" + + # Risultato finale con cleanup in caso di fallimento + if [ "$FAILED" -eq 0 ]; then + echo "[$(date '+%F %T')] [$SERVER_NAME] RESULT: OK, all databases dumped successfully" >> "$LOGFILE" + else + echo "[$(date '+%F %T')] [$SERVER_NAME] RESULT: FAILED, databases with errors:$FAILED_DBS" >> "$LOGFILE" + echo "[$(date '+%F %T')] [$SERVER_NAME] CLEANUP: removing incomplete backup directory $TARGET" >> "$LOGFILE" + rm -rf "$TARGET" + rm -f "$TMPLOG" + exit 1 + fi + + rm -f "$TMPLOG" + + # Rotazione: elimina backup più vecchi di 7 giorni + TO_DELETE=$(find "$BACKUP_BASE" -maxdepth 1 -type d -name "${PREFIX}*" -mtime +7 -print 2>/dev/null || true) + + if [ -n "$TO_DELETE" ]; then + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: removing old logic backups:" >> "$LOGFILE" + echo "$TO_DELETE" >> "$LOGFILE" + echo "$TO_DELETE" | tr '\n' '\0' | xargs -0 -r rm -rf -- + else + echo "[$(date '+%F %T')] [$SERVER_NAME] ROTATE: no logic backups to remove (<=7 days)" >> "$LOGFILE" + fi + + echo " " >> "$LOGFILE" + +) 9>"$LOCKFILE"