use find instead of mc for pruning local backups

This commit is contained in:
Frederik Ring 2021-08-20 08:56:04 +02:00
parent 0782af88f4
commit 278df9b2f7
4 changed files with 78 additions and 23 deletions

View File

@ -103,36 +103,62 @@ fi
info "Backup finished" info "Backup finished"
echo "Will wait for next scheduled backup." echo "Will wait for next scheduled backup."
probe_expired () {
local target=$1
local is_local=$2
if [ -z "$is_local" ]; then
mc rm $MC_GLOBAL_OPTIONS --fake --recursive --force \
--older-than "${BACKUP_RETENTION_DAYS}d" \
"$target"
else
find $target* -type f -mtime $BACKUP_RETENTION_DAYS
fi
}
probe_all () {
local target=$1
local is_local=$2
if [ -z "$is_local" ]; then
mc ls $MC_GLOBAL_OPTIONS "$target"
else
find $target* -type f
fi
}
delete () {
local target=$1
local is_local=$2
if [ -z "$is_local" ]; then
mc rm $MC_GLOBAL_OPTIONS --recursive --force \
--older-than "${BACKUP_RETENTION_DAYS}d" \
"$target"
else
find $target* -delete -type f -mtime $BACKUP_RETENTION_DAYS
fi
}
prune () { prune () {
target=$1 local target=$1
local is_local=$2
if [ ! -z "$BACKUP_PRUNING_PREFIX" ]; then if [ ! -z "$BACKUP_PRUNING_PREFIX" ]; then
target="$target/${BACKUP_PRUNING_PREFIX}" target="$target/${BACKUP_PRUNING_PREFIX}"
fi fi
rule_applies_to=$( rule_applies_to=$(probe_expired "$target" "$is_local" | wc -l)
mc rm $MC_GLOBAL_OPTIONS --fake --recursive --force \
--older-than "${BACKUP_RETENTION_DAYS}d" \
"$target" \
| wc -l
)
if [ "$rule_applies_to" == "0" ]; then if [ "$rule_applies_to" == "0" ]; then
echo "No backups found older than the configured retention period of $BACKUP_RETENTION_DAYS days." echo "No backups found older than the configured retention period of ${BACKUP_RETENTION_DAYS} days."
echo "Doing nothing." echo "Doing nothing."
exit 0 else
fi total=$(probe_all "$target" "$is_local" | wc -l)
total=$(mc ls $MC_GLOBAL_OPTIONS "$target" | wc -l)
if [ "$rule_applies_to" == "$total" ]; then if [ "$rule_applies_to" == "$total" ]; then
echo "Using a retention of ${BACKUP_RETENTION_DAYS} days would prune all currently existing backups, will not continue." echo "Using a retention of ${BACKUP_RETENTION_DAYS} days would prune all currently existing backups, will not continue."
echo "If this is what you want, please remove files manually instead of using this script." echo "If this is what you want, please remove files manually instead of using this script."
exit 1 else
fi delete "$target" "$is_local"
mc rm $MC_GLOBAL_OPTIONS \
--recursive --force \
--older-than "${BACKUP_RETENTION_DAYS}d" "$target"
echo "Successfully pruned ${rule_applies_to} backups older than ${BACKUP_RETENTION_DAYS} days." echo "Successfully pruned ${rule_applies_to} backups older than ${BACKUP_RETENTION_DAYS} days."
fi
fi
} }
if [ ! -z "$BACKUP_RETENTION_DAYS" ]; then if [ ! -z "$BACKUP_RETENTION_DAYS" ]; then
@ -145,6 +171,6 @@ if [ ! -z "$BACKUP_RETENTION_DAYS" ]; then
fi fi
if [ -d "$BACKUP_ARCHIVE" ]; then if [ -d "$BACKUP_ARCHIVE" ]; then
info "Pruning old backups from local archive" info "Pruning old backups from local archive"
prune "$BACKUP_ARCHIVE" prune "$BACKUP_ARCHIVE" "local"
fi fi
fi fi

View File

@ -25,6 +25,10 @@ services:
AWS_S3_BUCKET_NAME: backup AWS_S3_BUCKET_NAME: backup
BACKUP_FILENAME: test.tar.gz BACKUP_FILENAME: test.tar.gz
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
BACKUP_FORCE_PRUNE: ${BACKUP_FORCE_PRUNE:-}
BACKUP_PRUNING_LEEWAY: 5s
BACKUP_PRUNING_PREFIX: test
volumes: volumes:
- ./local:/archive - ./local:/archive
- app_data:/backup/app_data:ro - app_data:/backup/app_data:ro

View File

@ -29,4 +29,27 @@ fi
echo "[TEST:PASS] All containers running post backup." echo "[TEST:PASS] All containers running post backup."
docker-compose down
# The second part of this test checks if backups get deleted when the retention
# is set to 0 days (which it should not as it would mean all backups get deleted)
# TODO: find out if we can test actual deletion without having to wait for a day
BACKUP_RETENTION_DAYS="0" docker-compose up -d
sleep 5
docker-compose exec backup backup
docker run --rm -it \
-v compose_backup_data:/data alpine \
ash -c '[ $(find /data/backup/ -type f | wc -l) = "1" ]'
echo "[TEST:PASS] Remote backups have not been deleted."
if [ "$(find ./local -type f | wc -l)" != "1" ]; then
echo "[TEST:FAIL] Backups should not have been deleted, instead seen:"
find ./local -type f
fi
echo "[TEST:PASS] Local backups have not been deleted."
docker-compose down --volumes docker-compose down --volumes

View File

@ -33,6 +33,8 @@ services:
AWS_S3_BUCKET_NAME: backup AWS_S3_BUCKET_NAME: backup
BACKUP_FILENAME: test.tar.gz BACKUP_FILENAME: test.tar.gz
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_RETENTION_DAYS: 7
BACKUP_PRUNING_LEEWAY: 5s
volumes: volumes:
- pg_data:/backup/pg_data:ro - pg_data:/backup/pg_data:ro
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock