From 8a853a5b033b744eab6f770434b262f5f9ce84ea Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 6 Feb 2025 17:40:45 +0100 Subject: [PATCH] Update existing tests and test actual pruning --- cmd/backup/script.go | 2 +- docs/reference/index.md | 2 +- test/age-passphrase/docker-compose.yml | 2 +- test/age-publickey/docker-compose.yml | 2 +- test/azure/docker-compose.yml | 2 +- test/azure/run.sh | 4 ++-- test/certs/docker-compose.yml | 2 +- test/dropbox/docker-compose.yml | 2 +- test/dropbox/run.sh | 4 ++-- test/gpg-asym/docker-compose.yml | 2 +- test/gpg/docker-compose.yml | 2 +- test/local/docker-compose.yml | 2 +- test/local/run.sh | 4 ++-- test/lock/docker-compose.yml | 2 +- test/lock/run.sh | 2 +- test/pruning/docker-compose.yml | 2 +- test/retention/docker-compose.yml | 22 ++++++++++++++++++++ test/retention/run.sh | 28 ++++++++++++++++++++++++++ test/s3/docker-compose.yml | 2 +- test/s3/run.sh | 6 ++++-- test/secrets/docker-compose.yml | 2 +- test/services/docker-compose.yml | 2 +- test/ssh/docker-compose.yml | 2 +- test/ssh/run.sh | 4 ++-- test/swarm/docker-compose.yml | 2 +- test/webdav/docker-compose.yml | 2 +- test/webdav/run.sh | 4 ++-- 27 files changed, 83 insertions(+), 31 deletions(-) create mode 100644 test/retention/docker-compose.yml create mode 100755 test/retention/run.sh diff --git a/cmd/backup/script.go b/cmd/backup/script.go index e9a1ed7..f00268c 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -225,7 +225,7 @@ func (s *script) init() error { s.storages = append(s.storages, dropboxBackend) } - if s.c.BackupRetentionDays > 0 && s.c.BackupRetentionPeriod > 0 { + if s.c.BackupRetentionDays > -1 && s.c.BackupRetentionPeriod > 0 { return errwrap.Wrap(nil, "both BACKUP_RETENTION_DAYS and BACKUP_RETENTION_PERIOD were configured, which are mutually exclusive") } diff --git a/docs/reference/index.md b/docs/reference/index.md index 6c43ff9..8558bba 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -313,7 +313,7 @@ You can populate below template according to your requirements and use it as you # Define this value to enable automatic rotation of old backups. The value # declares the duration for which a backup is kept. It is formatted as per -# https://pkg.go.dev/time#ParseDuration +# https://pkg.go.dev/time#ParseDuration, e.g. 1 day turns into `24h` # BACKUP_RETENTION_PERIOD="168h" diff --git a/test/age-passphrase/docker-compose.yml b/test/age-passphrase/docker-compose.yml index b47ac72..bf7dbfd 100644 --- a/test/age-passphrase/docker-compose.yml +++ b/test/age-passphrase/docker-compose.yml @@ -6,7 +6,7 @@ services: BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_FILENAME: test.tar.gz BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} AGE_PASSPHRASE: "Dance.0Tonight.Go.Typical" volumes: - ${LOCAL_DIR:-./local}:/archive diff --git a/test/age-publickey/docker-compose.yml b/test/age-publickey/docker-compose.yml index 0e30d09..9d47468 100644 --- a/test/age-publickey/docker-compose.yml +++ b/test/age-publickey/docker-compose.yml @@ -6,7 +6,7 @@ services: BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_FILENAME: test.tar.gz BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} AGE_PUBLIC_KEYS: "${BACKUP_AGE_PUBLIC_KEYS}" volumes: - ${LOCAL_DIR:-./local}:/archive diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index ab64d36..0989557 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -37,7 +37,7 @@ services: AZURE_STORAGE_ACCESS_TIER: Hot BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test volumes: diff --git a/test/azure/run.sh b/test/azure/run.sh index 4163c97..1dcd4be 100755 --- a/test/azure/run.sh +++ b/test/azure/run.sh @@ -38,7 +38,7 @@ rm "$LOCAL_DIR/test.tar.gz" # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="1s" docker compose up -d sleep 5 docker compose exec backup backup @@ -52,7 +52,7 @@ pass "Remote backups have not been deleted." # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create first backup with no prune" diff --git a/test/certs/docker-compose.yml b/test/certs/docker-compose.yml index aa0db37..87e9abb 100644 --- a/test/certs/docker-compose.yml +++ b/test/certs/docker-compose.yml @@ -26,7 +26,7 @@ services: AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt AWS_S3_BUCKET_NAME: backup BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s volumes: - app_data:/backup/app_data:ro diff --git a/test/dropbox/docker-compose.yml b/test/dropbox/docker-compose.yml index 77df75e..8507d84 100644 --- a/test/dropbox/docker-compose.yml +++ b/test/dropbox/docker-compose.yml @@ -30,7 +30,7 @@ services: BACKUP_FILENAME_EXPAND: 'true' BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test DROPBOX_ENDPOINT: http://openapi_mock:8080 diff --git a/test/dropbox/run.sh b/test/dropbox/run.sh index 8b9136f..0b4b5d8 100755 --- a/test/dropbox/run.sh +++ b/test/dropbox/run.sh @@ -29,7 +29,7 @@ fi # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="1s" docker compose up -d sleep 5 logs=$(docker compose exec -T backup backup) @@ -43,7 +43,7 @@ fi # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create second backup and prune" diff --git a/test/gpg-asym/docker-compose.yml b/test/gpg-asym/docker-compose.yml index 097d030..3030535 100644 --- a/test/gpg-asym/docker-compose.yml +++ b/test/gpg-asym/docker-compose.yml @@ -6,7 +6,7 @@ services: BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_FILENAME: test.tar.gz BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} GPG_PUBLIC_KEY_RING_FILE: /keys/public_key.asc volumes: - ${KEY_DIR:-.}/public_key.asc:/keys/public_key.asc diff --git a/test/gpg/docker-compose.yml b/test/gpg/docker-compose.yml index 2c4880d..fe75d89 100644 --- a/test/gpg/docker-compose.yml +++ b/test/gpg/docker-compose.yml @@ -6,7 +6,7 @@ services: BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_FILENAME: test.tar.gz BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} GPG_PASSPHRASE: 1234#$$ecret volumes: - ${LOCAL_DIR:-./local}:/archive diff --git a/test/local/docker-compose.yml b/test/local/docker-compose.yml index 9a5f2cd..3714832 100644 --- a/test/local/docker-compose.yml +++ b/test/local/docker-compose.yml @@ -8,7 +8,7 @@ services: BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test volumes: diff --git a/test/local/run.sh b/test/local/run.sh index 4117f7d..4376352 100755 --- a/test/local/run.sh +++ b/test/local/run.sh @@ -41,7 +41,7 @@ pass "Found symlink to latest version in local backup." # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="1s" docker compose up -d sleep 5 docker compose exec backup backup @@ -54,7 +54,7 @@ pass "Local backups have not been deleted." # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create first backup with no prune" diff --git a/test/lock/docker-compose.yml b/test/lock/docker-compose.yml index 5d6d34c..8e8eeee 100644 --- a/test/lock/docker-compose.yml +++ b/test/lock/docker-compose.yml @@ -4,7 +4,7 @@ services: restart: always environment: BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: '7' + BACKUP_RETENTION_PERIOD: 168h volumes: - app_data:/backup/app_data:ro - /var/run/docker.sock:/var/run/docker.sock diff --git a/test/lock/run.sh b/test/lock/run.sh index 45d2e90..0f6d5de 100755 --- a/test/lock/run.sh +++ b/test/lock/run.sh @@ -13,7 +13,7 @@ sleep 5 ec=0 -docker compose exec -e BACKUP_RETENTION_DAYS=7 -e BACKUP_FILENAME=test.tar.gz backup backup & \ +docker compose exec -e BACKUP_RETENTION_PERIOD=168h -e BACKUP_FILENAME=test.tar.gz backup backup & \ { set +e; sleep 0.1; docker compose exec -e BACKUP_FILENAME=test2.tar.gz -e LOCK_TIMEOUT=1s backup backup; ec=$?;} if [ "$ec" = "0" ]; then diff --git a/test/pruning/docker-compose.yml b/test/pruning/docker-compose.yml index 08e7809..9c15e93 100644 --- a/test/pruning/docker-compose.yml +++ b/test/pruning/docker-compose.yml @@ -25,7 +25,7 @@ services: BACKUP_FILENAME_EXPAND: 'true' BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: 7 + BACKUP_RETENTION_PERIOD: 168h BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz diff --git a/test/retention/docker-compose.yml b/test/retention/docker-compose.yml new file mode 100644 index 0000000..2dcd9e0 --- /dev/null +++ b/test/retention/docker-compose.yml @@ -0,0 +1,22 @@ +services: + backup: + image: offen/docker-volume-backup:${TEST_VERSION:-canary} + restart: always + environment: + BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? + BACKUP_RETENTION_PERIOD: 15s + BACKUP_PRUNING_LEEWAY: 1s + volumes: + - app_data:/backup/app_data:ro + - /var/run/docker.sock:/var/run/docker.sock + - ${LOCAL_DIR:-./local}:/archive + + offen: + image: offen/offen:latest + labels: + - docker-volume-backup.stop-during-backup=true + volumes: + - app_data:/var/opt/offen + +volumes: + app_data: diff --git a/test/retention/run.sh b/test/retention/run.sh new file mode 100755 index 0000000..5e80c4f --- /dev/null +++ b/test/retention/run.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" +. ../util.sh +current_test=$(basename $(pwd)) + +export LOCAL_DIR=$(mktemp -d) + +docker compose up -d --quiet-pull +sleep 5 + +docker compose exec backup backup + +sleep 20 + +if [ $(ls -1 $LOCAL_DIR | wc -l) != "1" ]; then + fail "Unexpected number of backups after initial run" +fi +pass "Found 1 backup files." + +docker compose exec backup backup + +if [ $(ls -1 $LOCAL_DIR | wc -l) != "1" ]; then + fail "Unexpected number of backups after initial run" +fi +pass "Found 1 backup files." diff --git a/test/s3/docker-compose.yml b/test/s3/docker-compose.yml index 1b8bbac..59a1257 100644 --- a/test/s3/docker-compose.yml +++ b/test/s3/docker-compose.yml @@ -25,7 +25,7 @@ services: BACKUP_FILENAME_EXPAND: 'true' BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test volumes: diff --git a/test/s3/run.sh b/test/s3/run.sh index 4fc5ecd..5603a00 100755 --- a/test/s3/run.sh +++ b/test/s3/run.sh @@ -22,9 +22,11 @@ docker run --rm \ pass "Found relevant files in untared remote backups." +sleep 5 + # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="5s" docker compose up -d sleep 5 docker compose exec backup backup @@ -39,7 +41,7 @@ pass "Remote backups have not been deleted." # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create first backup with no prune" diff --git a/test/secrets/docker-compose.yml b/test/secrets/docker-compose.yml index c1d237d..5565af0 100644 --- a/test/secrets/docker-compose.yml +++ b/test/secrets/docker-compose.yml @@ -31,7 +31,7 @@ services: AWS_S3_BUCKET_NAME: backup BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: 7 + BACKUP_RETENTION_PERIOD: 168h BACKUP_PRUNING_LEEWAY: 5s volumes: - pg_data:/backup/pg_data:ro diff --git a/test/services/docker-compose.yml b/test/services/docker-compose.yml index a4e1cd4..529278a 100644 --- a/test/services/docker-compose.yml +++ b/test/services/docker-compose.yml @@ -25,7 +25,7 @@ services: AWS_S3_BUCKET_NAME: backup BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: 7 + BACKUP_RETENTION_PERIOD: 168h BACKUP_PRUNING_LEEWAY: 5s volumes: - pg_data:/backup/pg_data:ro diff --git a/test/ssh/docker-compose.yml b/test/ssh/docker-compose.yml index e31a7d0..1438c0a 100644 --- a/test/ssh/docker-compose.yml +++ b/test/ssh/docker-compose.yml @@ -19,7 +19,7 @@ services: BACKUP_FILENAME_EXPAND: 'true' BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test SSH_HOST_NAME: ssh diff --git a/test/ssh/run.sh b/test/ssh/run.sh index 6e40a03..6c0c885 100755 --- a/test/ssh/run.sh +++ b/test/ssh/run.sh @@ -28,7 +28,7 @@ pass "Found relevant files in decrypted and untared remote backups." # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="1s" docker compose up -d sleep 5 docker compose exec backup backup @@ -43,7 +43,7 @@ pass "Remote backups have not been deleted." # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create first backup with no prune" diff --git a/test/swarm/docker-compose.yml b/test/swarm/docker-compose.yml index 4a2b888..28d6098 100644 --- a/test/swarm/docker-compose.yml +++ b/test/swarm/docker-compose.yml @@ -31,7 +31,7 @@ services: AWS_S3_BUCKET_NAME: backup BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: 7 + BACKUP_RETENTION_PERIOD: 168h BACKUP_PRUNING_LEEWAY: 5s volumes: - pg_data:/backup/pg_data:ro diff --git a/test/webdav/docker-compose.yml b/test/webdav/docker-compose.yml index 69eb0a8..fdbe4ef 100644 --- a/test/webdav/docker-compose.yml +++ b/test/webdav/docker-compose.yml @@ -18,7 +18,7 @@ services: BACKUP_FILENAME_EXPAND: 'true' BACKUP_FILENAME: test-$$HOSTNAME.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? - BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_RETENTION_PERIOD: ${BACKUP_RETENTION_PERIOD:-168h} BACKUP_PRUNING_LEEWAY: 5s BACKUP_PRUNING_PREFIX: test WEBDAV_URL: http://webdav/ diff --git a/test/webdav/run.sh b/test/webdav/run.sh index 1665817..0863953 100755 --- a/test/webdav/run.sh +++ b/test/webdav/run.sh @@ -24,7 +24,7 @@ pass "Found relevant files in untared remote backup." # The second part of this test checks if backups get deleted when the retention # is set to 0 days (which it should not as it would mean all backups get deleted) -BACKUP_RETENTION_DAYS="0" docker compose up -d +BACKUP_RETENTION_PERIOD="1s" docker compose up -d sleep 5 docker compose exec backup backup @@ -39,7 +39,7 @@ pass "Remote backups have not been deleted." # The third part of this test checks if old backups get deleted when the retention # is set to 7 days (which it should) -BACKUP_RETENTION_DAYS="7" docker compose up -d +BACKUP_RETENTION_PERIOD="168h" docker compose up -d sleep 5 info "Create first backup with no prune"